[ML] Fix Index data visualizer reaching Elasticsearch rate request li…

…mits (#124898)

* Add pagination fetching, need fix on table sorting callback

* Reset fetch state

* Fix sorting

* Add max concurrent reqs to overall stats

* Add loading spinners, clean up

* Fix onTableChange

* [ML] Fix field stats missing

* Fix loading when show empty fields on

* Fix loading spinner for numeric fields

* Change switch map to map

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/data_visualizer/common/types/field_request_config.ts

@@ -17,6 +17,7 @@ export interface FieldRequestConfig {
   fieldName: string;
   type: JobFieldType;
   cardinality: number;
+  existsInDocs: boolean;
 }
 
 export interface DocumentCountBuckets {

x-pack/plugins/data_visualizer/public/application/common/components/fields_stats_grid/filter_fields.ts

@@ -6,15 +6,15 @@
  */
 
 import { JOB_FIELD_TYPES } from '../../../../../common/constants';
-import type {
-  FileBasedFieldVisConfig,
-  FileBasedUnknownFieldVisConfig,
-} from '../../../../../common/types/field_vis_config';
 
-export function filterFields(
-  fields: Array<FileBasedFieldVisConfig | FileBasedUnknownFieldVisConfig>,
-  visibleFieldNames: string[],
-  visibleFieldTypes: string[]
+interface CommonFieldConfig {
+  type: string;
+  fieldName?: string;
+}
+export function filterFields<T extends CommonFieldConfig>(
+  fields: T[],
+  visibleFieldNames: string[] | undefined,
+  visibleFieldTypes: string[] | undefined
 ) {
   let items = fields;
 

x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx

@@ -19,6 +19,7 @@ import {
   LEFT_ALIGNMENT,
   RIGHT_ALIGNMENT,
   EuiResizeObserver,
+  EuiLoadingSpinner,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { EuiTableComputedColumnType } from '@elastic/eui/src/components/basic_table/table_types';
@@ -279,6 +280,15 @@ export const DataVisualizerTable = <T extends DataVisualizerTableItem>({
         ),
         render: (item: DataVisualizerTableItem) => {
           if (item === undefined || showDistributions === false) return null;
+
+          if ('loading' in item && item.loading === true) {
+            return (
+              <EuiText textAlign="center">
+                <EuiLoadingSpinner size="s" />
+              </EuiText>
+            );
+          }
+
           if (
             (item.type === JOB_FIELD_TYPES.KEYWORD || item.type === JOB_FIELD_TYPES.IP) &&
             item.stats?.topValues !== undefined

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/index_data_visualizer_viewer.ts

@@ -6,3 +6,5 @@
  */
 
 export const DATA_VISUALIZER_INDEX_VIEWER = 'DATA_VISUALIZER_INDEX_VIEWER';
+
+export const MAX_CONCURRENT_REQUESTS = 10;

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx

@@ -84,6 +84,7 @@ export const EmbeddableWrapper = ({
     },
     [dataVisualizerListState, onOutputChange]
   );
+
   const { configs, searchQueryLanguage, searchString, extendedColumns, progress, setLastRefresh } =
     useDataVisualizerGridData(input, dataVisualizerListState);
 

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts

@@ -53,7 +53,6 @@ export const useDataVisualizerGridData = (
   const { services } = useDataVisualizerKibana();
   const { uiSettings, data } = services;
   const { samplerShardSize, visibleFieldTypes, showEmptyFields } = dataVisualizerListState;
-  const dataVisualizerListStateRef = useRef(dataVisualizerListState);
 
   const [lastRefresh, setLastRefresh] = useState(0);
   const searchSessionId = input.sessionId;
@@ -227,11 +226,11 @@ export const useDataVisualizerGridData = (
     if (overallStatsProgress.loaded < 100) return;
     const existMetricFields = metricConfigs
       .map((config) => {
-        if (config.existsInDocs === false) return;
         return {
           fieldName: config.fieldName,
           type: config.type,
           cardinality: config.stats?.cardinality ?? 0,
+          existsInDocs: config.existsInDocs,
         };
       })
       .filter((c) => c !== undefined) as FieldRequestConfig[];
@@ -240,11 +239,11 @@ export const useDataVisualizerGridData = (
     // Top values will be obtained on a sample if cardinality > 100000.
     const existNonMetricFields: FieldRequestConfig[] = nonMetricConfigs
       .map((config) => {
-        if (config.existsInDocs === false) return;
         return {
           fieldName: config.fieldName,
           type: config.type,
           cardinality: config.stats?.cardinality ?? 0,
+          existsInDocs: config.existsInDocs,
         };
       })
       .filter((c) => c !== undefined) as FieldRequestConfig[];
@@ -255,7 +254,7 @@ export const useDataVisualizerGridData = (
   const strategyResponse = useFieldStatsSearchStrategy(
     fieldStatsRequest,
     configsWithoutStats,
-    dataVisualizerListStateRef.current
+    dataVisualizerListState
   );
 
   const combinedProgress = useMemo(
@@ -325,7 +324,7 @@ export const useDataVisualizerGridData = (
         ...fieldData,
         fieldFormat: currentDataView.getFormatterForField(field),
         type: JOB_FIELD_TYPES.NUMBER,
-        loading: true,
+        loading: fieldData?.existsInDocs ?? true,
         aggregatable: true,
         deletable: field.runtimeField !== undefined,
       };
@@ -436,41 +435,46 @@ export const useDataVisualizerGridData = (
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [overallStats, showEmptyFields]);
 
-  const configs = useMemo(() => {
-    const fieldStats = strategyResponse.fieldStats;
-    let combinedConfigs = [...nonMetricConfigs, ...metricConfigs];
-    if (visibleFieldTypes && visibleFieldTypes.length > 0) {
-      combinedConfigs = combinedConfigs.filter(
-        (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1
-      );
-    }
-    if (visibleFieldNames && visibleFieldNames.length > 0) {
-      combinedConfigs = combinedConfigs.filter(
-        (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1
-      );
-    }
-
-    if (fieldStats) {
-      combinedConfigs = combinedConfigs.map((c) => {
-        const loadedFullStats = fieldStats.get(c.fieldName) ?? {};
-        return loadedFullStats
-          ? {
-              ...c,
-              loading: false,
-              stats: { ...c.stats, ...loadedFullStats },
-            }
-          : c;
-      });
-    }
+  const configs = useMemo(
+    () => {
+      const fieldStats = strategyResponse.fieldStats;
+      let combinedConfigs = [...nonMetricConfigs, ...metricConfigs];
+      if (visibleFieldTypes && visibleFieldTypes.length > 0) {
+        combinedConfigs = combinedConfigs.filter(
+          (config) => visibleFieldTypes.findIndex((field) => field === config.type) > -1
+        );
+      }
+      if (visibleFieldNames && visibleFieldNames.length > 0) {
+        combinedConfigs = combinedConfigs.filter(
+          (config) => visibleFieldNames.findIndex((field) => field === config.fieldName) > -1
+        );
+      }
 
-    return combinedConfigs;
-  }, [
-    nonMetricConfigs,
-    metricConfigs,
-    visibleFieldTypes,
-    visibleFieldNames,
-    strategyResponse.fieldStats,
-  ]);
+      if (fieldStats) {
+        combinedConfigs = combinedConfigs.map((c) => {
+          const loadedFullStats = fieldStats.get(c.fieldName) ?? {};
+          return loadedFullStats
+            ? {
+                ...c,
+                loading: false,
+                stats: { ...c.stats, ...loadedFullStats },
+              }
+            : c;
+        });
+      }
+      return combinedConfigs;
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [
+      nonMetricConfigs,
+      metricConfigs,
+      visibleFieldTypes,
+      visibleFieldNames,
+      strategyResponse.progress.loaded,
+      dataVisualizerListState.pageIndex,
+      dataVisualizerListState.pageSize,
+    ]
+  );
 
   // Some actions open up fly-out or popup
   // This variable is used to keep track of them and clean up when unmounting