[Maps] Add super-fine option to grid/cluster layer (#78201)

Adds a super-fine option for grids and cluster layers. This uses the .mvt tile format to deliver the data.

x-pack/plugins/maps/common/constants.ts

@@ -37,6 +37,7 @@ export const FONTS_API_PATH = `${GIS_API_PATH}/fonts`;
 export const API_ROOT_PATH = `/${GIS_API_PATH}`;
 
 export const MVT_GETTILE_API_PATH = 'mvt/getTile';
+export const MVT_GETGRIDTILE_API_PATH = 'mvt/getGridTile';
 export const MVT_SOURCE_LAYER_NAME = 'source_layer';
 export const KBN_TOO_MANY_FEATURES_PROPERTY = '__kbn_too_many_features__';
 export const KBN_TOO_MANY_FEATURES_IMAGE_ID = '__kbn_too_many_features_image_id__';
@@ -165,8 +166,13 @@ export enum GRID_RESOLUTION {
   COARSE = 'COARSE',
   FINE = 'FINE',
   MOST_FINE = 'MOST_FINE',
+  SUPER_FINE = 'SUPER_FINE',
 }
 
+export const SUPER_FINE_ZOOM_DELTA = 7; // (2 ^ SUPER_FINE_ZOOM_DELTA) ^ 2 =  number of cells in a given tile
+export const GEOTILE_GRID_AGG_NAME = 'gridSplit';
+export const GEOCENTROID_AGG_NAME = 'gridCentroid';
+
 export const TOP_TERM_PERCENTAGE_SUFFIX = '__percentage';
 
 export const COUNT_PROP_LABEL = i18n.translate('xpack.maps.aggs.defaultCountLabel', {
@@ -230,8 +236,6 @@ export enum SCALING_TYPES {
   MVT = 'MVT',
 }
 
-export const RGBA_0000 = 'rgba(0,0,0,0)';
-
 export enum MVT_FIELD_TYPE {
   STRING = 'String',
   NUMBER = 'Number',

x-pack/plugins/maps/common/elasticsearch_util/convert_to_geojson.d.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Feature } from 'geojson';
+import { RENDER_AS } from '../constants';
+
+export function convertCompositeRespToGeoJson(esResponse: any, renderAs: RENDER_AS): Feature[];
+export function convertRegularRespToGeoJson(esResponse: any, renderAs: RENDER_AS): Feature[];

x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/convert_to_geojson.js → x-pack/plugins/maps/common/elasticsearch_util/convert_to_geojson.js

@@ -5,12 +5,12 @@
  */
 
 import _ from 'lodash';
-import { RENDER_AS } from '../../../../common/constants';
-import { getTileBoundingBox } from './geo_tile_utils';
-import { extractPropertiesFromBucket } from '../../util/es_agg_utils';
-import { clamp } from '../../../../common/elasticsearch_geo_utils';
+import { RENDER_AS, GEOTILE_GRID_AGG_NAME, GEOCENTROID_AGG_NAME } from '../constants';
+import { getTileBoundingBox } from '../geo_tile_utils';
+import { extractPropertiesFromBucket } from './es_agg_utils';
+import { clamp } from './elasticsearch_geo_utils';
 
-const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid'];
+const GRID_BUCKET_KEYS_TO_IGNORE = ['key', GEOCENTROID_AGG_NAME];
 
 export function convertCompositeRespToGeoJson(esResponse, renderAs) {
   return convertToGeoJson(
@@ -20,7 +20,7 @@ export function convertCompositeRespToGeoJson(esResponse, renderAs) {
       return _.get(esResponse, 'aggregations.compositeSplit.buckets', []);
     },
     (gridBucket) => {
-      return gridBucket.key.gridSplit;
+      return gridBucket.key[GEOTILE_GRID_AGG_NAME];
     }
   );
 }
@@ -30,7 +30,7 @@ export function convertRegularRespToGeoJson(esResponse, renderAs) {
     esResponse,
     renderAs,
     (esResponse) => {
-      return _.get(esResponse, 'aggregations.gridSplit.buckets', []);
+      return _.get(esResponse, `aggregations.${GEOTILE_GRID_AGG_NAME}.buckets`, []);
     },
     (gridBucket) => {
       return gridBucket.key;
@@ -49,7 +49,7 @@ function convertToGeoJson(esResponse, renderAs, pluckGridBuckets, pluckGridKey)
       type: 'Feature',
       geometry: rowToGeometry({
         gridKey,
-        gridCentroid: gridBucket.gridCentroid,
+        [GEOCENTROID_AGG_NAME]: gridBucket[GEOCENTROID_AGG_NAME],
         renderAs,
       }),
       id: gridKey,

x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/convert_to_geojson.test.ts → x-pack/plugins/maps/common/elasticsearch_util/convert_to_geojson.test.ts

@@ -4,11 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-jest.mock('../../../kibana_services', () => {});
-
 // @ts-ignore
 import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson';
-import { RENDER_AS } from '../../../../common/constants';
+import { RENDER_AS } from '../constants';
 
 describe('convertCompositeRespToGeoJson', () => {
   const esResponse = {

x-pack/plugins/maps/common/elasticsearch_geo_utils.d.ts → x-pack/plugins/maps/common/elasticsearch_util/elasticsearch_geo_utils.d.ts

@@ -5,8 +5,8 @@
  */
 
 import { FeatureCollection, GeoJsonProperties } from 'geojson';
-import { MapExtent } from './descriptor_types';
-import { ES_GEO_FIELD_TYPE } from './constants';
+import { MapExtent } from '../descriptor_types';
+import { ES_GEO_FIELD_TYPE } from '../constants';
 
 export function scaleBounds(bounds: MapExtent, scaleFactor: number): MapExtent;
 

x-pack/plugins/maps/common/elasticsearch_geo_utils.js → x-pack/plugins/maps/common/elasticsearch_util/elasticsearch_geo_utils.js

@@ -15,9 +15,9 @@ import {
   POLYGON_COORDINATES_EXTERIOR_INDEX,
   LON_INDEX,
   LAT_INDEX,
-} from '../common/constants';
-import { getEsSpatialRelationLabel } from './i18n_getters';
-import { FILTERS } from '../../../../src/plugins/data/common';
+} from '../constants';
+import { getEsSpatialRelationLabel } from '../i18n_getters';
+import { FILTERS } from '../../../../../src/plugins/data/common';
 import turfCircle from '@turf/circle';
 
 const SPATIAL_FILTER_TYPE = FILTERS.SPATIAL_FILTER;

x-pack/plugins/maps/public/classes/util/es_agg_utils.ts → x-pack/plugins/maps/common/elasticsearch_util/es_agg_utils.ts

@@ -5,8 +5,8 @@
  */
 import { i18n } from '@kbn/i18n';
 import _ from 'lodash';
-import { IndexPattern, IFieldType } from '../../../../../../src/plugins/data/public';
-import { TOP_TERM_PERCENTAGE_SUFFIX } from '../../../common/constants';
+import { IndexPattern, IFieldType } from '../../../../../src/plugins/data/common';
+import { TOP_TERM_PERCENTAGE_SUFFIX } from '../constants';
 
 export function getField(indexPattern: IndexPattern, fieldName: string) {
   const field = indexPattern.fields.getByName(fieldName);

x-pack/plugins/maps/common/elasticsearch_util/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './es_agg_utils';
+export * from './convert_to_geojson';
+export * from './elasticsearch_geo_utils';

x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/geo_tile_utils.test.js → x-pack/plugins/maps/common/geo_tile_utils.test.js

@@ -4,8 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-jest.mock('../../../kibana_services', () => {});
-
 import { parseTileKey, getTileBoundingBox, expandToTileBoundaries } from './geo_tile_utils';
 
 it('Should parse tile key', () => {

x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/geo_tile_utils.js → x-pack/plugins/maps/common/geo_tile_utils.ts

@@ -5,18 +5,32 @@
  */
 
 import _ from 'lodash';
-import { DECIMAL_DEGREES_PRECISION } from '../../../../common/constants';
-import { clampToLatBounds } from '../../../../common/elasticsearch_geo_utils';
+import { DECIMAL_DEGREES_PRECISION } from './constants';
+import { clampToLatBounds } from './elasticsearch_util';
+import { MapExtent } from './descriptor_types';
 
 const ZOOM_TILE_KEY_INDEX = 0;
 const X_TILE_KEY_INDEX = 1;
 const Y_TILE_KEY_INDEX = 2;
 
-function getTileCount(zoom) {
+function getTileCount(zoom: number): number {
   return Math.pow(2, zoom);
 }
 
-export function parseTileKey(tileKey) {
+export interface ESBounds {
+  top_left: {
+    lon: number;
+    lat: number;
+  };
+  bottom_right: {
+    lon: number;
+    lat: number;
+  };
+}
+
+export function parseTileKey(
+  tileKey: string
+): { x: number; y: number; zoom: number; tileCount: number } {
   const tileKeyParts = tileKey.split('/');
 
   if (tileKeyParts.length !== 3) {
@@ -42,7 +56,7 @@ export function parseTileKey(tileKey) {
   return { x, y, zoom, tileCount };
 }
 
-function sinh(x) {
+function sinh(x: number): number {
   return (Math.exp(x) - Math.exp(-x)) / 2;
 }
 
@@ -55,24 +69,52 @@ function sinh(x) {
 // We add one extra decimal level of precision because, at high zoom
 // levels rounding exactly can cause the boxes to render as uneven sizes
 // (some will be slightly larger and some slightly smaller)
-function precisionRounding(v, minPrecision, binSize) {
+function precisionRounding(v: number, minPrecision: number, binSize: number): number {
   let precision = Math.ceil(Math.abs(Math.log10(binSize))) + 1;
   precision = Math.max(precision, minPrecision);
   return _.round(v, precision);
 }
 
-function tileToLatitude(y, tileCount) {
+export function tile2long(x: number, z: number): number {
+  const tileCount = getTileCount(z);
+  return tileToLongitude(x, tileCount);
+}
+
+export function tile2lat(y: number, z: number): number {
+  const tileCount = getTileCount(z);
+  return tileToLatitude(y, tileCount);
+}
+
+export function tileToESBbox(x: number, y: number, z: number): ESBounds {
+  const wLon = tile2long(x, z);
+  const sLat = tile2lat(y + 1, z);
+  const eLon = tile2long(x + 1, z);
+  const nLat = tile2lat(y, z);
+
+  return {
+    top_left: {
+      lon: wLon,
+      lat: nLat,
+    },
+    bottom_right: {
+      lon: eLon,
+      lat: sLat,
+    },
+  };
+}
+
+export function tileToLatitude(y: number, tileCount: number) {
   const radians = Math.atan(sinh(Math.PI - (2 * Math.PI * y) / tileCount));
   const lat = (180 / Math.PI) * radians;
   return precisionRounding(lat, DECIMAL_DEGREES_PRECISION, 180 / tileCount);
 }
 
-function tileToLongitude(x, tileCount) {
+export function tileToLongitude(x: number, tileCount: number) {
   const lon = (x / tileCount) * 360 - 180;
   return precisionRounding(lon, DECIMAL_DEGREES_PRECISION, 360 / tileCount);
 }
 
-export function getTileBoundingBox(tileKey) {
+export function getTileBoundingBox(tileKey: string) {
   const { x, y, tileCount } = parseTileKey(tileKey);
 
   return {
@@ -83,22 +125,22 @@ export function getTileBoundingBox(tileKey) {
   };
 }
 
-function sec(value) {
+function sec(value: number): number {
   return 1 / Math.cos(value);
 }
 
-function latitudeToTile(lat, tileCount) {
+function latitudeToTile(lat: number, tileCount: number) {
   const radians = (clampToLatBounds(lat) * Math.PI) / 180;
   const y = ((1 - Math.log(Math.tan(radians) + sec(radians)) / Math.PI) / 2) * tileCount;
   return Math.floor(y);
 }
 
-function longitudeToTile(lon, tileCount) {
+function longitudeToTile(lon: number, tileCount: number) {
   const x = ((lon + 180) / 360) * tileCount;
   return Math.floor(x);
 }
 
-export function expandToTileBoundaries(extent, zoom) {
+export function expandToTileBoundaries(extent: MapExtent, zoom: number): MapExtent {
   const tileCount = getTileCount(zoom);
 
   const upperLeftX = longitudeToTile(extent.minLon, tileCount);

x-pack/plugins/maps/public/actions/data_request_actions.ts

@@ -40,7 +40,7 @@ import { ILayer } from '../classes/layers/layer';
 import { IVectorLayer } from '../classes/layers/vector_layer/vector_layer';
 import { DataMeta, MapExtent, MapFilters } from '../../common/descriptor_types';
 import { DataRequestAbortError } from '../classes/util/data_request';
-import { scaleBounds, turfBboxToBounds } from '../../common/elasticsearch_geo_utils';
+import { scaleBounds, turfBboxToBounds } from '../../common/elasticsearch_util';
 import { IVectorStyle } from '../classes/styles/vector/vector_style';
 
 const FIT_TO_BOUNDS_SCALE_FACTOR = 0.1;

x-pack/plugins/maps/public/actions/map_actions.ts

@@ -54,7 +54,7 @@ import {
   MapRefreshConfig,
 } from '../../common/descriptor_types';
 import { INITIAL_LOCATION } from '../../common/constants';
-import { scaleBounds } from '../../common/elasticsearch_geo_utils';
+import { scaleBounds } from '../../common/elasticsearch_util';
 
 export function setMapInitError(errorMessage: string) {
   return {

x-pack/plugins/maps/public/classes/fields/es_agg_field.ts

@@ -12,7 +12,7 @@ import { IVectorSource } from '../sources/vector_source';
 import { ESDocField } from './es_doc_field';
 import { AGG_TYPE, FIELD_ORIGIN } from '../../../common/constants';
 import { isMetricCountable } from '../util/is_metric_countable';
-import { getField, addFieldToDSL } from '../util/es_agg_utils';
+import { getField, addFieldToDSL } from '../../../common/elasticsearch_util';
 import { TopTermPercentageField } from './top_term_percentage_field';
 import { ITooltipProperty, TooltipProperty } from '../tooltips/tooltip_property';
 import { ESAggTooltipProperty } from '../tooltips/es_agg_tooltip_property';
@@ -30,25 +30,29 @@ export class ESAggField implements IESAggField {
   private readonly _label?: string;
   private readonly _aggType: AGG_TYPE;
   private readonly _esDocField?: IField | undefined;
+  private readonly _canReadFromGeoJson: boolean;
 
   constructor({
     label,
     source,
     aggType,
     esDocField,
     origin,
+    canReadFromGeoJson = true,
   }: {
     label?: string;
     source: IESAggSource;
     aggType: AGG_TYPE;
     esDocField?: IField;
     origin: FIELD_ORIGIN;
+    canReadFromGeoJson?: boolean;
   }) {
     this._source = source;
     this._origin = origin;
     this._label = label;
     this._aggType = aggType;
     this._esDocField = esDocField;
+    this._canReadFromGeoJson = canReadFromGeoJson;
   }
 
   getSource(): IVectorSource {
@@ -132,18 +136,19 @@ export class ESAggField implements IESAggField {
   }
 
   supportsAutoDomain(): boolean {
-    return true;
+    return this._canReadFromGeoJson ? true : this.supportsFieldMeta();
   }
 
   canReadFromGeoJson(): boolean {
-    return true;
+    return this._canReadFromGeoJson;
   }
 }
 
 export function esAggFieldsFactory(
   aggDescriptor: AggDescriptor,
   source: IESAggSource,
-  origin: FIELD_ORIGIN
+  origin: FIELD_ORIGIN,
+  canReadFromGeoJson: boolean = true
 ): IESAggField[] {
   const aggField = new ESAggField({
     label: aggDescriptor.label,
@@ -153,12 +158,13 @@ export function esAggFieldsFactory(
     aggType: aggDescriptor.type,
     source,
     origin,
+    canReadFromGeoJson,
   });
 
   const aggFields: IESAggField[] = [aggField];
 
   if (aggDescriptor.field && aggDescriptor.type === AGG_TYPE.TERMS) {
-    aggFields.push(new TopTermPercentageField(aggField));
+    aggFields.push(new TopTermPercentageField(aggField, canReadFromGeoJson));
   }
 
   return aggFields;

x-pack/plugins/maps/public/classes/fields/field.ts

@@ -21,12 +21,12 @@ export interface IField {
   getOrdinalFieldMetaRequest(): Promise<unknown>;
   getCategoricalFieldMetaRequest(size: number): Promise<unknown>;
 
-  // Determines whether Maps-app can automatically determine the domain of the field-values
+  // Whether Maps-app can automatically determine the domain of the field-values
   // if this is not the case (e.g. for .mvt tiled data),
   // then styling properties that require the domain to be known cannot use this property.
   supportsAutoDomain(): boolean;
 
-  // Determinse wheter Maps-app can automatically deterime the domain of the field-values
+  // Whether Maps-app can automatically determine the domain of the field-values
   // _without_ having to retrieve the data as GeoJson
   // e.g. for ES-sources, this would use the extended_stats API
   supportsFieldMeta(): boolean;