[MQL] support different query languages (opensearch-project#6595)

* [MQL] support different query languages

Only adds the quick startup for OpenSearch cluster with a SQL
plugin and observability with:
```
yarn opensearch snapshot --sql
```

Also, adds SQL to the language selector stolen shamelessly from
opensearch-project#5623

Next steps to intercept and send to SQL API or just transform basic
syntax to DSL

Signed-off-by: Kawika Avilla <[email protected]>

Working query enhance

Signed-off-by: Kawika Avilla <[email protected]>

Reget search enhance

Signed-off-by: Kawika Avilla <[email protected]>

data frames

Signed-off-by: Kawika Avilla <[email protected]>

temp state need to pass df

Signed-off-by: Kawika Avilla <[email protected]>

sending data frame

Signed-off-by: Kawika Avilla <[email protected]>

gets df type

Signed-off-by: Kawika Avilla <[email protected]>

add some small ui things

Signed-off-by: Kawika Avilla <[email protected]>

move back query language switcher

Signed-off-by: Kawika Avilla <[email protected]>

updating side panel

Signed-off-by: Kawika Avilla <[email protected]>

add ui config on query enhancement

Signed-off-by: Kawika Avilla <[email protected]>

support query string input

Signed-off-by: Kawika Avilla <[email protected]>

update to say search bar

Signed-off-by: Kawika Avilla <[email protected]>

update ppl to call once

Signed-off-by: Kawika Avilla <[email protected]>

update when fields update

Signed-off-by: Kawika Avilla <[email protected]>

add unknown

Signed-off-by: Kawika Avilla <[email protected]>

* clean up

Signed-off-by: Kawika Avilla <[email protected]>

* more clean up

Signed-off-by: Kawika Avilla <[email protected]>

* clean up licenses

Signed-off-by: Kawika Avilla <[email protected]>

---------

Signed-off-by: Kawika Avilla <[email protected]>

Cleanup unused helper function

Signed-off-by: Kawika Avilla <[email protected]>

create index pattern somewhat working

Signed-off-by: Kawika Avilla <[email protected]>

config/opensearch_dashboards.yml

@@ -313,3 +313,5 @@
 
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
+opensearch_alerting.enabled: false
+opensearch_security.enabled: false

package.json

@@ -136,7 +136,7 @@
   },
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
-    "@elastic/datemath": "5.0.3",
+    "@elastic/datemath": "link:packages/opensearch-datemath",
     "@elastic/eui": "npm:@opensearch-project/[email protected]",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/[email protected]",

packages/opensearch-datemath/index.d.ts

@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

packages/opensearch-datemath/index.js

@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d) => moment.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

packages/osd-opensearch/src/cli_commands/snapshot.js

@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

packages/osd-opensearch/src/cluster.js

@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

src/cli/serve/serve.js

@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',

src/plugins/data/common/constants.ts

@@ -60,4 +60,5 @@ export const UI_SETTINGS = {
   INDEXPATTERN_PLACEHOLDER: 'indexPattern:placeholder',
   FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault',
   FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues',
+  DATAFRAME_HYDRATION_STRATEGY: 'dataframe:hydrationStrategy',
 } as const;

src/plugins/data/common/data_frames/_df_cache.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IDataFrame } from '..';
+
+export interface DfCache {
+  get: () => IDataFrame | undefined;
+  set: (value: IDataFrame) => IDataFrame;
+  clear: () => void;
+}
+
+export function createDataFrameCache(): DfCache {
+  let df: IDataFrame | undefined;
+  const cache: DfCache = {
+    get: () => {
+      return df;
+    },
+    set: (prom: IDataFrame) => {
+      df = prom;
+      return prom;
+    },
+    clear: () => {
+      df = undefined;
+    },
+  };
+  return cache;
+}

src/plugins/data/common/data_frames/fields/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './types';

src/plugins/data/common/data_frames/fields/types.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface IFieldType {
+  name: string;
+  type: string;
+  values: any[];
+  count?: number;
+  aggregatable?: boolean;
+  filterable?: boolean;
+  searchable?: boolean;
+  sortable?: boolean;
+  visualizable?: boolean;
+  displayName?: string;
+  format?: any;
+}

src/plugins/data/common/data_frames/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './types';
+export * from './utils';

src/plugins/data/common/data_frames/types.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IFieldType } from './fields';
+
+export * from './_df_cache';
+
+/** @public **/
+export enum DATA_FRAME_TYPES {
+  DEFAULT = 'data_frame',
+  POLLING = 'data_frame_polling',
+}
+
+export interface IDataFrame {
+  name?: string;
+  schema?: Array<Partial<IFieldType>>;
+  fields: IFieldType[];
+  size: number;
+}
+
+export interface DataFrameAgg {
+  key: string;
+  value: number;
+}
+
+export interface PartialDataFrame extends Omit<IDataFrame, 'fields' | 'size'> {
+  fields: Array<Partial<IFieldType>>;
+}
+
+/**
+ * To be utilize with aggregations and will map to buckets
+ * Plugins can get the aggreted value by their own logic
+ * Setting to null will disable the aggregation if plugin wishes
+ * In future, if the plugin doesn't intentionally set the value to null,
+ * we can calculate the value based on the fields.
+ */
+export interface IDataFrameWithAggs extends IDataFrame {
+  aggs: DataFrameAgg[] | null;
+}

src/plugins/data/common/data_frames/utils.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SearchResponse } from 'elasticsearch';
+import datemath from '@elastic/datemath';
+import { DATA_FRAME_TYPES, IDataFrame, IDataFrameWithAggs, PartialDataFrame } from './types';
+import { IFieldType } from './fields';
+import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns';
+import { IOpenSearchDashboardsSearchRequest } from '../search';
+
+export interface IDataFrameResponse extends SearchResponse<any> {
+  type: DATA_FRAME_TYPES;
+  body: IDataFrame | IDataFrameWithAggs;
+  took: number;
+}
+
+export const getRawQueryString = (
+  searchRequest: IOpenSearchDashboardsSearchRequest
+): string | undefined => {
+  return searchRequest.params?.body?.query?.queries[0]?.query;
+};
+
+export const convertResult = (response: IDataFrameResponse): SearchResponse<any> => {
+  const data = response.body;
+  const hits: any[] = [];
+  for (let index = 0; index < data.size; index++) {
+    const hit: { [key: string]: any } = {};
+    data.fields.forEach((field) => {
+      hit[field.name] = field.values[index];
+    });
+    hits.push({
+      _index: data.name ?? '',
+      _type: '',
+      _id: '',
+      _score: 0,
+      _source: hit,
+    });
+  }
+  const searchResponse: SearchResponse<any> = {
+    took: response.took,
+    timed_out: false,
+    _shards: {
+      total: 1,
+      successful: 1,
+      skipped: 0,
+      failed: 0,
+    },
+    hits: {
+      total: 0,
+      max_score: 0,
+      hits,
+    },
+  };
+
+  if (data.hasOwnProperty('aggs')) {
+    const dataWithAggs = data as IDataFrameWithAggs;
+    if (!dataWithAggs.aggs) {
+      // TODO: SQL best guess, get timestamp field and caculate it here
+      return searchResponse;
+    }
+    searchResponse.aggregations = {
+      2: {
+        buckets: dataWithAggs.aggs.map((agg) => {
+          searchResponse.hits.total += agg.value;
+          return {
+            key: new Date(agg.key).getTime(),
+            key_as_string: agg.key,
+            doc_count: agg.value,
+          };
+        }),
+      },
+    };
+  }
+
+  return searchResponse;
+};
+
+export const formatFieldValue = (field: IFieldType | Partial<IFieldType>, value: any): any => {
+  return field.format && field.format.convert ? field.format.convert(value) : value;
+};
+
+export const getFieldType = (field: IFieldType | Partial<IFieldType>): string | undefined => {
+  if (field.name) {
+    const fieldName = field.name.toLowerCase();
+    // TODO: feels little biased to check if timestamp.
+    // Has to be a better way to know so to be fair to all data sources
+    if (fieldName.includes('date') || fieldName.includes('timestamp')) {
+      return 'date';
+    }
+  }
+  if (!field.values) return field.type;
+  const firstValue = field.values.filter((value) => value !== null && value !== undefined)[0];
+  if (firstValue instanceof Date || datemath.isDateTime(firstValue)) {
+    return 'date';
+  }
+  return field.type;
+};
+
+export const getTimeField = (data: IDataFrame): IFieldType | undefined => {
+  return data.fields.find((field) => field.type === 'date');
+};
+
+export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => {
+  let size = 0;
+  const fields = partial.fields.map((field) => {
+    if (!field.values) {
+      field.values = new Array(size);
+    } else if (field.values.length > size) {
+      size = field.values.length;
+    }
+    field.type = getFieldType(field);
+    // if (!field.type) {
+    // need to think if this needs to be mapped to OSD field type for example
+    // PPL type for date is TIMESTAMP
+    // OSD is expecting date
+    //   field.type = get type
+    // }
+    // get timeseries field
+    return field as IFieldType;
+  });
+
+  return {
+    ...partial,
+    fields,
+    size,
+  };
+};
+
+export const dataFrameToSpec = (dataFrame: IDataFrame): IndexPatternSpec => {
+  return {
+    id: DATA_FRAME_TYPES.DEFAULT,
+    title: dataFrame.name,
+    timeFieldName: getTimeField(dataFrame)?.name,
+    fields: dataFrame.fields.reduce((acc, field) => {
+      acc[field.name] = {
+        name: field.name,
+        type: field.type,
+        aggregatable: true,
+        searchable: true,
+      };
+      return acc;
+    }, {} as IndexPatternFieldMap),
+    // TODO: SQL dataSourceRef
+  };
+};

src/plugins/data/common/index.ts

@@ -30,6 +30,7 @@
 
 export * from './constants';
 export * from './opensearch_query';
+export * from './data_frames';
 export * from './field_formats';
 export * from './field_mapping';
 export * from './index_patterns';