From b337bea2d13cc6c8330ba6f0326c51b53baad7f3 Mon Sep 17 00:00:00 2001 From: Kristen Tian <105667444+kristenTian@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:37:43 -0700 Subject: [PATCH] Enable data client with sample data server side (#4268) * Enable data client with sample data server side * Add dataSourceId into savedObject Signed-off-by: Kristen Tian * Functional list, install uninstall Signed-off-by: Kristen Tian * add change log Signed-off-by: Kristen Tian * address comments Signed-off-by: Kristen Tian * add ut Signed-off-by: Kristen Tian --------- Signed-off-by: Kristen Tian --- .lycheeexclude | 1 + CHANGELOG.md | 1 + .../server/client/configure_client.ts | 2 +- .../server/legacy/configure_legacy_client.ts | 2 +- src/plugins/home/opensearch_dashboards.json | 2 +- .../public/application/sample_data_client.js | 25 ++- .../sample_data/data_sets/ecommerce/index.ts | 11 +- .../sample_data/data_sets/flights/index.ts | 11 +- .../sample_data/data_sets/logs/index.ts | 11 +- .../services/sample_data/data_sets/util.ts | 84 +++++++++ .../lib/sample_dataset_registry_types.ts | 6 +- .../sample_data/routes/install.test.ts | 160 ++++++++++++++++++ .../services/sample_data/routes/install.ts | 60 +++++-- .../services/sample_data/routes/list.test.ts | 122 +++++++++++++ .../services/sample_data/routes/list.ts | 116 +++++++------ .../sample_data/routes/uninstall.test.ts | 101 +++++++++++ .../services/sample_data/routes/uninstall.ts | 31 ++-- .../sample_data/sample_data_registry.ts | 8 +- 18 files changed, 648 insertions(+), 106 deletions(-) create mode 100644 src/plugins/home/server/services/sample_data/data_sets/util.ts create mode 100644 src/plugins/home/server/services/sample_data/routes/install.test.ts create mode 100644 src/plugins/home/server/services/sample_data/routes/list.test.ts create mode 100644 src/plugins/home/server/services/sample_data/routes/uninstall.test.ts diff --git a/.lycheeexclude b/.lycheeexclude index 35ae861e8f9..bb378e30886 100644 --- a/.lycheeexclude +++ b/.lycheeexclude @@ -121,3 +121,4 @@ https://yarnpkg.com/latest.msi https://forum.opensearch.org/ https://facebook.github.io/jest/ https://facebook.github.io/jest/docs/cli.html +http://helpmenow.com/problem2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5afb7ee4b8a..081b56f4f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Replace re2 with RegExp in timeline and add unit tests ([#3908](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3908)) - Add category option within groups for context menus ([#4144](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4144)) - [Saved Object Service] Add Repository Factory Provider ([#4149](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4149)) +- [Multiple DataSource] Backend support for adding sample data ([#4268](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4268)) ### 🐛 Bug Fixes diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts index 77a0e067bbc..acbdfddb3fc 100644 --- a/src/plugins/data_source/server/client/configure_client.ts +++ b/src/plugins/data_source/server/client/configure_client.ts @@ -74,7 +74,7 @@ export const configureClient = async ( requireDecryption ); } catch (error: any) { - logger.error( + logger.debug( `Failed to get data source client for dataSourceId: [${dataSourceId}]. ${error}: ${error.stack}` ); // Re-throw as DataSourceError diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.ts index 0d074cf77d4..8341e844edc 100644 --- a/src/plugins/data_source/server/legacy/configure_legacy_client.ts +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.ts @@ -60,7 +60,7 @@ export const configureLegacyClient = async ( dataSourceId ); } catch (error: any) { - logger.error( + logger.debug( `Failed to get data source client for dataSourceId: [${dataSourceId}]. ${error}: ${error.stack}` ); // Re-throw as DataSourceError diff --git a/src/plugins/home/opensearch_dashboards.json b/src/plugins/home/opensearch_dashboards.json index 3cb42faaeec..35a81bc7adb 100644 --- a/src/plugins/home/opensearch_dashboards.json +++ b/src/plugins/home/opensearch_dashboards.json @@ -4,7 +4,7 @@ "server": true, "ui": true, "requiredPlugins": ["data", "urlForwarding"], - "optionalPlugins": ["usageCollection", "telemetry"], + "optionalPlugins": ["usageCollection", "telemetry", "dataSource"], "requiredBundles": [ "opensearchDashboardsReact" ] diff --git a/src/plugins/home/public/application/sample_data_client.js b/src/plugins/home/public/application/sample_data_client.js index bf30f516dfc..045736c428f 100644 --- a/src/plugins/home/public/application/sample_data_client.js +++ b/src/plugins/home/public/application/sample_data_client.js @@ -36,12 +36,14 @@ function clearIndexPatternsCache() { getServices().indexPatternService.clearCache(); } -export async function listSampleDataSets() { - return await getServices().http.get(sampleDataUrl); +export async function listSampleDataSets(dataSourceId) { + const query = buildQuery(dataSourceId); + return await getServices().http.get(sampleDataUrl, { query }); } -export async function installSampleDataSet(id, sampleDataDefaultIndex) { - await getServices().http.post(`${sampleDataUrl}/${id}`); +export async function installSampleDataSet(id, sampleDataDefaultIndex, dataSourceId) { + const query = buildQuery(dataSourceId); + await getServices().http.post(`${sampleDataUrl}/${id}`, { query }); if (getServices().uiSettings.isDefault('defaultIndex')) { getServices().uiSettings.set('defaultIndex', sampleDataDefaultIndex); @@ -50,8 +52,9 @@ export async function installSampleDataSet(id, sampleDataDefaultIndex) { clearIndexPatternsCache(); } -export async function uninstallSampleDataSet(id, sampleDataDefaultIndex) { - await getServices().http.delete(`${sampleDataUrl}/${id}`); +export async function uninstallSampleDataSet(id, sampleDataDefaultIndex, dataSourceId) { + const query = buildQuery(dataSourceId); + await getServices().http.delete(`${sampleDataUrl}/${id}`, { query }); const uiSettings = getServices().uiSettings; @@ -64,3 +67,13 @@ export async function uninstallSampleDataSet(id, sampleDataDefaultIndex) { clearIndexPatternsCache(); } + +function buildQuery(dataSourceId) { + const query = {}; + + if (dataSourceId) { + query.data_source_id = dataSourceId; + } + + return query; +} diff --git a/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts b/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts index 43cf0925a8b..0321c428cf6 100644 --- a/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts +++ b/src/plugins/home/server/services/sample_data/data_sets/ecommerce/index.ts @@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n'; import { getSavedObjects } from './saved_objects'; import { fieldMappings } from './field_mappings'; import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types'; +import { getSavedObjectsWithDataSource, appendDataSourceId } from '../util'; const ecommerceName = i18n.translate('home.sampleData.ecommerceSpecTitle', { defaultMessage: 'Sample eCommerce orders', @@ -42,6 +43,9 @@ const ecommerceDescription = i18n.translate('home.sampleData.ecommerceSpecDescri }); const initialAppLinks = [] as AppLinkSchema[]; +const DEFAULT_INDEX = 'ff959d40-b880-11e8-a6d9-e546fe2bba5f'; +const DASHBOARD_ID = '722b74f0-b882-11e8-a6d9-e546fe2bba5f'; + export const ecommerceSpecProvider = function (): SampleDatasetSchema { return { id: 'ecommerce', @@ -49,10 +53,11 @@ export const ecommerceSpecProvider = function (): SampleDatasetSchema { description: ecommerceDescription, previewImagePath: '/plugins/home/assets/sample_data_resources/ecommerce/dashboard.png', darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/ecommerce/dashboard_dark.png', - overviewDashboard: '722b74f0-b882-11e8-a6d9-e546fe2bba5f', + overviewDashboard: appendDataSourceId(DASHBOARD_ID), appLinks: initialAppLinks, - defaultIndex: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f', - savedObjects: getSavedObjects(), + defaultIndex: appendDataSourceId(DEFAULT_INDEX), + savedObjects: (dataSourceId?: string, dataSourceTitle?: string) => + getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle), dataIndices: [ { id: 'ecommerce', diff --git a/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts b/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts index e90cdaea3b8..038744419f3 100644 --- a/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts +++ b/src/plugins/home/server/services/sample_data/data_sets/flights/index.ts @@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n'; import { getSavedObjects } from './saved_objects'; import { fieldMappings } from './field_mappings'; import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types'; +import { getSavedObjectsWithDataSource, appendDataSourceId } from '../util'; const flightsName = i18n.translate('home.sampleData.flightsSpecTitle', { defaultMessage: 'Sample flight data', @@ -42,6 +43,9 @@ const flightsDescription = i18n.translate('home.sampleData.flightsSpecDescriptio }); const initialAppLinks = [] as AppLinkSchema[]; +const DEFAULT_INDEX = 'd3d7af60-4c81-11e8-b3d7-01146121b73d'; +const DASHBOARD_ID = '7adfa750-4c81-11e8-b3d7-01146121b73d'; + export const flightsSpecProvider = function (): SampleDatasetSchema { return { id: 'flights', @@ -49,10 +53,11 @@ export const flightsSpecProvider = function (): SampleDatasetSchema { description: flightsDescription, previewImagePath: '/plugins/home/assets/sample_data_resources/flights/dashboard.png', darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/flights/dashboard_dark.png', - overviewDashboard: '7adfa750-4c81-11e8-b3d7-01146121b73d', + overviewDashboard: appendDataSourceId(DASHBOARD_ID), appLinks: initialAppLinks, - defaultIndex: 'd3d7af60-4c81-11e8-b3d7-01146121b73d', - savedObjects: getSavedObjects(), + defaultIndex: appendDataSourceId(DEFAULT_INDEX), + savedObjects: (dataSourceId?: string, dataSourceTitle?: string) => + getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle), dataIndices: [ { id: 'flights', diff --git a/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts b/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts index 1438de8ad64..e4afabcdd21 100644 --- a/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts +++ b/src/plugins/home/server/services/sample_data/data_sets/logs/index.ts @@ -33,6 +33,7 @@ import { i18n } from '@osd/i18n'; import { getSavedObjects } from './saved_objects'; import { fieldMappings } from './field_mappings'; import { SampleDatasetSchema, AppLinkSchema } from '../../lib/sample_dataset_registry_types'; +import { appendDataSourceId, getSavedObjectsWithDataSource } from '../util'; const logsName = i18n.translate('home.sampleData.logsSpecTitle', { defaultMessage: 'Sample web logs', @@ -42,6 +43,9 @@ const logsDescription = i18n.translate('home.sampleData.logsSpecDescription', { }); const initialAppLinks = [] as AppLinkSchema[]; +const DEFAULT_INDEX = '90943e30-9a47-11e8-b64d-95841ca0b247'; +const DASHBOARD_ID = 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b'; + export const logsSpecProvider = function (): SampleDatasetSchema { return { id: 'logs', @@ -49,10 +53,11 @@ export const logsSpecProvider = function (): SampleDatasetSchema { description: logsDescription, previewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard.png', darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard_dark.png', - overviewDashboard: 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b', + overviewDashboard: appendDataSourceId(DASHBOARD_ID), appLinks: initialAppLinks, - defaultIndex: '90943e30-9a47-11e8-b64d-95841ca0b247', - savedObjects: getSavedObjects(), + defaultIndex: appendDataSourceId(DEFAULT_INDEX), + savedObjects: (dataSourceId?: string, dataSourceTitle?: string) => + getSavedObjectsWithDataSource(getSavedObjects(), dataSourceId, dataSourceTitle), dataIndices: [ { id: 'logs', diff --git a/src/plugins/home/server/services/sample_data/data_sets/util.ts b/src/plugins/home/server/services/sample_data/data_sets/util.ts new file mode 100644 index 00000000000..46022f1c22d --- /dev/null +++ b/src/plugins/home/server/services/sample_data/data_sets/util.ts @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObject } from 'opensearch-dashboards/server'; + +export const appendDataSourceId = (id: string) => { + return (dataSourceId?: string) => (dataSourceId ? `${dataSourceId}_` + id : id); +}; + +export const getSavedObjectsWithDataSource = ( + saveObjectList: SavedObject[], + dataSourceId?: string, + dataSourceTitle?: string +): SavedObject[] => { + if (dataSourceId) { + return saveObjectList.map((saveObject) => { + saveObject.id = `${dataSourceId}_` + saveObject.id; + // update reference + if (saveObject.type === 'dashboard') { + saveObject.references.map((reference) => { + if (reference.id) { + reference.id = `${dataSourceId}_` + reference.id; + } + }); + } + + // update reference + if (saveObject.type === 'visualization' || saveObject.type === 'search') { + const searchSourceString = saveObject.attributes?.kibanaSavedObjectMeta?.searchSourceJSON; + const visStateString = saveObject.attributes?.visState; + + if (searchSourceString) { + const searchSource = JSON.parse(searchSourceString); + if (searchSource.index) { + searchSource.index = `${dataSourceId}_` + searchSource.index; + saveObject.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify( + searchSource + ); + } + } + + if (visStateString) { + const visState = JSON.parse(visStateString); + const controlList = visState.params?.controls; + if (controlList) { + controlList.map((control) => { + if (control.indexPattern) { + control.indexPattern = `${dataSourceId}_` + control.indexPattern; + } + }); + } + saveObject.attributes.visState = JSON.stringify(visState); + } + } + + // update reference + if (saveObject.type === 'index-pattern') { + saveObject.references = [ + { + id: `${dataSourceId}`, + type: 'data-source', + name: 'dataSource', + }, + ]; + } + + if (dataSourceTitle) { + if ( + saveObject.type === 'dashboard' || + saveObject.type === 'visualization' || + saveObject.type === 'search' + ) { + saveObject.attributes.title = saveObject.attributes.title + `_${dataSourceTitle}`; + } + } + + return saveObject; + }); + } + + return saveObjectList; +}; diff --git a/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts b/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts index 86ac5bcc577..76632ee4489 100644 --- a/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts +++ b/src/plugins/home/server/services/sample_data/lib/sample_dataset_registry_types.ts @@ -87,15 +87,15 @@ export interface SampleDatasetSchema { darkPreviewImagePath: string; // saved object id of main dashboard for sample data set - overviewDashboard: string; + overviewDashboard: (dataSourceId?: string) => string; appLinks: AppLinkSchema[]; // saved object id of default index-pattern for sample data set - defaultIndex: string; + defaultIndex: (dataSourceId?: string) => string; // OpenSearch Dashboards saved objects (index patter, visualizations, dashboard, ...) // Should provide a nice demo of OpenSearch Dashboards's functionality with the sample data set - savedObjects: Array>; + savedObjects: (dataSourceId?: string, dataSourceTitle?: string) => Array>; dataIndices: DataIndexSchema[]; status?: string | undefined; statusMsg?: unknown; diff --git a/src/plugins/home/server/services/sample_data/routes/install.test.ts b/src/plugins/home/server/services/sample_data/routes/install.test.ts new file mode 100644 index 00000000000..ad7b421c23d --- /dev/null +++ b/src/plugins/home/server/services/sample_data/routes/install.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CoreSetup, RequestHandlerContext } from 'src/core/server'; +import { coreMock, httpServerMock } from '../../../../../../core/server/mocks'; +import { flightsSpecProvider } from '../data_sets'; +import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; +import { createInstallRoute } from './install'; + +const flightsSampleDataset = flightsSpecProvider(); + +const sampleDatasets: SampleDatasetSchema[] = [flightsSampleDataset]; + +describe('sample data install route', () => { + let mockCoreSetup: MockedKeys; + let mockLogger; + let mockUsageTracker; + + beforeEach(() => { + mockCoreSetup = coreMock.createSetup(); + mockLogger = { + warn: jest.fn(), + }; + + mockUsageTracker = { + addInstall: jest.fn(), + addUninstall: jest.fn(), + }; + }); + + it('handler calls expected api with the given request', async () => { + const mockClient = jest.fn().mockResolvedValue(true); + + const mockSOClientGetResponse = { + saved_objects: [ + { + type: 'dashboard', + id: '12345', + namespaces: ['default'], + attributes: { title: 'dashboard' }, + }, + ], + }; + const mockSOClient = { bulkCreate: jest.fn().mockResolvedValue(mockSOClientGetResponse) }; + + const mockContext = { + core: { + opensearch: { + legacy: { + client: { callAsCurrentUser: mockClient }, + }, + }, + savedObjects: { client: mockSOClient }, + }, + }; + const mockBody = { id: 'flights' }; + const mockQuery = {}; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + params: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createInstallRoute( + mockCoreSetup.http.createRouter(), + sampleDatasets, + mockLogger, + mockUsageTracker + ); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.post.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient.mock.calls[1][1].body.settings).toMatchObject({ + index: { number_of_shards: 1, auto_expand_replicas: '0-1' }, + }); + + // expect(mockClient).toBeCalledTimes(2); + expect(mockResponse.ok).toBeCalled(); + expect(mockResponse.ok.mock.calls[0][0]).toMatchObject({ + body: { + opensearchIndicesCreated: { opensearch_dashboards_sample_data_flights: 13059 }, + opensearchDashboardsSavedObjectsLoaded: 20, + }, + }); + }); + + it('handler calls expected api with the given request with data source', async () => { + const mockDataSourceId = 'dataSource'; + + const mockClient = jest.fn().mockResolvedValue(true); + + const mockSOClientGetResponse = { + saved_objects: [ + { + type: 'dashboard', + id: '12345', + namespaces: ['default'], + attributes: { title: 'dashboard' }, + }, + ], + }; + const mockSOClient = { + bulkCreate: jest.fn().mockResolvedValue(mockSOClientGetResponse), + get: jest.fn().mockResolvedValue(mockSOClientGetResponse), + }; + + const mockContext = { + dataSource: { + opensearch: { + legacy: { + getClient: (id) => { + return { + callAPI: mockClient, + }; + }, + }, + }, + }, + core: { + savedObjects: { client: mockSOClient }, + }, + }; + const mockBody = { id: 'flights' }; + const mockQuery = { data_source_id: mockDataSourceId }; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + params: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createInstallRoute( + mockCoreSetup.http.createRouter(), + sampleDatasets, + mockLogger, + mockUsageTracker + ); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.post.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient.mock.calls[1][1].body.settings).toMatchObject({ + index: { number_of_shards: 1 }, + }); + + expect(mockResponse.ok).toBeCalled(); + expect(mockResponse.ok.mock.calls[0][0]).toMatchObject({ + body: { + opensearchIndicesCreated: { opensearch_dashboards_sample_data_flights: 13059 }, + opensearchDashboardsSavedObjectsLoaded: 20, + }, + }); + }); +}); diff --git a/src/plugins/home/server/services/sample_data/routes/install.ts b/src/plugins/home/server/services/sample_data/routes/install.ts index eec5f0d5bc8..3a321c3bdf0 100644 --- a/src/plugins/home/server/services/sample_data/routes/install.ts +++ b/src/plugins/home/server/services/sample_data/routes/install.ts @@ -29,7 +29,7 @@ */ import { schema } from '@osd/config-schema'; -import { IRouter, Logger, RequestHandlerContext } from 'src/core/server'; +import { IRouter, LegacyCallAPIOptions, Logger } from 'src/core/server'; import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; import { createIndexName } from '../lib/create_index_name'; import { @@ -44,7 +44,11 @@ const insertDataIntoIndex = ( dataIndexConfig: any, index: string, nowReference: string, - context: RequestHandlerContext, + caller: ( + endpoint: string, + clientParams?: Record, + options?: LegacyCallAPIOptions + ) => Promise, logger: Logger ) => { function updateTimestamps(doc: any) { @@ -73,7 +77,8 @@ const insertDataIntoIndex = ( bulk.push(insertCmd); bulk.push(updateTimestamps(doc)); }); - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser('bulk', { + + const resp = await caller('bulk', { body: bulk, }); if (resp.errors) { @@ -105,11 +110,16 @@ export function createInstallRoute( validate: { params: schema.object({ id: schema.string() }), // TODO validate now as date - query: schema.object({ now: schema.maybe(schema.string()) }), + query: schema.object({ + now: schema.maybe(schema.string()), + data_source_id: schema.maybe(schema.string()), + }), }, }, async (context, req, res) => { const { params, query } = req; + const dataSourceId = query.data_source_id; + const sampleDataset = sampleDatasets.find(({ id }) => id === params.id); if (!sampleDataset) { return res.notFound(); @@ -118,13 +128,36 @@ export function createInstallRoute( const now = query.now ? new Date(query.now) : new Date(); const nowReference = dateToIso8601IgnoringTime(now); const counts = {}; + const caller = dataSourceId + ? context.dataSource.opensearch.legacy.getClient(dataSourceId).callAPI + : context.core.opensearch.legacy.client.callAsCurrentUser; + + let dataSourceTitle; + try { + if (dataSourceId) { + const dataSource = await context.core.savedObjects.client + .get('data-source', dataSourceId) + .then((response) => { + const attributes: any = response?.attributes || {}; + return { + id: response.id, + title: attributes.title, + }; + }); + + dataSourceTitle = dataSource.title; + } + } catch (err) { + return res.internalError({ body: err }); + } + for (let i = 0; i < sampleDataset.dataIndices.length; i++) { const dataIndexConfig = sampleDataset.dataIndices[i]; const index = createIndexName(sampleDataset.id, dataIndexConfig.id); // clean up any old installation of dataset try { - await context.core.opensearch.legacy.client.callAsCurrentUser('indices.delete', { + await caller('indices.delete', { index, }); } catch (err) { @@ -135,14 +168,13 @@ export function createInstallRoute( const createIndexParams = { index, body: { - settings: { index: { number_of_shards: 1, auto_expand_replicas: '0-1' } }, + settings: dataSourceId + ? { index: { number_of_shards: 1 } } + : { index: { number_of_shards: 1, auto_expand_replicas: '0-1' } }, mappings: { properties: dataIndexConfig.fields }, }, }; - await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.create', - createIndexParams - ); + await caller('indices.create', createIndexParams); } catch (err) { const errMsg = `Unable to create sample data index "${index}", error: ${err.message}`; logger.warn(errMsg); @@ -154,7 +186,7 @@ export function createInstallRoute( dataIndexConfig, index, nowReference, - context, + caller, logger ); (counts as any)[index] = count; @@ -166,9 +198,11 @@ export function createInstallRoute( } let createResults; + const savedObjectsList = sampleDataset.savedObjects(dataSourceId, dataSourceTitle); + try { createResults = await context.core.savedObjects.client.bulkCreate( - sampleDataset.savedObjects.map(({ version, ...savedObject }) => savedObject), + savedObjectsList.map(({ version, ...savedObject }) => savedObject), { overwrite: true } ); } catch (err) { @@ -192,7 +226,7 @@ export function createInstallRoute( return res.ok({ body: { opensearchIndicesCreated: counts, - opensearchDashboardsSavedObjectsLoaded: sampleDataset.savedObjects.length, + opensearchDashboardsSavedObjectsLoaded: savedObjectsList.length, }, }); } diff --git a/src/plugins/home/server/services/sample_data/routes/list.test.ts b/src/plugins/home/server/services/sample_data/routes/list.test.ts new file mode 100644 index 00000000000..70201fafd06 --- /dev/null +++ b/src/plugins/home/server/services/sample_data/routes/list.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CoreSetup, RequestHandlerContext } from 'src/core/server'; +import { coreMock, httpServerMock } from '../../../../../../core/server/mocks'; +import { createListRoute } from './list'; +import { flightsSpecProvider } from '../data_sets'; +import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; + +const flightsSampleDataset = flightsSpecProvider(); + +const sampleDatasets: SampleDatasetSchema[] = [flightsSampleDataset]; + +describe('sample data list route', () => { + let mockCoreSetup: MockedKeys; + + beforeEach(() => { + mockCoreSetup = coreMock.createSetup(); + }); + + it('handler calls expected api with the given request', async () => { + const mockClient = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce({ count: 1 }); + + const mockSOClientGetResponse = { + saved_objects: [ + { + type: 'dashboard', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + namespaces: ['default'], + attributes: { title: 'dashboard' }, + }, + ], + }; + const mockSOClient = { get: jest.fn().mockResolvedValue(mockSOClientGetResponse) }; + + const mockContext = { + core: { + opensearch: { + legacy: { + client: { callAsCurrentUser: mockClient }, + }, + }, + savedObjects: { client: mockSOClient }, + }, + }; + const mockBody = {}; + const mockQuery = {}; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + body: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createListRoute(mockCoreSetup.http.createRouter(), sampleDatasets); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.get.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient).toBeCalledTimes(2); + expect(mockResponse.ok).toBeCalled(); + expect(mockSOClient.get.mock.calls[0][1]).toMatch('7adfa750-4c81-11e8-b3d7-01146121b73d'); + }); + + it('handler calls expected api with the given request with data source', async () => { + const mockDataSourceId = 'dataSource'; + const mockClient = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce({ count: 1 }); + + const mockSOClientGetResponse = { + saved_objects: [ + { + type: 'dashboard', + id: `${mockDataSourceId}_7adfa750-4c81-11e8-b3d7-01146121b73d`, + namespaces: ['default'], + attributes: { title: 'dashboard' }, + }, + ], + }; + const mockSOClient = { get: jest.fn().mockResolvedValue(mockSOClientGetResponse) }; + + const mockContext = { + dataSource: { + opensearch: { + legacy: { + getClient: (id) => { + return { + callAPI: mockClient, + }; + }, + }, + }, + }, + core: { + savedObjects: { client: mockSOClient }, + }, + }; + + const mockBody = {}; + const mockQuery = { data_source_id: mockDataSourceId }; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + body: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createListRoute(mockCoreSetup.http.createRouter(), sampleDatasets); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.get.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient).toBeCalledTimes(2); + expect(mockResponse.ok).toBeCalled(); + expect(mockSOClient.get.mock.calls[0][1]).toMatch( + `${mockDataSourceId}_7adfa750-4c81-11e8-b3d7-01146121b73d` + ); + }); +}); diff --git a/src/plugins/home/server/services/sample_data/routes/list.ts b/src/plugins/home/server/services/sample_data/routes/list.ts index a0665132dd1..bc6fac2b76f 100644 --- a/src/plugins/home/server/services/sample_data/routes/list.ts +++ b/src/plugins/home/server/services/sample_data/routes/list.ts @@ -29,6 +29,7 @@ */ import { IRouter } from 'src/core/server'; +import { schema } from '@osd/config-schema'; import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; import { createIndexName } from '../lib/create_index_name'; @@ -37,66 +38,79 @@ const INSTALLED = 'installed'; const UNKNOWN = 'unknown'; export const createListRoute = (router: IRouter, sampleDatasets: SampleDatasetSchema[]) => { - router.get({ path: '/api/sample_data', validate: false }, async (context, req, res) => { - const registeredSampleDatasets = sampleDatasets.map((sampleDataset) => { - return { - id: sampleDataset.id, - name: sampleDataset.name, - description: sampleDataset.description, - previewImagePath: sampleDataset.previewImagePath, - darkPreviewImagePath: sampleDataset.darkPreviewImagePath, - overviewDashboard: sampleDataset.overviewDashboard, - appLinks: sampleDataset.appLinks, - defaultIndex: sampleDataset.defaultIndex, - dataIndices: sampleDataset.dataIndices.map(({ id }) => ({ id })), - status: sampleDataset.status, - statusMsg: sampleDataset.statusMsg, - }; - }); - const isInstalledPromises = registeredSampleDatasets.map(async (sampleDataset) => { - for (let i = 0; i < sampleDataset.dataIndices.length; i++) { - const dataIndexConfig = sampleDataset.dataIndices[i]; - const index = createIndexName(sampleDataset.id, dataIndexConfig.id); - try { - const indexExists = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.exists', - { index } - ); - if (!indexExists) { - sampleDataset.status = NOT_INSTALLED; + router.get( + { + path: '/api/sample_data', + validate: { + query: schema.object({ data_source_id: schema.maybe(schema.string()) }), + }, + }, + async (context, req, res) => { + const dataSourceId = req.query.data_source_id; + + const registeredSampleDatasets = sampleDatasets.map((sampleDataset) => { + return { + id: sampleDataset.id, + name: sampleDataset.name, + description: sampleDataset.description, + previewImagePath: sampleDataset.previewImagePath, + darkPreviewImagePath: sampleDataset.darkPreviewImagePath, + overviewDashboard: sampleDataset.overviewDashboard(dataSourceId), + appLinks: sampleDataset.appLinks, + defaultIndex: sampleDataset.defaultIndex(dataSourceId), + dataIndices: sampleDataset.dataIndices.map(({ id }) => ({ id })), + status: sampleDataset.status, + statusMsg: sampleDataset.statusMsg, + }; + }); + const isInstalledPromises = registeredSampleDatasets.map(async (sampleDataset) => { + const caller = dataSourceId + ? context.dataSource.opensearch.legacy.getClient(dataSourceId).callAPI + : context.core.opensearch.legacy.client.callAsCurrentUser; + + for (let i = 0; i < sampleDataset.dataIndices.length; i++) { + const dataIndexConfig = sampleDataset.dataIndices[i]; + const index = createIndexName(sampleDataset.id, dataIndexConfig.id); + try { + const indexExists = await caller('indices.exists', { index }); + + if (!indexExists) { + sampleDataset.status = NOT_INSTALLED; + return; + } + + const { count } = await caller('count', { + index, + }); + + if (count === 0) { + sampleDataset.status = NOT_INSTALLED; + return; + } + } catch (err) { + sampleDataset.status = UNKNOWN; + sampleDataset.statusMsg = err.message; return; } - - const { count } = await context.core.opensearch.legacy.client.callAsCurrentUser('count', { - index, - }); - if (count === 0) { + } + try { + await context.core.savedObjects.client.get('dashboard', sampleDataset.overviewDashboard); + } catch (err) { + if (context.core.savedObjects.client.errors.isNotFoundError(err)) { sampleDataset.status = NOT_INSTALLED; return; } - } catch (err) { + sampleDataset.status = UNKNOWN; sampleDataset.statusMsg = err.message; return; } - } - try { - await context.core.savedObjects.client.get('dashboard', sampleDataset.overviewDashboard); - } catch (err) { - if (context.core.savedObjects.client.errors.isNotFoundError(err)) { - sampleDataset.status = NOT_INSTALLED; - return; - } - - sampleDataset.status = UNKNOWN; - sampleDataset.statusMsg = err.message; - return; - } - sampleDataset.status = INSTALLED; - }); + sampleDataset.status = INSTALLED; + }); - await Promise.all(isInstalledPromises); - return res.ok({ body: registeredSampleDatasets }); - }); + await Promise.all(isInstalledPromises); + return res.ok({ body: registeredSampleDatasets }); + } + ); }; diff --git a/src/plugins/home/server/services/sample_data/routes/uninstall.test.ts b/src/plugins/home/server/services/sample_data/routes/uninstall.test.ts new file mode 100644 index 00000000000..7d9797d752c --- /dev/null +++ b/src/plugins/home/server/services/sample_data/routes/uninstall.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CoreSetup, RequestHandlerContext } from 'src/core/server'; +import { coreMock, httpServerMock } from '../../../../../../core/server/mocks'; +import { flightsSpecProvider } from '../data_sets'; +import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; +import { createUninstallRoute } from './uninstall'; + +const flightsSampleDataset = flightsSpecProvider(); + +const sampleDatasets: SampleDatasetSchema[] = [flightsSampleDataset]; + +describe('sample data uninstall route', () => { + let mockCoreSetup: MockedKeys; + let mockUsageTracker; + let mockClient; + let mockSOClient; + + beforeEach(() => { + mockCoreSetup = coreMock.createSetup(); + + mockUsageTracker = { + addInstall: jest.fn(), + addUninstall: jest.fn(), + }; + + mockClient = jest.fn(); + mockSOClient = { delete: jest.fn().mockResolvedValue(true) }; + }); + + it('handler calls expected api with the given request', async () => { + const mockContext = { + core: { + opensearch: { + legacy: { + client: { callAsCurrentUser: mockClient }, + }, + }, + savedObjects: { client: mockSOClient }, + }, + }; + const mockBody = { id: 'flights' }; + const mockQuery = {}; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + params: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createUninstallRoute(mockCoreSetup.http.createRouter(), sampleDatasets, mockUsageTracker); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.delete.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient).toBeCalled(); + expect(mockSOClient.delete).toBeCalled(); + }); + + it('handler calls expected api with the given request with data source', async () => { + const mockDataSourceId = 'dataSource'; + + const mockContext = { + dataSource: { + opensearch: { + legacy: { + getClient: (id) => { + return { + callAPI: mockClient, + }; + }, + }, + }, + }, + core: { + savedObjects: { client: mockSOClient }, + }, + }; + const mockBody = { id: 'flights' }; + const mockQuery = { data_source_id: mockDataSourceId }; + const mockRequest = httpServerMock.createOpenSearchDashboardsRequest({ + params: mockBody, + query: mockQuery, + }); + const mockResponse = httpServerMock.createResponseFactory(); + + createUninstallRoute(mockCoreSetup.http.createRouter(), sampleDatasets, mockUsageTracker); + + const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value; + const handler = mockRouter.delete.mock.calls[0][1]; + + await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); + + expect(mockClient).toBeCalled(); + expect(mockSOClient.delete).toBeCalled(); + }); +}); diff --git a/src/plugins/home/server/services/sample_data/routes/uninstall.ts b/src/plugins/home/server/services/sample_data/routes/uninstall.ts index e38a2224aae..730de6afff3 100644 --- a/src/plugins/home/server/services/sample_data/routes/uninstall.ts +++ b/src/plugins/home/server/services/sample_data/routes/uninstall.ts @@ -45,34 +45,29 @@ export function createUninstallRoute( path: '/api/sample_data/{id}', validate: { params: schema.object({ id: schema.string() }), + query: schema.object({ + data_source_id: schema.maybe(schema.string()), + }), }, }, - async ( - { - core: { - opensearch: { - legacy: { - client: { callAsCurrentUser }, - }, - }, - savedObjects: { client: savedObjectsClient }, - }, - }, - request, - response - ) => { + async (context, request, response) => { const sampleDataset = sampleDatasets.find(({ id }) => id === request.params.id); + const dataSourceId = request.query.data_source_id; if (!sampleDataset) { return response.notFound(); } + const caller = dataSourceId + ? context.dataSource.opensearch.legacy.getClient(dataSourceId).callAPI + : context.core.opensearch.legacy.client.callAsCurrentUser; + for (let i = 0; i < sampleDataset.dataIndices.length; i++) { const dataIndexConfig = sampleDataset.dataIndices[i]; const index = createIndexName(sampleDataset.id, dataIndexConfig.id); try { - await callAsCurrentUser('indices.delete', { index }); + await caller('indices.delete', { index }); } catch (err) { return response.customError({ statusCode: err.status, @@ -83,8 +78,10 @@ export function createUninstallRoute( } } - const deletePromises = sampleDataset.savedObjects.map(({ type, id }) => - savedObjectsClient.delete(type, id) + const savedObjectsList = sampleDataset.savedObjects(dataSourceId); + + const deletePromises = savedObjectsList.map(({ type, id }) => + context.core.savedObjects.client.delete(type, id) ); try { diff --git a/src/plugins/home/server/services/sample_data/sample_data_registry.ts b/src/plugins/home/server/services/sample_data/sample_data_registry.ts index d90de532c33..5f298fd86ef 100644 --- a/src/plugins/home/server/services/sample_data/sample_data_registry.ts +++ b/src/plugins/home/server/services/sample_data/sample_data_registry.ts @@ -84,23 +84,23 @@ export class SampleDataRegistry { } const defaultIndexSavedObjectJson = value.savedObjects.find((savedObjectJson: any) => { return ( - savedObjectJson.type === 'index-pattern' && savedObjectJson.id === value.defaultIndex + savedObjectJson.type === 'index-pattern' && savedObjectJson.id === value.defaultIndex() ); }); if (!defaultIndexSavedObjectJson) { throw new Error( - `Unable to register sample dataset spec, defaultIndex: "${value.defaultIndex}" does not exist in savedObjects list.` + `Unable to register sample dataset spec, defaultIndex: "${value.defaultIndex()}" does not exist in savedObjects list.` ); } const dashboardSavedObjectJson = value.savedObjects.find((savedObjectJson: any) => { return ( - savedObjectJson.type === 'dashboard' && savedObjectJson.id === value.overviewDashboard + savedObjectJson.type === 'dashboard' && savedObjectJson.id === value.overviewDashboard() ); }); if (!dashboardSavedObjectJson) { throw new Error( - `Unable to register sample dataset spec, overviewDashboard: "${value.overviewDashboard}" does not exist in savedObject list.` + `Unable to register sample dataset spec, overviewDashboard: "${value.overviewDashboard()}" does not exist in savedObject list.` ); } this.sampleDatasets.push(value);