support generating schemas/tables from query without authentication (#…

…2487)
finos · Aug 10, 2023 · 246fa96 · 246fa96
1 parent ade5e57
commit 246fa96
Show file tree

Hide file tree

Showing 12 changed files with 326 additions and 18 deletions.
diff --git a/.changeset/brave-kiwis-yell.md b/.changeset/brave-kiwis-yell.md
@@ -0,0 +1,5 @@
+---
+'@finos/legend-application-studio': patch
+---
+
+support generating schemas/tables from query without authentication
diff --git a/.changeset/red-carpets-taste.md b/.changeset/red-carpets-taste.md
@@ -0,0 +1,3 @@
+---
+'@finos/legend-graph': patch
+---
diff --git a/...udio/src/components/editor/editor-group/service-editor/testable/ServiceTestDataEditor.tsx b/...udio/src/components/editor/editor-group/service-editor/testable/ServiceTestDataEditor.tsx
@@ -16,10 +16,12 @@
 
 import {
   BlankPanelPlaceholder,
+  CaretDownIcon,
   clsx,
   ContextMenu,
   CustomSelectorInput,
   Dialog,
+  DropdownMenu,
   InfoCircleIcon,
   MaskIcon,
   MenuContent,
@@ -199,6 +201,12 @@ export const ConnectionTestDataEditor = observer(
       }
     };
 
+    const generateQuerySchemas = (): void => {
+      flowResult(connectionTestDataState.generateQuerySchemas()).catch(
+        applicationStore.alertUnhandledError,
+      );
+    };
+
     return (
       <div className="service-test-data-editor">
         <div className="service-test-suite-editor__header">
@@ -231,22 +239,45 @@ export const ConnectionTestDataEditor = observer(
                 <MaskIcon />
               </button>
             </div>
-            <button
-              className="panel__header__action service-execution-editor__test-data__generate-btn"
-              onClick={generateTestData}
-              title="Generate test data if possible"
-              disabled={
-                connectionTestDataState.generatingTestDataState.isInProgress
-              }
-              tabIndex={-1}
-            >
-              <div className="service-execution-editor__test-data__generate-btn__label">
-                <RefreshIcon className="service-execution-editor__test-data__generate-btn__label__icon" />
-                <div className="service-execution-editor__test-data__generate-btn__label__title">
+            <div className="btn__dropdown-combo btn__dropdown-combo--primary">
+              <button
+                className="btn__dropdown-combo__label"
+                onClick={generateTestData}
+                title="Generate test data if possible"
+                disabled={
+                  connectionTestDataState.generatingTestDataState.isInProgress
+                }
+                tabIndex={-1}
+              >
+                <RefreshIcon className="btn__dropdown-combo__label__icon" />
+                <div className="btn__dropdown-combo__label__title">
                   Generate
                 </div>
-              </div>
-            </button>
+              </button>
+              <DropdownMenu
+                className="btn__dropdown-combo__dropdown-btn"
+                content={
+                  <MenuContent>
+                    <MenuContentItem
+                      className="btn__dropdown-combo__option"
+                      onClick={generateQuerySchemas}
+                      disabled={
+                        connectionTestDataState.generateSchemaQueryState
+                          .isInProgress
+                      }
+                    >
+                      Generate Query Schemas
+                    </MenuContentItem>
+                  </MenuContent>
+                }
+                menuProps={{
+                  anchorOrigin: { vertical: 'bottom', horizontal: 'right' },
+                  transformOrigin: { vertical: 'top', horizontal: 'right' },
+                }}
+              >
+                <CaretDownIcon />
+              </DropdownMenu>
+            </div>
             <button
               className="panel__header__action service-execution-editor__test-data__generate-btn"
               onClick={openShared}
@@ -631,10 +662,16 @@ export const ServiceTestDataEditor = observer(
               </ResizablePanelSplitter>
               <ResizablePanel minSize={600}>
                 <PanelLoadingIndicator
-                  isLoading={Boolean(
-                    testDataState.selectedDataState?.generatingTestDataState
-                      .isInProgress,
-                  )}
+                  isLoading={
+                    Boolean(
+                      testDataState.selectedDataState?.generatingTestDataState
+                        .isInProgress,
+                    ) ||
+                    Boolean(
+                      testDataState.selectedDataState?.generateSchemaQueryState
+                        .isInProgress,
+                    )
+                  }
                 />
                 {testDataState.selectedDataState && (
                   <ConnectionTestDataEditor

diff --git a/.../stores/editor/editor-state/element-editor-state/service/testable/ServiceTestDataState.ts b/.../stores/editor/editor-state/element-editor-state/service/testable/ServiceTestDataState.ts
@@ -23,6 +23,7 @@ import {
   type RawLambda,
   type DataElement,
   type Mapping,
+  type TestDataGenerationResult,
   ConnectionTestData,
   PureSingleExecution,
   PureMultiExecution,
@@ -37,6 +38,7 @@ import {
   getAllIdentifiedConnectionsFromRuntime,
   getAllIdentifiedServiceConnections,
   Database,
+  RuntimePointer,
 } from '@finos/legend-graph';
 import {
   type GeneratorFn,
@@ -47,6 +49,8 @@ import {
   guaranteeNonNullable,
   returnUndefOnError,
   getNullableFirstEntry,
+  assertType,
+  assertTrue,
 } from '@finos/legend-shared';
 import { action, flow, flowResult, makeObservable, observable } from 'mobx';
 import type { EditorStore } from '../../../../EditorStore.js';
@@ -156,6 +160,7 @@ export class ConnectionTestDataState {
   readonly connectionData: ConnectionTestData;
   readonly parametersState: ServiceTestDataParametersState;
   readonly generatingTestDataState = ActionState.create();
+  readonly generateSchemaQueryState = ActionState.create();
   useSharedModal = false;
 
   embeddedEditorState: EmbeddedDataEditorState;
@@ -167,6 +172,7 @@ export class ConnectionTestDataState {
   ) {
     makeObservable(this, {
       generatingTestDataState: observable,
+      generateSchemaQueryState: observable,
       embeddedEditorState: observable,
       useSharedModal: observable,
       anonymizeGeneratedData: observable,
@@ -175,6 +181,7 @@ export class ConnectionTestDataState {
       buildEmbeddedEditorState: action,
       generateTestData: flow,
       generateTestDataForDatabaseConnection: flow,
+      generateQuerySchemas: flow,
     });
     this.testDataState = testDataState;
     this.editorStore = testDataState.editorStore;
@@ -326,6 +333,60 @@ export class ConnectionTestDataState {
     }
   }
 
+  *generateQuerySchemas(): GeneratorFn<void> {
+    try {
+      this.generateSchemaQueryState.inProgress();
+      const connection = guaranteeNonNullable(
+        this.resolveConnectionValue(this.connectionData.connectionId),
+        `Unable to resolve connection ID '${this.connectionData.connectionId}`,
+      );
+      if (connection instanceof DatabaseConnection) {
+        const serviceExecutionParameters = guaranteeNonNullable(
+          this.testDataState.testSuiteState.testableState.serviceEditorState
+            .executionState.serviceExecutionParameters,
+        );
+        assertType(
+          serviceExecutionParameters.runtime,
+          RuntimePointer,
+          'Expected runtime type to be RuntimePointer',
+        );
+        const testDataGenerationResult =
+          (yield this.editorStore.graphManagerState.graphManager.generateTestData(
+            getExecutionQueryFromRawLambda(
+              serviceExecutionParameters.query,
+              this.parametersState.parameterStates,
+              this.editorStore.graphManagerState,
+            ),
+            serviceExecutionParameters.mapping.path,
+            serviceExecutionParameters.runtime.packageableRuntime.value.path,
+            this.editorStore.graphManagerState.graph,
+          )) as TestDataGenerationResult;
+        assertTrue(
+          testDataGenerationResult.data.length >= 1,
+          'Expected generated data to at least have one data',
+        );
+        service_setConnectionTestDataEmbeddedData(
+          this.connectionData,
+          guaranteeNonNullable(testDataGenerationResult.data[0]),
+          this.editorStore.changeDetectionState.observerContext,
+        );
+        this.embeddedEditorState = new EmbeddedDataEditorState(
+          this.testDataState.editorStore,
+          this.connectionData.testData,
+        );
+      }
+      this.generateSchemaQueryState.pass();
+    } catch (error) {
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.notificationService.notifyError(
+        `Unable to generate query schemas: ${error.message}`,
+      );
+      this.generateSchemaQueryState.fail();
+    } finally {
+      this.generateSchemaQueryState.complete();
+    }
+  }
+
   changeEmbeddedData(val: EmbeddedData): void {
     service_setConnectionTestDataEmbeddedData(
       this.connectionData,

diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts
@@ -101,6 +101,7 @@ import type { FunctionActivatorConfiguration } from './action/functionActivator/
 import type { FunctionActivator } from '../graph/metamodel/pure/packageableElements/function/FunctionActivator.js';
 import type { RelationalDatabaseConnection } from '../STO_Relational_Exports.js';
 import type { ArtifactGenerationExtensionResult } from './action/generation/ArtifactGenerationExtensionResult.js';
+import type { TestDataGenerationResult } from '../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
 
 export interface TEMPORARY__EngineSetupConfig {
   env: string;
@@ -412,6 +413,15 @@ export abstract class AbstractPureGraphManager {
     graph: PureModel,
   ): Promise<Entity[]>;
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  abstract generateTestData(
+    query: RawLambda,
+    mapping: string,
+    runtime: string,
+    graph: PureModel,
+  ): Promise<TestDataGenerationResult>;
+
   // ------------------------------------------- External Format ----------------------------------
 
   abstract getAvailableExternalFormatsDescriptions(): Promise<

diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts
@@ -304,6 +304,9 @@ import {
   V1_buildArtifactsByExtensionElement,
 } from './engine/generation/V1_ArtifactGenerationExtensionApi.js';
 import type { V1_RawValueSpecification } from './model/rawValueSpecification/V1_RawValueSpecification.js';
+import { V1_TestDataGenerationInput } from './engine/service/V1_TestDataGenerationInput.js';
+import type { TestDataGenerationResult } from '../../../../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
+import { V1_buildTestDataGenerationResult } from './engine/service/V1_TestDataGenerationResult.js';
 
 class V1_PureModelContextDataIndex {
   elements: V1_PackageableElement[] = [];
@@ -2060,6 +2063,37 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     return this.pureModelContextDataToEntities(generatedModel);
   }
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  async generateTestData(
+    query: RawLambda,
+    mapping: string,
+    runtime: string,
+    graph: PureModel,
+  ): Promise<TestDataGenerationResult> {
+    const testDataGenerationInput = new V1_TestDataGenerationInput();
+    testDataGenerationInput.query = V1_transformRawLambda(
+      query,
+      new V1_GraphTransformerContextBuilder(
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ).build(),
+    );
+    testDataGenerationInput.mapping = mapping;
+    testDataGenerationInput.runtime = runtime;
+    const graphData = this.getFullGraphModelContext(
+      graph,
+      V1_PureGraphManager.DEV_PROTOCOL_VERSION,
+    );
+    testDataGenerationInput.model = graphData;
+    return V1_buildTestDataGenerationResult(
+      await this.engine.generateTestData(
+        testDataGenerationInput,
+        this.pluginManager.getPureProtocolProcessorPlugins(),
+      ),
+      this.getBuilderContext(graph, graph, new V1_Mapping()), // use a dummy V1_PackageableElement to generate V1_GraphBuilderContext which is used in V1_buildEmbeddedData()
+    );
+  }
+
   // ------------------------------------------- Test  -------------------------------------------
 
   async runTests(

diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_Engine.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_Engine.ts
@@ -136,6 +136,11 @@ import {
   V1_ArtifactGenerationExtensionInput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
 import { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
+import { V1_TestDataGenerationInput } from './service/V1_TestDataGenerationInput.js';
+import {
+  type V1_TestDataGenerationResult,
+  V1_testDataGenerationResultModelSchema,
+} from './service/V1_TestDataGenerationResult.js';
 
 class V1_EngineConfig extends TEMPORARY__AbstractEngineConfig {
   private engine: V1_Engine;
@@ -719,6 +724,20 @@ export class V1_Engine {
     );
   }
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  async generateTestData(
+    input: V1_TestDataGenerationInput,
+    plugins: PureProtocolProcessorPlugin[],
+  ): Promise<V1_TestDataGenerationResult> {
+    return deserialize(
+      V1_testDataGenerationResultModelSchema(plugins),
+      await this.engineServerClient.generateTestData(
+        V1_TestDataGenerationInput.serialization.toJson(input),
+      ),
+    );
+  }
+
   // ------------------------------------------- File Generation -------------------------------------------
 
   async getAvailableGenerationConfigurationDescriptions(): Promise<

diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineServerClient.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineServerClient.ts
@@ -78,12 +78,15 @@ import type {
   V1_ArtifactGenerationExtensionOutput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
 import type { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
+import type { V1_TestDataGenerationInput } from './service/V1_TestDataGenerationInput.js';
+import type { V1_TestDataGenerationResult } from './service/V1_TestDataGenerationResult.js';
 
 enum CORE_ENGINE_ACTIVITY_TRACE {
   GRAMMAR_TO_JSON = 'transform Pure code to protocol',
   JSON_TO_GRAMMAR = 'transform protocol to Pure code',
 
   DATABASE_TO_MODELS = 'generate models from database',
+  TEST_DATA_GENERATION = 'generate test data',
 
   EXTERNAL_FORMAT_TO_PROTOCOL = 'transform external format code to protocol',
   GENERATE_FILE = 'generate file',
@@ -502,6 +505,20 @@ export class V1_EngineServerClient extends AbstractServerClient {
     PlainObject<V1_GenerationConfigurationDescription>[]
   > => this.get(`${this._pure()}/schemaGeneration/availableGenerations`);
 
+  // --------------------------------------------- Test Data Generation ---------------------------------------------
+
+  _testDataGeneration = (): string => `${this._pure()}/testData/generation`;
+
+  generateTestData(
+    input: PlainObject<V1_TestDataGenerationInput>,
+  ): Promise<PlainObject<V1_TestDataGenerationResult>> {
+    return this.postWithTracing(
+      this.getTraceData(CORE_ENGINE_ACTIVITY_TRACE.TEST_DATA_GENERATION),
+      `${this._testDataGeneration()}/DONOTUSE_generateTestData`,
+      this.debugPayload(input, CORE_ENGINE_ACTIVITY_TRACE.TEST_DATA_GENERATION),
+    );
+  }
+
   // ------------------------------------------- Compile -------------------------------------------
 
   compile = (