generate models from relational database (#2455)

.changeset/sixty-carpets-cheat.md

@@ -0,0 +1,5 @@
+---
+'@finos/legend-application-studio': minor
+'@finos/legend-graph': minor
+---
+Support generating models from relational database. 

packages/legend-application-studio/src/components/editor/side-bar/Explorer.tsx

@@ -111,8 +111,13 @@ import {
   guaranteeRelationalDatabaseConnection,
   extractDependencyGACoordinateFromRootPackageName,
   type FunctionActivatorConfiguration,
+  Database,
 } from '@finos/legend-graph';
-import { useApplicationStore } from '@finos/legend-application';
+import {
+  ActionAlertActionType,
+  ActionAlertType,
+  useApplicationStore,
+} from '@finos/legend-application';
 import {
   getPackageableElementOptionFormatter,
   type PackageableElementOption,
@@ -452,6 +457,9 @@ const isRelationalDatabaseConnection = (
   val instanceof PackageableConnection &&
   val.connectionValue instanceof RelationalDatabaseConnection;
 
+const isRelationalDatabase = (val: PackageableElement | undefined): boolean =>
+  val instanceof Database;
+
 const ExplorerContextMenu = observer(
   forwardRef<
     HTMLDivElement,
@@ -523,6 +531,46 @@ const ExplorerContextMenu = observer(
         }
       },
     );
+    const generateModelsFromDatabaseSpecification =
+      editorStore.applicationStore.guardUnhandledError(async () => {
+        if (isRelationalDatabase(node?.packageableElement)) {
+          const databasePath = guaranteeNonEmptyString(
+            node?.packageableElement.path,
+          );
+          const graph = editorStore.graphManagerState.graph;
+          if (graph.getDatabase(databasePath).joins.length === 0) {
+            applicationStore.alertService.setActionAlertInfo({
+              message:
+                'You are attempting to generate models but have defined no joins. Are you sure you wish to proceed?',
+              type: ActionAlertType.CAUTION,
+              actions: [
+                {
+                  label: 'Proceed',
+                  type: ActionAlertActionType.PROCEED_WITH_CAUTION,
+                  handler: () => {
+                    flowResult(
+                      editorStore.explorerTreeState.generateModelsFromDatabaseSpecification(
+                        databasePath,
+                        graph,
+                      ),
+                    ).catch(applicationStore.alertUnhandledError);
+                  },
+                },
+                {
+                  label: 'Abort',
+                  type: ActionAlertActionType.PROCEED,
+                  default: true,
+                },
+              ],
+            });
+          } else {
+            editorStore.explorerTreeState.generateModelsFromDatabaseSpecification(
+              databasePath,
+              graph,
+            );
+          }
+        }
+      });
     const openSQLPlayground = (): void => {
       if (isRelationalDatabaseConnection(node?.packageableElement)) {
         editorStore.panelGroupDisplayState.open();
@@ -812,6 +860,14 @@ const ExplorerContextMenu = observer(
             <MenuContentDivider />
           </>
         )}
+        {isRelationalDatabase(node.packageableElement) && (
+          <>
+            <MenuContentItem onClick={generateModelsFromDatabaseSpecification}>
+              Generate Models
+            </MenuContentItem>
+            <MenuContentDivider />
+          </>
+        )}
         {extraExplorerContextMenuItems}
         {Boolean(extraExplorerContextMenuItems.length) && (
           <MenuContentDivider />

packages/legend-application-studio/src/stores/editor/ExplorerTreeState.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { action, observable, makeObservable } from 'mobx';
+import { action, observable, makeObservable, flow, flowResult } from 'mobx';
 import type { EditorStore } from './EditorStore.js';
 import {
   LogEvent,
@@ -23,6 +23,8 @@ import {
   UnsupportedOperationError,
   guaranteeNonNullable,
   ActionState,
+  type GeneratorFn,
+  assertErrorThrown,
 } from '@finos/legend-shared';
 import {
   getDependenciesPackableElementTreeData,
@@ -49,9 +51,12 @@ import {
   isDependencyElement,
   type Class,
   type RelationalDatabaseConnection,
+  type PureModel,
 } from '@finos/legend-graph';
 import { APPLICATION_EVENT } from '@finos/legend-application';
 import { DatabaseBuilderWizardState } from './editor-state/element-editor-state/connection/DatabaseBuilderWizardState.js';
+import type { Entity } from '@finos/legend-storage';
+import { EntityChangeType, type EntityChange } from '@finos/legend-server-sdlc';
 
 export enum ExplorerTreeRootPackageLabel {
   FILE_GENERATION = 'generated-files',
@@ -107,6 +112,7 @@ export class ExplorerTreeState {
       setDatabaseBuilderState: action,
       onTreeNodeSelect: action,
       openNode: action,
+      generateModelsFromDatabaseSpecification: flow,
     });
 
     this.editorStore = editorStore;
@@ -189,6 +195,50 @@ export class ExplorerTreeState {
     this.setDatabaseBuilderState(dbBuilderState);
   }
 
+  *generateModelsFromDatabaseSpecification(
+    databasePath: string,
+    graph: PureModel,
+  ): GeneratorFn<void> {
+    try {
+      const entities =
+        (yield this.editorStore.graphManagerState.graphManager.generateModelsFromDatabaseSpecification(
+          databasePath,
+          graph,
+        )) as Entity[];
+      const newEntities: EntityChange[] = [];
+      for (const entity of entities) {
+        let entityChangeType: EntityChangeType;
+        if (graph.getNullableElement(entity.path) === undefined) {
+          entityChangeType = EntityChangeType.CREATE;
+        } else {
+          entityChangeType = EntityChangeType.MODIFY;
+        }
+        newEntities.push({
+          type: entityChangeType,
+          entityPath: entity.path,
+          content: entity.content,
+        });
+      }
+      yield flowResult(
+        this.editorStore.graphState.loadEntityChangesToGraph(
+          newEntities,
+          undefined,
+        ),
+      );
+      this.editorStore.applicationStore.notificationService.notifySuccess(
+        'Generated models successfully!',
+      );
+    } catch (error) {
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.logService.error(
+        LogEvent.create(LEGEND_STUDIO_APP_EVENT.GENERATION_FAILURE),
+        error,
+      );
+      this.editorStore.applicationStore.notificationService.notifyError(error);
+      throw error;
+    }
+  }
+
   setSelectedNode(node: PackageTreeNodeData | undefined): void {
     if (this.selectedNode) {
       this.selectedNode.isSelected = false;

packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts

@@ -570,6 +570,13 @@ export abstract class AbstractPureGraphManager {
     graphData: GraphData,
   ): Promise<void>;
 
+  // --------------------------------------------- Relational ---------------------------------------------
+
+  abstract generateModelsFromDatabaseSpecification(
+    databasePath: string,
+    graph: PureModel,
+  ): Promise<Entity[]>;
+
   // ------------------------------------------- Service -------------------------------------------
   /**
    * @modularize

packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts

@@ -291,6 +291,7 @@ import { FunctionActivatorConfiguration } from '../../../action/functionActivato
 import { V1_FunctionActivatorInput } from './engine/functionActivator/V1_FunctionActivatorInput.js';
 import { V1_FunctionActivator } from './model/packageableElements/function/V1_FunctionActivator.js';
 import { V1_INTERNAL__UnknownFunctionActivator } from './model/packageableElements/function/V1_INTERNAL__UnknownFunctionActivator.js';
+import { V1_DatabaseToModelGenerationInput } from './engine/relational/V1_DatabaseToModelGenerationInput.js';
 import type { RelationalDatabaseConnection } from '../../../../STO_Relational_Exports.js';
 import { V1_RawSQLExecuteInput } from './engine/execution/V1_RawSQLExecuteInput.js';
 import type { SubtypeInfo } from '../../../action/protocol/ProtocolInfo.js';
@@ -2996,6 +2997,21 @@ export class V1_PureGraphManager extends AbstractPureGraphManager {
     await this.engine.publishFunctionActivatorToSandbox(input);
   }
 
+  // --------------------------------------------- Relational ---------------------------------------------
+
+  async generateModelsFromDatabaseSpecification(
+    databasePath: string,
+    graph: PureModel,
+  ): Promise<Entity[]> {
+    const graphData = this.graphToPureModelContextData(graph);
+    const input = new V1_DatabaseToModelGenerationInput();
+    input.databasePath = databasePath;
+    input.modelData = graphData;
+    const generatedModel =
+      await this.engine.generateModelsFromDatabaseSpecification(input);
+    return this.pureModelContextDataToEntities(generatedModel);
+  }
+
   // --------------------------------------------- Service ---------------------------------------------
 
   async registerService(

packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_Engine.ts

@@ -135,6 +135,7 @@ import {
   V1_ArtifactGenerationExtensionOutput,
   V1_ArtifactGenerationExtensionInput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
+import { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
 
 class V1_EngineConfig extends TEMPORARY__AbstractEngineConfig {
   private engine: V1_Engine;
@@ -1013,4 +1014,31 @@ export class V1_Engine {
       );
     }
   }
+
+  // ------------------------------------------- Relational -------------------------------------------
+
+  async generateModelsFromDatabaseSpecification(
+    input: V1_DatabaseToModelGenerationInput,
+  ): Promise<V1_PureModelContextData> {
+    try {
+      const json =
+        await this.engineServerClient.generateModelsFromDatabaseSpecification(
+          V1_DatabaseToModelGenerationInput.serialization.toJson(input),
+        );
+      return V1_deserializePureModelContextData(json);
+    } catch (error) {
+      assertErrorThrown(error);
+      if (
+        error instanceof NetworkClientError &&
+        error.response.status === HttpStatus.BAD_REQUEST
+      ) {
+        throw V1_buildParserError(
+          V1_ParserError.serialization.fromJson(
+            error.payload as PlainObject<V1_ParserError>,
+          ),
+        );
+      }
+      throw error;
+    }
+  }
 }

packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineServerClient.ts

@@ -77,11 +77,14 @@ import type {
   V1_ArtifactGenerationExtensionInput,
   V1_ArtifactGenerationExtensionOutput,
 } from './generation/V1_ArtifactGenerationExtensionApi.js';
+import type { V1_DatabaseToModelGenerationInput } from './relational/V1_DatabaseToModelGenerationInput.js';
 
 enum CORE_ENGINE_ACTIVITY_TRACE {
   GRAMMAR_TO_JSON = 'transform Pure code to protocol',
   JSON_TO_GRAMMAR = 'transform protocol to Pure code',
 
+  DATABASE_TO_MODELS = 'generate models from database',
+
   EXTERNAL_FORMAT_TO_PROTOCOL = 'transform external format code to protocol',
   GENERATE_FILE = 'generate file',
 
@@ -774,6 +777,20 @@ export class V1_EngineServerClient extends AbstractServerClient {
     );
   }
 
+  // ------------------------------------------- Relational ---------------------------------------
+
+  _relationalElement = (): string => `${this._pure()}/relational`;
+
+  generateModelsFromDatabaseSpecification(
+    input: PlainObject<V1_DatabaseToModelGenerationInput>,
+  ): Promise<PlainObject<V1_PureModelContextData>> {
+    return this.postWithTracing(
+      this.getTraceData(CORE_ENGINE_ACTIVITY_TRACE.DATABASE_TO_MODELS),
+      `${this._relationalElement()}/generateModelsFromDatabaseSpecification`,
+      this.debugPayload(input, CORE_ENGINE_ACTIVITY_TRACE.DATABASE_TO_MODELS),
+    );
+  }
+
   // ------------------------------------------- Service -------------------------------------------
 
   _service = (serviceServerUrl?: string): string =>

packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/relational/V1_DatabaseToModelGenerationInput.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { primitive, createModelSchema } from 'serializr';
+import { SerializationFactory } from '@finos/legend-shared';
+import type { V1_PureModelContextData } from '../../model/context/V1_PureModelContextData.js';
+import { V1_pureModelContextDataPropSchema } from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js';
+
+export class V1_DatabaseToModelGenerationInput {
+  databasePath!: string;
+  modelData!: V1_PureModelContextData;
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(V1_DatabaseToModelGenerationInput, {
+      databasePath: primitive(),
+      modelData: V1_pureModelContextDataPropSchema,
+    }),
+  );
+}