-
Notifications
You must be signed in to change notification settings - Fork 114
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use PMCD returned by mappinganalysis to build minimal graph for query #3488
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
--- | ||
'@finos/legend-extension-dsl-data-space': minor | ||
'@finos/legend-application-query': minor | ||
--- | ||
|
||
Use PMCD returned by mapping analysis to build minimal graph for query |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
--- | ||
'@finos/legend-extension-dsl-data-space-studio': patch | ||
'@finos/legend-application-studio': patch | ||
'@finos/legend-query-builder': patch | ||
'@finos/legend-server-depot': patch | ||
'@finos/legend-graph': patch | ||
--- |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,9 +15,9 @@ | |
*/ | ||
|
||
import { | ||
type QuerySetupActionConfiguration, | ||
LegendQueryApplicationPlugin, | ||
QuerySetupActionTag, | ||
type QuerySetupActionConfiguration, | ||
} from '../stores/LegendQueryApplicationPlugin.js'; | ||
import packageJson from '../../package.json' with { type: 'json' }; | ||
import type { QuerySetupLandingPageStore } from '../stores/QuerySetupStore.js'; | ||
|
@@ -50,10 +50,10 @@ import { | |
LEGEND_QUERY_ROUTE_PATTERN, | ||
} from '../__lib__/LegendQueryNavigation.js'; | ||
import { | ||
ActionAlertActionType, | ||
ActionAlertType, | ||
type ApplicationPageEntry, | ||
type LegendApplicationSetup, | ||
ActionAlertActionType, | ||
ActionAlertType, | ||
} from '@finos/legend-application'; | ||
import { CloneQueryServiceSetup } from './CloneQueryServiceSetup.js'; | ||
import { QueryProductionizerSetup } from './QueryProductionizerSetup.js'; | ||
|
@@ -65,28 +65,60 @@ import { | |
generateDataSpaceQuerySetupRoute, | ||
} from '../__lib__/DSL_DataSpace_LegendQueryNavigation.js'; | ||
import { | ||
QUERY_BUILDER_SUPPORTED_GET_ALL_FUNCTIONS, | ||
type QueryBuilderState, | ||
type QueryBuilderHeaderActionConfiguration, | ||
type QueryBuilderMenuActionConfiguration, | ||
type QueryBuilderPropagateExecutionContextChangeHelper, | ||
QUERY_BUILDER_SUPPORTED_GET_ALL_FUNCTIONS, | ||
} from '@finos/legend-query-builder'; | ||
import { | ||
ExistingQueryEditorStore, | ||
QueryBuilderActionConfig_QueryApplication, | ||
} from '../stores/QueryEditorStore.js'; | ||
import { | ||
DataSpaceQueryBuilderState, | ||
DataSpacesDepotRepository, | ||
generateDataSpaceTemplateQueryPromotionRoute, | ||
} from '@finos/legend-extension-dsl-data-space/application'; | ||
import { RuntimePointer } from '@finos/legend-graph'; | ||
import { | ||
createGraphBuilderReport, | ||
GRAPH_MANAGER_EVENT, | ||
LegendSDLC, | ||
PackageableElementPointerType, | ||
resolvePackagePathAndElementName, | ||
RuntimePointer, | ||
V1_EngineRuntime, | ||
V1_Mapping, | ||
V1_PackageableElementPointer, | ||
V1_PackageableRuntime, | ||
V1_PureGraphManager, | ||
} from '@finos/legend-graph'; | ||
import { LegendQueryTelemetryHelper } from '../__lib__/LegendQueryTelemetryHelper.js'; | ||
import { StoreProjectData } from '@finos/legend-server-depot'; | ||
import { buildUrl } from '@finos/legend-shared'; | ||
import { resolveVersion, StoreProjectData } from '@finos/legend-server-depot'; | ||
import { | ||
ActionState, | ||
assertErrorThrown, | ||
buildUrl, | ||
getNullableFirstEntry, | ||
guaranteeNonNullable, | ||
guaranteeType, | ||
LogEvent, | ||
StopWatch, | ||
uniq, | ||
} from '@finos/legend-shared'; | ||
import { parseProjectIdentifier } from '@finos/legend-storage'; | ||
import { QueryEditorExistingQueryHeader } from './QueryEditor.js'; | ||
import { DataSpaceTemplateQueryCreatorStore } from '../stores/data-space/DataSpaceTemplateQueryCreatorStore.js'; | ||
import { createViewSDLCProjectHandler } from '../stores/data-space/DataSpaceQueryBuilderHelper.js'; | ||
import { DataSpaceQueryCreatorStore } from '../stores/data-space/DataSpaceQueryCreatorStore.js'; | ||
import { configureCodeEditorComponent } from '@finos/legend-lego/code-editor'; | ||
import { | ||
resolveUsableDataSpaceClasses, | ||
V1_DataSpace, | ||
V1_DataSpaceExecutionContext, | ||
} from '@finos/legend-extension-dsl-data-space/graph'; | ||
import { flowResult } from 'mobx'; | ||
import { LEGEND_QUERY_APP_EVENT } from '../__lib__/LegendQueryEvent.js'; | ||
|
||
export class Core_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { | ||
static NAME = packageJson.extensions.applicationQueryPlugin; | ||
|
@@ -725,4 +757,246 @@ export class Core_LegendQueryApplicationPlugin extends LegendQueryApplicationPlu | |
}, | ||
}; | ||
} | ||
|
||
getExtraQueryBuilderPropagateExecutionContextChangeHelper?(): QueryBuilderPropagateExecutionContextChangeHelper[] { | ||
YannanGao-gs marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return [ | ||
( | ||
queryBuilderState: QueryBuilderState, | ||
isGraphBuildingNotRequired?: boolean, | ||
): (() => Promise<void>) | undefined => { | ||
/** | ||
* Propagation after changing the execution context: | ||
* - The mapping will be updated to the mapping of the execution context | ||
* - The runtime will be updated to the default runtime of the execution context | ||
* - If no class is chosen, try to choose a compatible class | ||
* - If the chosen class is compatible with the new selected execution context mapping, do nothing, otherwise, try to choose a compatible class | ||
*/ | ||
const propagateExecutionContextChange = async (): Promise<void> => { | ||
YannanGao-gs marked this conversation as resolved.
Show resolved
Hide resolved
|
||
const dataSpaceQueryBuilderState = guaranteeType( | ||
queryBuilderState, | ||
DataSpaceQueryBuilderState, | ||
); | ||
const mapping = | ||
dataSpaceQueryBuilderState.executionContext.mapping.value; | ||
const mappingModelCoverageAnalysisResult = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult?.mappingToMappingCoverageResult?.get( | ||
mapping.path, | ||
); | ||
const editorStore = ( | ||
queryBuilderState.workflowState | ||
.actionConfig as QueryBuilderActionConfig_QueryApplication | ||
).editorStore; | ||
if ( | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult && | ||
mappingModelCoverageAnalysisResult | ||
) { | ||
if ( | ||
!isGraphBuildingNotRequired && | ||
dataSpaceQueryBuilderState.isLightGraphEnabled | ||
) { | ||
const supportBuildMinimalGraph = | ||
editorStore.applicationStore.config.options | ||
.TEMPORARY__enableMinimalGraph; | ||
if ( | ||
editorStore.enableMinialGraphForDataSpaceLoadingPerformance && | ||
supportBuildMinimalGraph | ||
) { | ||
try { | ||
const stopWatch = new StopWatch(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. additionally below code is repeated. |
||
const graph = | ||
dataSpaceQueryBuilderState.graphManagerState.createNewGraph(); | ||
const graph_buildReport = createGraphBuilderReport(); | ||
const graphManager = guaranteeType( | ||
dataSpaceQueryBuilderState.graphManagerState.graphManager, | ||
V1_PureGraphManager, | ||
); | ||
// Create dummy mappings and runtimes | ||
// TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should | ||
// simplify the model here or potentially refactor the backend analytics endpoint to return these as model | ||
const mappingModels = uniq( | ||
Array.from( | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), | ||
).map((context) => context.mapping), | ||
).map((m) => { | ||
const _mapping = new V1_Mapping(); | ||
const [packagePath, name] = | ||
resolvePackagePathAndElementName(m.path); | ||
_mapping.package = packagePath; | ||
_mapping.name = name; | ||
return graphManager.elementProtocolToEntity(_mapping); | ||
}); | ||
const runtimeModels = uniq( | ||
Array.from( | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), | ||
) | ||
.map((context) => context.defaultRuntime) | ||
.concat( | ||
Array.from( | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), | ||
).flatMap((val) => val.compatibleRuntimes), | ||
), | ||
).map((r) => { | ||
const runtime = new V1_PackageableRuntime(); | ||
const [packagePath, name] = | ||
resolvePackagePathAndElementName(r.path); | ||
runtime.package = packagePath; | ||
runtime.name = name; | ||
runtime.runtimeValue = new V1_EngineRuntime(); | ||
return graphManager.elementProtocolToEntity(runtime); | ||
}); | ||
// The DataSpace entity is excluded from AnalyticsResult.Json to reduce the JSON size | ||
// because all its information can be found in V1_DataSpaceAnalysisResult. | ||
// Therefore, we are building a simple v1_DataSpace entity based on V1_DataSpaceAnalysisResult. | ||
const dataspaceProtocol = new V1_DataSpace(); | ||
dataspaceProtocol.name = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.name; | ||
dataspaceProtocol.package = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.package; | ||
dataspaceProtocol.supportInfo = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.supportInfo; | ||
dataspaceProtocol.executionContexts = Array.from( | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult | ||
.executionContextsIndex, | ||
).map(([key, execContext]) => { | ||
const contextProtocol = new V1_DataSpaceExecutionContext(); | ||
contextProtocol.name = execContext.name; | ||
contextProtocol.title = execContext.title; | ||
contextProtocol.description = execContext.description; | ||
contextProtocol.mapping = new V1_PackageableElementPointer( | ||
PackageableElementPointerType.MAPPING, | ||
execContext.mapping.path, | ||
); | ||
contextProtocol.defaultRuntime = | ||
new V1_PackageableElementPointer( | ||
PackageableElementPointerType.RUNTIME, | ||
execContext.defaultRuntime.path, | ||
); | ||
return contextProtocol; | ||
}); | ||
dataspaceProtocol.defaultExecutionContext = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.defaultExecutionContext.name; | ||
dataspaceProtocol.title = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.title; | ||
dataspaceProtocol.description = | ||
dataSpaceQueryBuilderState.dataSpaceAnalysisResult.description; | ||
const dataspaceEntity = | ||
graphManager.elementProtocolToEntity(dataspaceProtocol); | ||
|
||
const graphEntities = guaranteeNonNullable( | ||
mappingModelCoverageAnalysisResult.entities, | ||
) | ||
.concat(mappingModels) | ||
.concat(runtimeModels) | ||
.concat(dataspaceEntity) | ||
// NOTE: if an element could be found in the graph already it means it comes from system | ||
// so we could rid of it | ||
.filter( | ||
(el) => | ||
!graph.getNullableElement(el.path, false) && | ||
!el.path.startsWith('meta::'), | ||
); | ||
let option; | ||
if ( | ||
dataSpaceQueryBuilderState.dataSpaceRepo instanceof | ||
DataSpacesDepotRepository | ||
) { | ||
option = new LegendSDLC( | ||
dataSpaceQueryBuilderState.dataSpaceRepo.project.groupId, | ||
dataSpaceQueryBuilderState.dataSpaceRepo.project.artifactId, | ||
resolveVersion( | ||
dataSpaceQueryBuilderState.dataSpaceRepo.project | ||
.versionId, | ||
), | ||
); | ||
} | ||
await dataSpaceQueryBuilderState.graphManagerState.graphManager.buildGraph( | ||
graph, | ||
graphEntities, | ||
ActionState.create(), | ||
option | ||
? { | ||
origin: option, | ||
} | ||
: {}, | ||
graph_buildReport, | ||
); | ||
dataSpaceQueryBuilderState.graphManagerState.graph = graph; | ||
const dependency_buildReport = createGraphBuilderReport(); | ||
// report | ||
stopWatch.record( | ||
GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS, | ||
); | ||
const graphBuilderReportData = { | ||
timings: | ||
dataSpaceQueryBuilderState.applicationStore.timeService.finalizeTimingsRecord( | ||
stopWatch, | ||
), | ||
dependencies: dependency_buildReport, | ||
dependenciesCount: | ||
dataSpaceQueryBuilderState.graphManagerState.graph | ||
.dependencyManager.numberOfDependencies, | ||
graph: graph_buildReport, | ||
}; | ||
editorStore.logBuildGraphMetrics(graphBuilderReportData); | ||
dataSpaceQueryBuilderState.applicationStore.logService.info( | ||
LogEvent.create( | ||
GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS, | ||
), | ||
graphBuilderReportData, | ||
); | ||
} catch (error) { | ||
assertErrorThrown(error); | ||
editorStore.applicationStore.logService.error( | ||
LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), | ||
error, | ||
); | ||
editorStore.graphManagerState.graph = | ||
editorStore.graphManagerState.createNewGraph(); | ||
await flowResult(editorStore.buildFullGraph()); | ||
} | ||
} else { | ||
editorStore.graphManagerState.graph = | ||
editorStore.graphManagerState.createNewGraph(); | ||
await flowResult(editorStore.buildFullGraph()); | ||
} | ||
} | ||
dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = | ||
mappingModelCoverageAnalysisResult; | ||
} | ||
const compatibleClasses = resolveUsableDataSpaceClasses( | ||
dataSpaceQueryBuilderState.dataSpace, | ||
mapping, | ||
dataSpaceQueryBuilderState.graphManagerState, | ||
dataSpaceQueryBuilderState, | ||
); | ||
dataSpaceQueryBuilderState.changeMapping(mapping); | ||
dataSpaceQueryBuilderState.changeRuntime( | ||
new RuntimePointer( | ||
dataSpaceQueryBuilderState.executionContext.defaultRuntime, | ||
), | ||
); | ||
// if there is no chosen class or the chosen one is not compatible | ||
// with the mapping then pick a compatible class if possible | ||
if ( | ||
!dataSpaceQueryBuilderState.class || | ||
!compatibleClasses.includes(dataSpaceQueryBuilderState.class) | ||
) { | ||
const possibleNewClass = getNullableFirstEntry(compatibleClasses); | ||
if (possibleNewClass) { | ||
dataSpaceQueryBuilderState.changeClass(possibleNewClass); | ||
} | ||
} | ||
dataSpaceQueryBuilderState.explorerState.refreshTreeData(); | ||
}; | ||
if ( | ||
queryBuilderState instanceof DataSpaceQueryBuilderState && | ||
queryBuilderState.workflowState.actionConfig instanceof | ||
QueryBuilderActionConfig_QueryApplication | ||
) { | ||
return propagateExecutionContextChange; | ||
} | ||
return undefined; | ||
}, | ||
]; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
dont think this is needed
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is needed as when trying to build minimal graph, mapping and pmcd are fetched from corresponding sexecutionContext key
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Still don't understand. DataspaceInfo should have the template info( which we get from the template id) which has the executon context we need ? right
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is the entry point of building DataspaceAnalyticsResult and light graph is built when building DataspaceAnalyticsResult.
Based on current implementation, template info is built at the last step