Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generic handlers & DE OMAS performance improvements #7412

Merged
merged 7 commits into from
Mar 9, 2023
Merged

Generic handlers & DE OMAS performance improvements #7412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
14d00e2
Remove latestChange anchor creation for uncheckedLinkElement
popa-raluca Feb 14, 2023
f350cb1
DE OMAS - setAnchorGUID for SchemaType & SchemaAttribute
popa-raluca Feb 15, 2023
4f516e8
DE OMAS - enable junit5 tests in gradle build
popa-raluca Feb 15, 2023
acd9c3d
DE OMAS - fix gradle build
popa-raluca Feb 16, 2023
d444711
DE OMAS - fix unit tests
popa-raluca Feb 16, 2023
5c65afa
Merge branch 'main' into de-anchors
popa-raluca Mar 7, 2023
9d13554
Merge branch 'main' into de-anchors
Mar 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions open-metadata-implementation/access-services/data-engine/data-engine-client/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@ description = 'Data Engine OMAS Client'
java {
withJavadocJar()
}

5 changes: 0 additions & 5 deletions open-metadata-implementation/access-services/data-engine/data-engine-server/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,3 @@ java {
withJavadocJar()
}

test {
useTestNG()
// This will default to standard search pattern - see https://docs.gradle.org/current/userguide/java_testing.html#sec:test_detection
scanForTestClasses = false
}
2 changes: 1 addition & 1 deletion ...dpi/openmetadata/accessservices/dataengine/server/handlers/DataEngineDataFileHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public String upsertFileAssetIntoCatalog(String fileTypeName, String fileTypeGui
fileGuid = createFileInRepository(fileTypeName, fileTypeGuid, file, extendedProperties, externalSourceGuid,
externalSourceName, userId, methodName);
}
String schemaTypeGuid = dataEngineSchemaTypeHandler.upsertSchemaType(userId, schemaType, externalSourceName);
String schemaTypeGuid = dataEngineSchemaTypeHandler.upsertSchemaType(userId, schemaType, fileGuid, externalSourceName);
dataEngineCommonHandler.upsertExternalRelationship(userId, fileGuid, schemaTypeGuid, ASSET_TO_SCHEMA_TYPE_TYPE_NAME,
fileTypeName, SCHEMA_TYPE_TYPE_NAME, externalSourceName, null);
dataEngineFolderHierarchyHandler.upsertFolderHierarchy(fileGuid, file.getFileType(), file.getPathName(),
Expand Down
5 changes: 4 additions & 1 deletion ...i/openmetadata/accessservices/dataengine/server/handlers/DataEngineSchemaTypeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public DataEngineSchemaTypeHandler(String serviceName, String serverName, Invali
* @throws UserNotAuthorizedException user not authorized to issue this request
* @throws PropertyServerException problem accessing the property server
*/
public String upsertSchemaType(String userId, SchemaType schemaType, String externalSourceName) throws InvalidParameterException,
public String upsertSchemaType(String userId, SchemaType schemaType, String anchorGUID, String externalSourceName) throws InvalidParameterException,
PropertyServerException,
UserNotAuthorizedException {
String methodName = "upsertSchemaType";
Expand All @@ -107,6 +107,9 @@ public String upsertSchemaType(String userId, SchemaType schemaType, String exte
Optional<EntityDetail> originalSchemaTypeEntity = findSchemaTypeEntity(userId, schemaType.getQualifiedName());

SchemaTypeBuilder schemaTypeBuilder = getSchemaTypeBuilder(schemaType);
if (anchorGUID != null) {
schemaTypeBuilder.setAnchors(userId, anchorGUID, methodName);
}

String externalSourceGUID = dataEngineRegistrationHandler.getExternalDataEngine(userId, externalSourceName);

Expand Down
2 changes: 1 addition & 1 deletion ...rg/odpi/openmetadata/accessservices/dataengine/server/service/DataEngineRESTServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ public String upsertSchemaType(String userId, String serverName, String portImpl
DataEngineSchemaTypeHandler dataEngineSchemaTypeHandler = instanceHandler.getDataEngineSchemaTypeHandler(userId, serverName, methodName);
DataEnginePortHandler dataEnginePortHandler = instanceHandler.getPortHandler(userId, serverName, methodName);

String schemaTypeGUID = dataEngineSchemaTypeHandler.upsertSchemaType(userId, schemaType, externalSourceName);
String schemaTypeGUID = dataEngineSchemaTypeHandler.upsertSchemaType(userId, schemaType, portImplementationGUID, externalSourceName);
if (StringUtils.isNotEmpty(portImplementationGUID)) {
dataEnginePortHandler.addPortSchemaRelationship(userId, portImplementationGUID, schemaTypeGUID, methodName);
}
Expand Down
20 changes: 15 additions & 5 deletions ...openmetadata/accessservices/dataengine/server/handlers/DataEngineDataFileHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.odpi.openmetadata.repositoryservices.ffdc.exception.FunctionNotSupportedException;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -111,6 +110,11 @@ void insertCsvFileToCatalog() throws InvalidParameterException, PropertyServerEx
mockDataEngineCommonHandler(false);
mockDataEngineSchemaTypeHandler();

when(fileHandler.createAssetInRepository(USER, EXTERNAL_SOURCE_GUID, EXTERNAL_SOURCE_NAME, QUALIFIED_NAME, NAME, null,
DESCRIPTION, null, OWNER, 0, null, null,
null, null, CSV_FILE_TYPE_GUID, CSV_FILE_TYPE_NAME, getExtendedProperties(),
null, null, InstanceStatus.ACTIVE, null, METHOD)).thenReturn(GUID_VALUE);

String guid = dataEngineDataFileHandler.upsertFileAssetIntoCatalog(CSV_FILE_TYPE_NAME, CSV_FILE_TYPE_GUID, csvFile, schemaType,
getExtendedProperties(), EXTERNAL_SOURCE_GUID, EXTERNAL_SOURCE_NAME, USER, METHOD);

Expand All @@ -120,7 +124,7 @@ void insertCsvFileToCatalog() throws InvalidParameterException, PropertyServerEx
null, OWNER, 0, null, null,
null, null, CSV_FILE_TYPE_GUID, CSV_FILE_TYPE_NAME,
getExtendedProperties(), null, null, InstanceStatus.ACTIVE, null, METHOD);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_NAME);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, guid, EXTERNAL_SOURCE_NAME);
verify(dataEngineCommonHandler, times(1)).upsertExternalRelationship(USER, guid, SCHEMA_TYPE_GUID,
ASSET_TO_SCHEMA_TYPE_TYPE_NAME, CSV_FILE_TYPE_NAME, SCHEMA_TYPE_TYPE_NAME, EXTERNAL_SOURCE_NAME, null);
verify(dataEngineFolderHierarchyHandler, times(1)).upsertFolderHierarchy(guid, FILE_TYPE,
Expand All @@ -139,6 +143,12 @@ void insertIncompleteCsvFileToCatalog() throws InvalidParameterException, Proper
mockDataEngineCommonHandler(false);
mockDataEngineSchemaTypeHandler();

when(fileHandler.createAssetInRepository(USER, EXTERNAL_SOURCE_GUID, EXTERNAL_SOURCE_NAME, QUALIFIED_NAME, NAME, null,
DESCRIPTION, null, OWNER, 0, null, null,
null, null, CSV_FILE_TYPE_GUID, CSV_FILE_TYPE_NAME, getExtendedProperties(),
null, null, InstanceStatus.ACTIVE, null, METHOD)).thenReturn(GUID_VALUE);


String guid = dataEngineDataFileHandler.upsertFileAssetIntoCatalog(CSV_FILE_TYPE_NAME, CSV_FILE_TYPE_GUID, csvFile, schemaType,
getExtendedProperties(), EXTERNAL_SOURCE_GUID, EXTERNAL_SOURCE_NAME, USER, METHOD);

Expand All @@ -148,7 +158,7 @@ void insertIncompleteCsvFileToCatalog() throws InvalidParameterException, Proper
null, OWNER, 0, null, null,
null, null, CSV_FILE_TYPE_GUID, CSV_FILE_TYPE_NAME,
getExtendedProperties(), null, null, InstanceStatus.ACTIVE, null, METHOD);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_NAME);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, guid, EXTERNAL_SOURCE_NAME);
verify(dataEngineCommonHandler, times(1)).upsertExternalRelationship(USER, guid, SCHEMA_TYPE_GUID,
ASSET_TO_SCHEMA_TYPE_TYPE_NAME, CSV_FILE_TYPE_NAME, SCHEMA_TYPE_TYPE_NAME, EXTERNAL_SOURCE_NAME, null);
verify(dataEngineFolderHierarchyHandler, times(1)).upsertFolderHierarchy(guid, FILE_TYPE,
Expand Down Expand Up @@ -180,7 +190,7 @@ void updateCsvFileToCatalog() throws InvalidParameterException, PropertyServerEx
updateAsset(USER, EXTERNAL_SOURCE_GUID, EXTERNAL_SOURCE_NAME, GUID_VALUE, CommonMapper.GUID_PROPERTY_NAME,
QUALIFIED_NAME, NAME, null, DESCRIPTION, null, CSV_FILE_TYPE_GUID, CSV_FILE_TYPE_NAME,
getExtendedProperties(),null, null, true, false, false, null, METHOD);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_NAME);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, schemaType, guid, EXTERNAL_SOURCE_NAME);
verify(dataEngineCommonHandler, times(1)).upsertExternalRelationship(USER, guid, SCHEMA_TYPE_GUID,
ASSET_TO_SCHEMA_TYPE_TYPE_NAME, CSV_FILE_TYPE_NAME, SCHEMA_TYPE_TYPE_NAME, EXTERNAL_SOURCE_NAME, null);
verify(dataEngineFolderHierarchyHandler, times(1)).upsertFolderHierarchy(guid, FILE_TYPE,
Expand Down Expand Up @@ -239,7 +249,7 @@ private void mockDataEngineSchemaTypeHandler() throws InvalidParameterException,
SchemaType schemaType = getTabularSchema();
List<Attribute> columns = getTabularColumns();
schemaType.setAttributeList(columns);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_NAME)).thenReturn(SCHEMA_TYPE_GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, GUID_VALUE, EXTERNAL_SOURCE_NAME)).thenReturn(SCHEMA_TYPE_GUID);
}

private CSVFile getCsvFile() {
Expand Down
6 changes: 3 additions & 3 deletions ...enmetadata/accessservices/dataengine/server/handlers/DataEngineSchemaTypeHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ void createSchemaTypeWithSchemaAttribute() throws InvalidParameterException, Pro

mockTypeDef(TYPE_EMBEDDED_ATTRIBUTE_CLASSIFICATION_TYPE_NAME, TYPE_TO_ATTRIBUTE_RELATIONSHIP_TYPE_GUID);

String result = dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
String result = dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);

assertEquals(GUID, result);
verify(invalidParameterHandler, times(1)).validateUserId(USER, methodName);
Expand Down Expand Up @@ -161,7 +161,7 @@ void updateSchemaTypeWithSchemaAttribute() throws InvalidParameterException, Pro
when(dataEngineRegistrationHandler.getExternalDataEngine(USER, EXTERNAL_SOURCE_DE_QUALIFIED_NAME))
.thenReturn(EXTERNAL_SOURCE_DE_GUID);

String result = dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
String result = dataEngineSchemaTypeHandler.upsertSchemaType(USER, schemaType, null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);

assertEquals(GUID, result);
verify(invalidParameterHandler, times(1)).validateUserId(USER, methodName);
Expand Down Expand Up @@ -200,7 +200,7 @@ void createSchemaType_throwsUserNotAuthorizedException() throws InvalidParameter
null, false, false, null,
methodName)).thenThrow(mockedException);
UserNotAuthorizedException thrown = assertThrows(UserNotAuthorizedException.class, () ->
dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME));
dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME));

assertTrue(thrown.getMessage().contains("OMAS-DATA-ENGINE-404-001 "));
}
Expand Down
17 changes: 9 additions & 8 deletions ...dpi/openmetadata/accessservices/dataengine/server/service/DataEngineRESTServicesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ void getExternalDataEngineByQualifiedName_ResponseWithCapturedInvalidParameterEx
void upsertSchemaType() throws InvalidParameterException, PropertyServerException, UserNotAuthorizedException {
mockSchemaTypeHandler("upsertSchemaType");

when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(GUID);

SchemaTypeRequestBody requestBody = mockSchemaTypeRequestBody();

Expand All @@ -366,7 +366,7 @@ void upsertSchemaType_ResponseWithCapturedInvalidParameterException() throws Inv
mockSchemaTypeHandler(methodName);

InvalidParameterException mockedException = mockException(InvalidParameterException.class, methodName);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenThrow(mockedException);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenThrow(mockedException);

SchemaTypeRequestBody requestBody = mockSchemaTypeRequestBody();

Expand All @@ -387,7 +387,7 @@ void upsertSchemaType_ResponseWithCapturedUserNotAuthorizedException() throws In
mockSchemaTypeHandler(methodName);

UserNotAuthorizedException mockedException = mockException(UserNotAuthorizedException.class, methodName);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenThrow(mockedException);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), null, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenThrow(mockedException);

SchemaTypeRequestBody requestBody = mockSchemaTypeRequestBody();

Expand All @@ -407,7 +407,7 @@ void createPortImplementation() throws InvalidParameterException, PropertyServer

when(dataEnginePortHandler.createPortImplementation(USER, portImplementation, PROCESS_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME))
.thenReturn(PORT_GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), PORT_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);

GUIDResponse response = dataEngineRESTServices.upsertPortImplementation(USER, SERVER_NAME, requestBody);

Expand Down Expand Up @@ -480,7 +480,7 @@ void updatePortImplementation() throws InvalidParameterException, PropertyServer

Optional<EntityDetail> portEntity = mockEntityDetail(PORT_GUID);
when(dataEnginePortHandler.findPortImplementationEntity(USER, QUALIFIED_NAME)).thenReturn(portEntity);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), PORT_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);
GUIDResponse response = dataEngineRESTServices.upsertPortImplementation(USER, SERVER_NAME, requestBody);

verify(dataEnginePortHandler, times(1)).updatePortImplementation(USER, portEntity.get(), portImplementation,
Expand Down Expand Up @@ -617,6 +617,7 @@ void createProcess() throws InvalidParameterException, PropertyServerException,
mockCollectionHandler("createCollection");
mockCollectionHandler("addProcessCollectionRelationship");

when(dataEnginePortHandler.createPortImplementation(USER, portImplementation, PROCESS_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(PORT_GUID);
when(dataEnginePortHandler.createPortAlias(USER, portAlias, PROCESS_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(GUID);

when(processHandler.createProcess(USER, process, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(PROCESS_GUID);
Expand All @@ -627,7 +628,7 @@ void createProcess() throws InvalidParameterException, PropertyServerException,

GUIDResponse response = dataEngineRESTServices.upsertProcess(USER, SERVER_NAME, requestBody);

verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, getSchemaType(), PORT_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEnginePortHandler, times(1)).createPortImplementation(USER, portImplementation, PROCESS_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEnginePortHandler, times(1)).addPortDelegationRelationship(USER, GUID, PortType.INOUT_PORT, DELEGATED_QUALIFIED_NAME,
EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
Expand Down Expand Up @@ -704,7 +705,7 @@ void updateProcess() throws InvalidParameterException, PropertyServerException,

EntityDetail mockedSchemaType = mockEntityDetailWithQualifiedName(OLD_SCHEMA_GUID, OLD_SCHEMA_QUALIFIED_NAME);
when(dataEnginePortHandler.findSchemaTypeForPort(USER, PORT_GUID)).thenReturn(Optional.of(mockedSchemaType));
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);
when(dataEngineSchemaTypeHandler.upsertSchemaType(USER, getSchemaType(), PORT_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(SCHEMA_GUID);

when(dataEnginePortHandler.createPortAlias(USER, portAlias, PROCESS_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME)).thenReturn(PORT_GUID);

Expand All @@ -718,7 +719,7 @@ void updateProcess() throws InvalidParameterException, PropertyServerException,

GUIDResponse response = dataEngineRESTServices.upsertProcess(USER, SERVER_NAME, requestBody);

verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, getSchemaType(), EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEngineSchemaTypeHandler, times(1)).upsertSchemaType(USER, getSchemaType(), PORT_GUID, EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEnginePortHandler, times(1)).updatePortImplementation(USER, portEntity.get(), portImplementation,
EXTERNAL_SOURCE_DE_QUALIFIED_NAME);
verify(dataEnginePortHandler, times(1)).addPortDelegationRelationship(USER, PORT_GUID, PortType.INOUT_PORT,
Expand Down
Loading