-
Notifications
You must be signed in to change notification settings - Fork 2.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[WIP] feat(gms): Adds custom ownership types #7623
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
namespace com.linkedin.metadata.key | ||
|
||
/** | ||
* Key for a Ownership Type | ||
*/ | ||
@Aspect = { | ||
"name": "ownershipTypeKey" | ||
} | ||
record OwnershipTypeKey { | ||
/** | ||
* Unique ID for the data ownership type name i.e. Business Owner, Data Steward, Technical Owner, etc.. | ||
* Should be separate from the name used for displaying an Ownership Type. | ||
*/ | ||
id: string | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
namespace com.linkedin.ownership | ||
|
||
import com.linkedin.common.AuditStamp | ||
|
||
/** | ||
* Information about an ownership type | ||
*/ | ||
@Aspect = { | ||
"name": "ownershipTypeInfo" | ||
} | ||
record OwnershipTypeInfo { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As far as I could see from other schemas, you'll need to add a URN in here too, since the types will become a dynamic asset from now on. Is that correct? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This aspect is merely an information view of the new entity that this PR introduces. The urn is in the key aspect. |
||
|
||
/** | ||
* Display name of the Ownership Type | ||
*/ | ||
@Searchable = { | ||
"fieldType": "TEXT_PARTIAL", | ||
"enableAutocomplete": true, | ||
"boostScore": 10.0 | ||
} | ||
name: string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. displayName might be a better name in here |
||
|
||
/** | ||
* Description of the Ownership Type | ||
*/ | ||
description: optional string | ||
|
||
/** | ||
* Created Audit stamp | ||
*/ | ||
@Searchable = { | ||
"/time": { | ||
"fieldName": "createdTime", | ||
"fieldType": "DATETIME" | ||
} | ||
} | ||
createdAt: optional AuditStamp | ||
|
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
package com.linkedin.metadata.boot.factories; | ||
|
||
import com.linkedin.gms.factory.entity.EntityServiceFactory; | ||
import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; | ||
import com.linkedin.metadata.boot.steps.IngestOwnershipTypesStep; | ||
import com.linkedin.metadata.entity.EntityService; | ||
import javax.annotation.Nonnull; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.beans.factory.annotation.Qualifier; | ||
import org.springframework.beans.factory.annotation.Value; | ||
import org.springframework.context.annotation.Bean; | ||
import org.springframework.context.annotation.Configuration; | ||
import org.springframework.context.annotation.Import; | ||
import org.springframework.context.annotation.PropertySource; | ||
import org.springframework.context.annotation.Scope; | ||
|
||
|
||
@Configuration | ||
@Import({EntityServiceFactory.class}) | ||
@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) | ||
public class IngestOwnershipTypesStepFactory { | ||
|
||
@Autowired | ||
@Qualifier("entityService") | ||
private EntityService _entityService; | ||
|
||
@Value("${bootstrap.ingestDefaultOwnershipTypes.enabled}") | ||
private Boolean _enableOwnershipTypeBootstrap; | ||
|
||
@Bean(name = "ingestMetadataTestsStep") | ||
@Scope("singleton") | ||
@Nonnull | ||
protected IngestOwnershipTypesStep createInstance() { | ||
return new IngestOwnershipTypesStep(_entityService, _entityService.getEntityRegistry(), _enableOwnershipTypeBootstrap); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
package com.linkedin.metadata.boot.steps; | ||
|
||
import com.datahub.util.RecordUtils; | ||
import com.fasterxml.jackson.databind.JsonNode; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.linkedin.common.AuditStamp; | ||
import com.linkedin.common.urn.Urn; | ||
import com.linkedin.events.metadata.ChangeType; | ||
import com.linkedin.metadata.Constants; | ||
import com.linkedin.metadata.boot.BootstrapStep; | ||
import com.linkedin.metadata.entity.EntityService; | ||
import com.linkedin.metadata.models.AspectSpec; | ||
import com.linkedin.metadata.models.registry.EntityRegistry; | ||
import com.linkedin.metadata.utils.EntityKeyUtils; | ||
import com.linkedin.metadata.utils.GenericRecordUtils; | ||
import com.linkedin.mxe.GenericAspect; | ||
import com.linkedin.mxe.MetadataChangeProposal; | ||
import com.linkedin.ownership.OwnershipTypeInfo; | ||
import javax.annotation.Nonnull; | ||
import lombok.RequiredArgsConstructor; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.springframework.core.io.ClassPathResource; | ||
|
||
import static com.linkedin.metadata.Constants.*; | ||
|
||
|
||
/** | ||
* This bootstrap step is responsible for ingesting default ownership types. | ||
* <p></p> | ||
* If system has never bootstrapped this step will: | ||
* For each ownership type defined in the yaml file, it checks whether the urn exists. | ||
* If not, it ingests the ownership type into DataHub. | ||
*/ | ||
@Slf4j | ||
@RequiredArgsConstructor | ||
public class IngestOwnershipTypesStep implements BootstrapStep { | ||
|
||
private static final String UPGRADE_ID = "ingest-default-metadata-ownership-types"; | ||
private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID); | ||
|
||
private static final int SLEEP_SECONDS = 60; | ||
|
||
private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); | ||
|
||
private final EntityService _entityService; | ||
|
||
private final EntityRegistry _entityRegistry; | ||
|
||
private final boolean _enableOwnershipTypeBootstrap; | ||
|
||
@Override | ||
public String name() { | ||
return "IngestOwnershipTypesStep"; | ||
} | ||
|
||
@Override | ||
public void execute() throws Exception { | ||
// 0. Execute preflight check to see whether we need to ingest Roles | ||
// If ownership bootstrap is disabled, skip | ||
if (!_enableOwnershipTypeBootstrap) { | ||
log.info("{} disabled. Skipping.", this.name()); | ||
return; | ||
} | ||
|
||
if (_entityService.exists(UPGRADE_ID_URN)) { | ||
log.info("Default ownership types were already ingested. Skipping ingesting again."); | ||
return; | ||
} | ||
|
||
log.info("Ingesting default ownership types..."); | ||
|
||
// Sleep to ensure deployment process finishes. | ||
Thread.sleep(SLEEP_SECONDS * 1000); | ||
|
||
// 1. Read from the file into JSON. | ||
final JsonNode ownershipTypesObj = JSON_MAPPER.readTree(new ClassPathResource("./boot/ownership_types.json") | ||
.getFile()); | ||
|
||
if (!ownershipTypesObj.isArray()) { | ||
throw new RuntimeException(String.format("Found malformed ownership file, expected an Array but found %s", | ||
ownershipTypesObj.getNodeType())); | ||
} | ||
|
||
final AspectSpec ownershipTypeInfoAspectSpec = | ||
_entityRegistry.getEntitySpec(OWNERSHIP_TYPE_ENTITY_NAME).getAspectSpec(OWNERSHIP_TYPE_INFO_ASPECT_NAME); | ||
final AuditStamp auditStamp = | ||
new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); | ||
|
||
log.info("Ingesting {} ownership types", ownershipTypesObj.size()); | ||
int numIngested = 0; | ||
for (final JsonNode roleObj : ownershipTypesObj) { | ||
final Urn urn = Urn.createFromString(roleObj.get("urn").asText()); | ||
|
||
// If the info is not there, it means that the ownership type was there before, but must now be removed | ||
if (!roleObj.has("info")) { | ||
log.warn("Could not find info aspect for ownership type urn: {}. This means that the ownership type was there " | ||
+ "before, but must now be removed.", urn); | ||
_entityService.deleteUrn(urn); | ||
continue; | ||
} | ||
|
||
final OwnershipTypeInfo info = RecordUtils.toRecordTemplate(OwnershipTypeInfo.class, roleObj.get("info") | ||
.toString()); | ||
ingestOwnershipType(urn, info, auditStamp, ownershipTypeInfoAspectSpec); | ||
numIngested++; | ||
} | ||
log.info("Ingested {} new ownership types", numIngested); | ||
} | ||
|
||
private void ingestOwnershipType(final Urn ownershipTypeUrn, final OwnershipTypeInfo info, final AuditStamp auditStamp, | ||
final AspectSpec ownershipTypeInfoAspectSpec) { | ||
|
||
// 3. Write key & aspect | ||
final MetadataChangeProposal keyAspectProposal = new MetadataChangeProposal(); | ||
final AspectSpec keyAspectSpec = _entityService.getKeyAspectSpec(ownershipTypeUrn); | ||
GenericAspect aspect = | ||
GenericRecordUtils.serializeAspect(EntityKeyUtils.convertUrnToEntityKey(ownershipTypeUrn, keyAspectSpec)); | ||
keyAspectProposal.setAspect(aspect); | ||
keyAspectProposal.setAspectName(keyAspectSpec.getName()); | ||
keyAspectProposal.setEntityType(OWNERSHIP_TYPE_ENTITY_NAME); | ||
keyAspectProposal.setChangeType(ChangeType.UPSERT); | ||
keyAspectProposal.setEntityUrn(ownershipTypeUrn); | ||
|
||
_entityService.ingestProposal(keyAspectProposal, auditStamp, false); | ||
|
||
final MetadataChangeProposal proposal = new MetadataChangeProposal(); | ||
proposal.setEntityUrn(ownershipTypeUrn); | ||
proposal.setEntityType(OWNERSHIP_TYPE_ENTITY_NAME); | ||
proposal.setAspectName(OWNERSHIP_TYPE_INFO_ASPECT_NAME); | ||
info.setCreatedAt(auditStamp); // Set optional createdAt field for bootstrapped ownership types. | ||
proposal.setAspect(GenericRecordUtils.serializeAspect(info)); | ||
proposal.setChangeType(ChangeType.UPSERT); | ||
|
||
_entityService.ingestProposal(proposal, auditStamp, false); | ||
|
||
_entityService.produceMetadataChangeLog(ownershipTypeUrn, OWNERSHIP_TYPE_ENTITY_NAME, OWNERSHIP_TYPE_INFO_ASPECT_NAME, | ||
ownershipTypeInfoAspectSpec, null, info, null, null, auditStamp, | ||
ChangeType.RESTATE); | ||
} | ||
|
||
@Nonnull | ||
@Override | ||
public ExecutionMode getExecutionMode() { | ||
return ExecutionMode.ASYNC; | ||
} | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
WDYT about adding a default to this field, it can be "None" as default.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Also, a name like
ownershipTypeUrn
would fit better in here. WDYT? Are you planning to migrate current owner definitions to the new one in this pr?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hello @mmmeeedddsss this PR includes a DataHub Bootstrap step that will populate the database with new ownership type entity instances based on the existing non-deprecated ownership types.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@pedro93 As far as I can see, these custom values are not used in anywhere. Do you plan to open a separate pr for the usage of these? (Such as graphql changes, frontend changes etc)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This part is only the modelling and bootstrapping part of the feature, it was opened prematurely. I will close it and continue development in a fork. Once it is feature complete I will open the PR again.