[WIP] feat(gms): Adds custom ownership types

li-utils/src/main/java/com/linkedin/metadata/Constants.java

@@ -55,6 +55,7 @@ public class Constants {
   public static final String DATAHUB_STEP_STATE_ENTITY_NAME = "dataHubStepState";
   public static final String DATAHUB_VIEW_ENTITY_NAME = "dataHubView";
   public static final String QUERY_ENTITY_NAME = "query";
+  public static final String OWNERSHIP_TYPE_ENTITY_NAME = "ownershipType";
 
   /**
    * Aspects
@@ -257,6 +258,9 @@ public class Constants {
   public static final String QUERY_PROPERTIES_ASPECT_NAME = "queryProperties";
   public static final String QUERY_SUBJECTS_ASPECT_NAME = "querySubjects";
 
+  // Ownership Types
+  public static final String OWNERSHIP_TYPE_INFO_ASPECT_NAME = "ownershipTypeInfo";
+
   // Settings
   public static final String GLOBAL_SETTINGS_ENTITY_NAME = "globalSettings";
   public static final String GLOBAL_SETTINGS_INFO_ASPECT_NAME = "globalSettingsInfo";

metadata-ingestion/src/datahub/cli/quickstart_versioning.py

@@ -29,7 +29,7 @@ def _is_it_a_version(version: str) -> bool:
     :param version: The string to check.
     :return: True if the string is a valid version, False otherwise.
     """
-    return re.match(r"v?\d+\.\d+(\.\d+)?", version) is not None
+    return re.match(r"^v?\d+\.\d+(\.\d+)?$", version) is not None
 
 
 class QuickstartVersionMappingConfig(BaseModel):
@@ -122,7 +122,7 @@ def get_quickstart_execution_plan(
         )
         # new CLI version is downloading the composefile corresponding to the requested version
         # if the version is older than v0.10.1, it doesn't contain the setup job labels and the
-        # the checks will fail, so in those cases we pick the composefile from v0.10.1 which contains
+        # checks will fail, so in those cases we pick the composefile from v0.10.1 which contains
         # the setup job labels
         if _is_it_a_version(result.composefile_git_ref):
             if parse("v0.10.1") > parse(result.composefile_git_ref):

metadata-models/src/main/pegasus/com/linkedin/common/Owner.pdl

@@ -26,8 +26,19 @@ record Owner {
   /**
    * The type of the ownership
    */
+  @deprecated
   type: OwnershipType
 
+  /**
+   * The type of the ownership
+   * Urn of type O
+   */
+  @Relationship = {
+    "name": "ownershipType",
+    "entityTypes": [ "ownershipType" ]
+  }
+  customType: optional Urn
+
   /**
    * Source information for the ownership
    */

metadata-models/src/main/pegasus/com/linkedin/metadata/key/OwnershipTypeKey.pdl

@@ -0,0 +1,15 @@
+namespace com.linkedin.metadata.key
+
+/**
+ * Key for a Ownership Type
+ */
+@Aspect = {
+  "name": "ownershipTypeKey"
+}
+record OwnershipTypeKey {
+  /**
+   * Unique ID for the data ownership type name i.e. Business Owner, Data Steward, Technical Owner, etc..
+   * Should be separate from the name used for displaying an Ownership Type.
+   */
+  id: string
+}

metadata-models/src/main/pegasus/com/linkedin/ownership/OwnershipTypeInfo.pdl

@@ -0,0 +1,39 @@
+namespace com.linkedin.ownership
+
+import com.linkedin.common.AuditStamp
+
+/**
+ * Information about an ownership type
+ */
+@Aspect = {
+  "name": "ownershipTypeInfo"
+}
+record OwnershipTypeInfo {
+
+  /**
+   * Display name of the Ownership Type
+   */
+  @Searchable = {
+    "fieldType": "TEXT_PARTIAL",
+    "enableAutocomplete": true,
+    "boostScore": 10.0
+  }
+  name: string
+
+  /**
+   * Description of the Ownership Type
+   */
+  description: optional string
+
+  /**
+   * Created Audit stamp
+   */
+   @Searchable = {
+     "/time": {
+      "fieldName": "createdTime",
+      "fieldType": "DATETIME"
+     }
+   }
+   createdAt: optional AuditStamp
+
+}

metadata-models/src/main/resources/entity-registry.yml

@@ -297,4 +297,11 @@ entities:
       - queryProperties
       - querySubjects
       - status
+  - name: ownershipType
+    doc: Ownership Type represents a created-runtime concept ownership for a person or group who is responsible for technical aspects of the asset.
+    category: core
+    keyAspect: ownershipTypeKey
+    aspects:
+      - ownershipTypeInfo
+      - status
 events:

metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java

@@ -13,6 +13,7 @@
 import com.linkedin.metadata.boot.steps.IngestDataPlatformInstancesStep;
 import com.linkedin.metadata.boot.steps.IngestDataPlatformsStep;
 import com.linkedin.metadata.boot.steps.IngestDefaultGlobalSettingsStep;
+import com.linkedin.metadata.boot.steps.IngestOwnershipTypesStep;
 import com.linkedin.metadata.boot.steps.IngestPoliciesStep;
 import com.linkedin.metadata.boot.steps.IngestRetentionPoliciesStep;
 import com.linkedin.metadata.boot.steps.IngestRolesStep;
@@ -69,6 +70,10 @@ public class BootstrapManagerFactory {
   @Qualifier("ingestRetentionPoliciesStep")
   private IngestRetentionPoliciesStep _ingestRetentionPoliciesStep;
 
+  @Autowired
+  @Qualifier("ingestMetadataTestsStep")
+  private IngestOwnershipTypesStep _ingestOwnershipTypesStep;
+
   @Autowired
   @Qualifier("dataHubUpgradeKafkaListener")
   private BootstrapDependency _dataHubUpgradeKafkaListener;
@@ -110,6 +115,7 @@ protected BootstrapManager createInstance() {
             ingestDataPlatformsStep,
             ingestDataPlatformInstancesStep,
             _ingestRetentionPoliciesStep,
+            _ingestOwnershipTypesStep,
             ingestSettingsStep,
             restoreGlossaryIndicesStep,
             removeClientIdAspectStep,

metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestOwnershipTypesStepFactory.java

@@ -0,0 +1,36 @@
+package com.linkedin.metadata.boot.factories;
+
+import com.linkedin.gms.factory.entity.EntityServiceFactory;
+import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
+import com.linkedin.metadata.boot.steps.IngestOwnershipTypesStep;
+import com.linkedin.metadata.entity.EntityService;
+import javax.annotation.Nonnull;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.context.annotation.Scope;
+
+
+@Configuration
+@Import({EntityServiceFactory.class})
+@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class)
+public class IngestOwnershipTypesStepFactory {
+
+  @Autowired
+  @Qualifier("entityService")
+  private EntityService _entityService;
+
+  @Value("${bootstrap.ingestDefaultOwnershipTypes.enabled}")
+  private Boolean _enableOwnershipTypeBootstrap;
+
+  @Bean(name = "ingestMetadataTestsStep")
+  @Scope("singleton")
+  @Nonnull
+  protected IngestOwnershipTypesStep createInstance() {
+    return new IngestOwnershipTypesStep(_entityService, _entityService.getEntityRegistry(), _enableOwnershipTypeBootstrap);
+  }
+}

metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java

@@ -0,0 +1,147 @@
+package com.linkedin.metadata.boot.steps;
+
+import com.datahub.util.RecordUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.boot.BootstrapStep;
+import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.metadata.utils.EntityKeyUtils;
+import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.mxe.GenericAspect;
+import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.ownership.OwnershipTypeInfo;
+import javax.annotation.Nonnull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.ClassPathResource;
+
+import static com.linkedin.metadata.Constants.*;
+
+
+/**
+ * This bootstrap step is responsible for ingesting default ownership types.
+ * <p></p>
+ * If system has never bootstrapped this step will:
+ * For each ownership type defined in the yaml file, it checks whether the urn exists.
+ * If not, it ingests the ownership type into DataHub.
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class IngestOwnershipTypesStep implements BootstrapStep {
+
+  private static final String UPGRADE_ID = "ingest-default-metadata-ownership-types";
+  private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID);
+
+  private static final int SLEEP_SECONDS = 60;
+
+  private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
+
+  private final EntityService _entityService;
+
+  private final EntityRegistry _entityRegistry;
+
+  private final boolean _enableOwnershipTypeBootstrap;
+
+  @Override
+  public String name() {
+    return "IngestOwnershipTypesStep";
+  }
+
+  @Override
+  public void execute() throws Exception {
+    // 0. Execute preflight check to see whether we need to ingest Roles
+    // If ownership bootstrap is disabled, skip
+    if (!_enableOwnershipTypeBootstrap) {
+      log.info("{} disabled. Skipping.", this.name());
+      return;
+    }
+
+    if (_entityService.exists(UPGRADE_ID_URN)) {
+      log.info("Default ownership types were already ingested. Skipping ingesting again.");
+      return;
+    }
+
+    log.info("Ingesting default ownership types...");
+
+    // Sleep to ensure deployment process finishes.
+    Thread.sleep(SLEEP_SECONDS * 1000);
+
+    // 1. Read from the file into JSON.
+    final JsonNode ownershipTypesObj = JSON_MAPPER.readTree(new ClassPathResource("./boot/ownership_types.json")
+        .getFile());
+
+    if (!ownershipTypesObj.isArray()) {
+      throw new RuntimeException(String.format("Found malformed ownership file, expected an Array but found %s",
+              ownershipTypesObj.getNodeType()));
+    }
+
+    final AspectSpec ownershipTypeInfoAspectSpec =
+        _entityRegistry.getEntitySpec(OWNERSHIP_TYPE_ENTITY_NAME).getAspectSpec(OWNERSHIP_TYPE_INFO_ASPECT_NAME);
+    final AuditStamp auditStamp =
+        new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis());
+
+    log.info("Ingesting {} ownership types", ownershipTypesObj.size());
+    int numIngested = 0;
+    for (final JsonNode roleObj : ownershipTypesObj) {
+      final Urn urn = Urn.createFromString(roleObj.get("urn").asText());
+
+      // If the info is not there, it means that the ownership type was there before, but must now be removed
+      if (!roleObj.has("info")) {
+        log.warn("Could not find info aspect for ownership type urn: {}. This means that the ownership type was there "
+            + "before, but must now be removed.", urn);
+        _entityService.deleteUrn(urn);
+        continue;
+      }
+
+      final OwnershipTypeInfo info = RecordUtils.toRecordTemplate(OwnershipTypeInfo.class, roleObj.get("info")
+          .toString());
+      ingestOwnershipType(urn, info, auditStamp, ownershipTypeInfoAspectSpec);
+      numIngested++;
+    }
+    log.info("Ingested {} new ownership types", numIngested);
+  }
+
+  private void ingestOwnershipType(final Urn ownershipTypeUrn, final OwnershipTypeInfo info, final AuditStamp auditStamp,
+      final AspectSpec ownershipTypeInfoAspectSpec) {
+
+    // 3. Write key & aspect
+    final MetadataChangeProposal keyAspectProposal = new MetadataChangeProposal();
+    final AspectSpec keyAspectSpec = _entityService.getKeyAspectSpec(ownershipTypeUrn);
+    GenericAspect aspect =
+        GenericRecordUtils.serializeAspect(EntityKeyUtils.convertUrnToEntityKey(ownershipTypeUrn, keyAspectSpec));
+    keyAspectProposal.setAspect(aspect);
+    keyAspectProposal.setAspectName(keyAspectSpec.getName());
+    keyAspectProposal.setEntityType(OWNERSHIP_TYPE_ENTITY_NAME);
+    keyAspectProposal.setChangeType(ChangeType.UPSERT);
+    keyAspectProposal.setEntityUrn(ownershipTypeUrn);
+
+    _entityService.ingestProposal(keyAspectProposal, auditStamp, false);
+
+    final MetadataChangeProposal proposal = new MetadataChangeProposal();
+    proposal.setEntityUrn(ownershipTypeUrn);
+    proposal.setEntityType(OWNERSHIP_TYPE_ENTITY_NAME);
+    proposal.setAspectName(OWNERSHIP_TYPE_INFO_ASPECT_NAME);
+    info.setCreatedAt(auditStamp); // Set optional createdAt field for bootstrapped ownership types.
+    proposal.setAspect(GenericRecordUtils.serializeAspect(info));
+    proposal.setChangeType(ChangeType.UPSERT);
+
+    _entityService.ingestProposal(proposal, auditStamp, false);
+
+    _entityService.produceMetadataChangeLog(ownershipTypeUrn, OWNERSHIP_TYPE_ENTITY_NAME, OWNERSHIP_TYPE_INFO_ASPECT_NAME,
+        ownershipTypeInfoAspectSpec, null, info, null, null, auditStamp,
+        ChangeType.RESTATE);
+  }
+
+  @Nonnull
+  @Override
+  public ExecutionMode getExecutionMode() {
+    return ExecutionMode.ASYNC;
+  }
+}
+

metadata-service/factories/src/main/resources/application.yml

@@ -247,6 +247,8 @@ siblings:
 bootstrap:
   upgradeDefaultBrowsePaths:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
+  ingestDefaultOwnershipTypes:
+    enabled: ${INGEST_DEFAULT_OWNERSHIP_TYPES:true} # enable to ingest default ownership types during datahub bootstrap stage.
 
 systemUpdate:
   initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000}

metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json

@@ -1139,7 +1139,17 @@
           "STAKEHOLDER" : true
         }
       },
-      "doc" : "The type of the ownership"
+      "doc" : "The type of the ownership",
+      "deprecated" : true
+    }, {
+      "name" : "customType",
+      "type" : "Urn",
+      "doc" : "The type of the ownership\nUrn of type O",
+      "optional" : true,
+      "Relationship" : {
+        "entityTypes" : [ "ownershipType" ],
+        "name" : "ownershipType"
+      }
     }, {
       "name" : "source",
       "type" : {

metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json

@@ -1150,7 +1150,17 @@
           "STAKEHOLDER" : true
         }
       },
-      "doc" : "The type of the ownership"
+      "doc" : "The type of the ownership",
+      "deprecated" : true
+    }, {
+      "name" : "customType",
+      "type" : "Urn",
+      "doc" : "The type of the ownership\nUrn of type O",
+      "optional" : true,
+      "Relationship" : {
+        "entityTypes" : [ "ownershipType" ],
+        "name" : "ownershipType"
+      }
     }, {
       "name" : "source",
       "type" : {