Merge branch 'master' into feat/platform_instance_ownerhip_in_policies

.github/scripts/check_event_type.py

@@ -1,7 +1,7 @@
 import sys
 
 java_events = set()
-with open("./metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
+with open("./metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
     for line in java_file:
         if '''Event"''' not in line:
             continue

build.gradle

@@ -129,6 +129,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20230227',
+    'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
     'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",

datahub-graphql-core/build.gradle

@@ -7,6 +7,8 @@ dependencies {
     compile project(':metadata-service:restli-client')
     compile project(':metadata-service:auth-impl')
     compile project(':metadata-service:auth-config')
+    compile project(':metadata-service:configuration')
+    compile project(':metadata-service:services')
     compile project(':metadata-io')
     compile project(':metadata-utils')
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -37,7 +37,6 @@
 import com.linkedin.datahub.graphql.generated.CorpUser;
 import com.linkedin.datahub.graphql.generated.CorpUserInfo;
 import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings;
-import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
 import com.linkedin.datahub.graphql.generated.Dashboard;
 import com.linkedin.datahub.graphql.generated.DashboardInfo;
 import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
@@ -63,9 +62,9 @@
 import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
 import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
-import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult;
 import com.linkedin.datahub.graphql.generated.ListDomainsResult;
 import com.linkedin.datahub.graphql.generated.ListGroupsResult;
+import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult;
 import com.linkedin.datahub.graphql.generated.ListQueriesResult;
 import com.linkedin.datahub.graphql.generated.ListTestsResult;
 import com.linkedin.datahub.graphql.generated.ListViewsResult;
@@ -80,6 +79,7 @@
 import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
 import com.linkedin.datahub.graphql.generated.Notebook;
 import com.linkedin.datahub.graphql.generated.Owner;
+import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue;
 import com.linkedin.datahub.graphql.generated.QueryEntity;
 import com.linkedin.datahub.graphql.generated.QuerySubject;
@@ -196,9 +196,9 @@
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver;
 import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver;
+import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.DeleteOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.ListOwnershipTypesResolver;
-import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.ownership.UpdateOwnershipTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver;
@@ -303,6 +303,7 @@
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
+import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -312,12 +313,11 @@
 import com.linkedin.metadata.recommendation.RecommendationsService;
 import com.linkedin.metadata.secret.SecretService;
 import com.linkedin.metadata.service.DataProductService;
+import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.service.OwnershipTypeService;
 import com.linkedin.metadata.service.QueryService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
-import com.linkedin.metadata.service.LineageService;
-import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.version.GitVersion;
@@ -340,6 +340,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.dataloader.BatchLoaderContextProvider;
@@ -355,6 +356,7 @@
  * A {@link GraphQLEngine} configured to provide access to the entities and aspects on the the GMS graph.
  */
 @Slf4j
+@Getter
 public class GmsGraphQLEngine {
 
     private final EntityClient entityClient;
@@ -613,7 +615,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
      * @param builder
      */
     private void configurePluginResolvers(final RuntimeWiring.Builder builder) {
-        this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder));
+        this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder, this));
     }
 
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java

@@ -16,6 +16,7 @@
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
+import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -28,7 +29,6 @@
 import com.linkedin.metadata.service.QueryService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
-import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
 import com.linkedin.metadata.version.GitVersion;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java

@@ -35,9 +35,10 @@ public interface GmsGraphQLPlugin {
 
   /**
    * Optional callback that a plugin can implement to configure any Query, Mutation or Type specific resolvers.
-   * @param wiringBuilder
+   * @param wiringBuilder : the builder being used to configure the runtime wiring
+   * @param baseEngine : a reference to the core engine and its graphql types
    */
-  default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder) {
+  default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder, final GmsGraphQLEngine baseEngine) {
 
   }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -26,8 +26,8 @@
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.ViewsConfiguration;
-import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
+import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.version.GitVersion;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.ingest.execution;
 
+import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.StringMap;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -9,7 +10,6 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.execution.ExecutionRequestInput;
 import com.linkedin.execution.ExecutionRequestSource;
-import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.key.ExecutionRequestKey;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.IngestionUtils;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java

@@ -135,7 +135,8 @@ public static void addTermsToResource(
   ) throws URISyntaxException {
     if (subResource == null || subResource.equals("")) {
       com.linkedin.common.GlossaryTerms terms =
-          (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms());
+          (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME,
+              entityService, new GlossaryTerms());
       terms.setAuditStamp(getAuditStamp(actor));
 
       if (!terms.hasTerms()) {
@@ -320,7 +321,8 @@ private static MetadataChangeProposal buildRemoveTagsToEntityProposal(
       EntityService entityService
   ) {
     com.linkedin.common.GlobalTags tags =
-        (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags());
+        (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME,
+            entityService, new GlobalTags());
 
     if (!tags.hasTags()) {
       tags.setTags(new TagAssociationArray());
@@ -357,7 +359,8 @@ private static MetadataChangeProposal buildAddTagsToEntityProposal(
       EntityService entityService
   ) throws URISyntaxException {
     com.linkedin.common.GlobalTags tags =
-        (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags());
+        (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME,
+            entityService, new GlobalTags());
 
     if (!tags.hasTags()) {
       tags.setTags(new TagAssociationArray());
@@ -449,7 +452,8 @@ private static MetadataChangeProposal buildAddTermsToEntityProposal(
       EntityService entityService
   ) throws URISyntaxException {
     com.linkedin.common.GlossaryTerms terms =
-        (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms());
+        (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME,
+            entityService, new GlossaryTerms());
     terms.setAuditStamp(getAuditStamp(actor));
 
     if (!terms.hasTerms()) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java

@@ -72,8 +72,7 @@ public static void removeOwnersFromResources(
   private static MetadataChangeProposal buildAddOwnersProposal(List<OwnerInput> owners, Urn resourceUrn, Urn actor, EntityService entityService) {
     Ownership ownershipAspect = (Ownership) getAspectFromEntity(
         resourceUrn.toString(),
-        Constants.OWNERSHIP_ASPECT_NAME,
-        entityService,
+        Constants.OWNERSHIP_ASPECT_NAME, entityService,
         new Ownership());
     for (OwnerInput input : owners) {
       addOwner(ownershipAspect, UrnUtils.getUrn(input.getOwnerUrn()), input.getType(), UrnUtils.getUrn(input.getOwnershipTypeUrn()));

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java

@@ -33,7 +33,6 @@
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES;
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.resolveView;
-import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*;
 
 
 @Slf4j

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java

@@ -1,6 +1,7 @@
 package com.linkedin.datahub.graphql.resolvers.ingest.execution;
 
 import com.datahub.authentication.Authentication;
+import com.linkedin.metadata.config.IngestionConfiguration;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -11,7 +12,6 @@
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
-import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.r2.RemoteInvocationException;
 import graphql.schema.DataFetchingEnvironment;

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java

@@ -1,10 +1,10 @@
 package com.linkedin.datahub.graphql.resolvers.ingest.execution;
 
 import com.datahub.authentication.Authentication;
+import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.CreateTestConnectionRequestInput;
 import com.linkedin.entity.client.EntityClient;
-import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetchingEnvironment;
 import org.mockito.Mockito;

datahub-upgrade/build.gradle

@@ -15,6 +15,7 @@ dependencies {
   compile project(':metadata-io')
   compile project(':metadata-service:factories')
   compile project(':metadata-service:restli-client')
+  compile project(':metadata-service:configuration')
   implementation externalDependency.charle
 
   compile externalDependency.javaxInject

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java

@@ -9,11 +9,11 @@
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.utils.PegasusUtils;
 import com.datahub.util.RecordUtils;
-import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV1;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV2;
 import com.linkedin.metadata.models.EntitySpec;

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java

@@ -33,8 +33,7 @@ public NoCodeUpgrade(
       final Authentication systemAuthentication,
       final RestliEntityClient entityClient) {
     _steps = buildUpgradeSteps(
-        server,
-        entityService,
+        server, entityService,
         entityRegistry,
         systemAuthentication,
         entityClient);

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java

@@ -1,11 +1,11 @@
 package com.linkedin.datahub.upgrade.system.elasticsearch.steps;
 
+import com.linkedin.metadata.config.search.ElasticSearchConfiguration;
 import com.linkedin.datahub.upgrade.UpgradeContext;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils;
-import com.linkedin.metadata.config.search.ElasticSearchConfiguration;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder;
 import com.linkedin.metadata.shared.ElasticSearchIndexed;
 import lombok.extern.slf4j.Slf4j;

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java

@@ -22,7 +22,7 @@ public class UpgradeCliApplicationTestConfiguration {
     private EbeanServer ebeanServer;
 
     @MockBean
-    private EntityService entityService;
+    private EntityService _entityService;
 
     @MockBean
     private SearchService searchService;

datahub-web-react/src/app/analytics/event.ts

@@ -245,7 +245,7 @@ export interface BrowseV2ToggleNodeEvent extends BaseEvent {
  */
 export interface BrowseV2SelectNodeEvent extends BaseEvent {
     type: EventType.BrowseV2SelectNodeEvent;
-    targetNode: 'browse';
+    targetNode: 'browse' | 'platform';
     action: 'select' | 'deselect';
     entity: string;
     environment?: string;

datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx

@@ -45,12 +45,14 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour
     const placeholderRecipe = getPlaceholderRecipe(ingestionSources, type);
 
     const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || placeholderRecipe);
+    const [stagedRecipeName, setStagedRecipeName] = useState(state.name);
 
     useEffect(() => {
         if (existingRecipeYaml) {
+            setStagedRecipeName(state.name);
             setStagedRecipeYml(existingRecipeYaml);
         }
-    }, [existingRecipeYaml]);
+    }, [existingRecipeYaml, state.name]);
 
     const [stepComplete, setStepComplete] = useState(false);
 
@@ -97,6 +99,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour
     if (type && CONNECTORS_WITH_FORM.has(type)) {
         return (
             <RecipeBuilder
+                key={stagedRecipeName}
                 state={state}
                 isEditing={isEditing}
                 displayRecipe={displayRecipe}

datahub-web-react/src/app/search/SearchResultList.tsx

@@ -48,6 +48,7 @@ const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>`
         margin: 0 auto 8px auto;
         padding: 8px 16px;
         max-width: 1200px;
+        border-bottom: 1px solid ${ANTD_GRAY[5]};
     `}
 `;