Merge branch 'master' into modify-menus

.github/workflows/docker-ingestion-base.yml

@@ -0,0 +1,40 @@
+name: ingestion base
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "docker/datahub-ingestion/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  build-base:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/datahub-ingestion
+          file: ./docker/datahub-ingestion/base.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: acryldata/datahub-ingestion-base:latest
+          push: true

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -84,6 +84,7 @@
 import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
 import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
 import com.linkedin.datahub.graphql.resolvers.domain.CreateDomainResolver;
+import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver;
 import com.linkedin.datahub.graphql.resolvers.domain.DomainEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver;
 import com.linkedin.datahub.graphql.resolvers.domain.SetDomainResolver;
@@ -153,6 +154,8 @@
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
+import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver;
+import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver;
 import com.linkedin.datahub.graphql.resolvers.test.CreateTestResolver;
 import com.linkedin.datahub.graphql.resolvers.test.DeleteTestResolver;
@@ -672,8 +675,10 @@ private String getUrnField(DataFetchingEnvironment env) {
     private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
         builder.type("Mutation", typeWiring -> typeWiring
             .dataFetcher("updateDataset", new MutableTypeResolver<>(datasetType))
+            .dataFetcher("createTag", new CreateTagResolver(this.entityClient))
             .dataFetcher("updateTag", new MutableTypeResolver<>(tagType))
             .dataFetcher("setTagColor", new SetTagColorResolver(entityClient, entityService))
+            .dataFetcher("deleteTag", new DeleteTagResolver(entityClient))
             .dataFetcher("updateChart", new MutableTypeResolver<>(chartType))
             .dataFetcher("updateDashboard", new MutableTypeResolver<>(dashboardType))
             .dataFetcher("updateNotebook", new MutableTypeResolver<>(notebookType))
@@ -703,6 +708,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient))
             .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient))
             .dataFetcher("createDomain", new CreateDomainResolver(this.entityClient))
+            .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient))
             .dataFetcher("setDomain", new SetDomainResolver(this.entityClient, this.entityService))
             .dataFetcher("updateDeprecation", new UpdateDeprecationResolver(this.entityClient, this.entityService))
             .dataFetcher("unsetDomain", new UnsetDomainResolver(this.entityClient, this.entityService))

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -5,14 +5,24 @@
 import com.datahub.authorization.Authorizer;
 import com.datahub.authorization.ResourceSpec;
 import com.google.common.collect.ImmutableList;
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.metadata.authorization.PoliciesConfig;
+import java.time.Clock;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 
 
 public class AuthorizationUtils {
 
+  private static final Clock CLOCK = Clock.systemUTC();
+
+  public static AuditStamp createAuditStamp(@Nonnull QueryContext context) {
+    return new AuditStamp().setTime(CLOCK.millis()).setActor(UrnUtils.getUrn(context.getActorUrn()));
+  }
+
   public static boolean canManageUsersAndGroups(@Nonnull QueryContext context) {
     return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_USERS_AND_GROUPS_PRIVILEGE);
   }
@@ -25,6 +35,24 @@ public static boolean canManageTokens(@Nonnull QueryContext context) {
     return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_ACCESS_TOKENS);
   }
 
+  /**
+   * Returns true if the current used is able to create Domains. This is true if the user has the 'Manage Domains' or 'Create Domains' platform privilege.
+   */
+  public static boolean canCreateDomains(@Nonnull QueryContext context) {
+    final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(
+        ImmutableList.of(
+            new ConjunctivePrivilegeGroup(ImmutableList.of(
+                PoliciesConfig.CREATE_DOMAINS_PRIVILEGE.getType())),
+            new ConjunctivePrivilegeGroup(ImmutableList.of(
+                PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType()))
+        ));
+
+    return AuthorizationUtils.isAuthorized(
+        context.getAuthorizer(),
+        context.getActorUrn(),
+        orPrivilegeGroups);
+  }
+
   public static boolean canManageDomains(@Nonnull QueryContext context) {
     return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE);
   }
@@ -33,6 +61,32 @@ public static boolean canManageGlossaries(@Nonnull QueryContext context) {
     return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE);
   }
 
+  /**
+   * Returns true if the current used is able to create Tags. This is true if the user has the 'Manage Tags' or 'Create Tags' platform privilege.
+   */
+  public static boolean canCreateTags(@Nonnull QueryContext context) {
+    final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(
+        ImmutableList.of(
+            new ConjunctivePrivilegeGroup(ImmutableList.of(
+            PoliciesConfig.CREATE_TAGS_PRIVILEGE.getType())),
+            new ConjunctivePrivilegeGroup(ImmutableList.of(
+                PoliciesConfig.MANAGE_TAGS_PRIVILEGE.getType()))
+        ));
+
+    return AuthorizationUtils.isAuthorized(
+        context.getAuthorizer(),
+        context.getActorUrn(),
+        orPrivilegeGroups);
+  }
+
+  public static boolean canManageTags(@Nonnull QueryContext context) {
+    return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE);
+  }
+
+  public static boolean canDeleteEntity(@Nonnull Urn entityUrn, @Nonnull QueryContext context) {
+    return isAuthorized(context, Optional.of(new ResourceSpec(entityUrn.getEntityType(), entityUrn.toString())), PoliciesConfig.DELETE_ENTITY_PRIVILEGE);
+  }
+
   public static boolean canManageUserCredentials(@Nonnull QueryContext context) {
     return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE);
   }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java

@@ -5,6 +5,7 @@
 import com.datahub.authorization.Authorizer;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
 import com.linkedin.datahub.graphql.generated.AuthenticatedUser;
 import com.linkedin.datahub.graphql.generated.CorpUser;
 import com.linkedin.datahub.graphql.generated.PlatformPrivileges;
@@ -65,6 +66,9 @@ public CompletableFuture<AuthenticatedUser> get(DataFetchingEnvironment environm
         platformPrivileges.setManageTests(canManageTests(context));
         platformPrivileges.setManageGlossaries(canManageGlossaries(context));
         platformPrivileges.setManageUserCredentials(canManageUserCredentials(context));
+        platformPrivileges.setCreateDomains(AuthorizationUtils.canCreateDomains(context));
+        platformPrivileges.setCreateTags(AuthorizationUtils.canCreateTags(context));
+        platformPrivileges.setManageTags(AuthorizationUtils.canManageTags(context));
 
         // Construct and return authenticated user object.
         final AuthenticatedUser authUser = new AuthenticatedUser();

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java

@@ -14,14 +14,17 @@
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.r2.RemoteInvocationException;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.concurrent.CompletableFuture;
+import lombok.extern.slf4j.Slf4j;
 
 
 /**
  * GraphQL Resolver that deletes an Assertion.
  */
+@Slf4j
 public class DeleteAssertionResolver implements DataFetcher<CompletableFuture<Boolean>>  {
 
   private final EntityClient _entityClient;
@@ -46,6 +49,16 @@ public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
       if (isAuthorizedToDeleteAssertion(context, assertionUrn)) {
           try {
             _entityClient.deleteEntity(assertionUrn, context.getAuthentication());
+
+            // Asynchronously Delete all references to the entity (to return quickly)
+            CompletableFuture.runAsync(() -> {
+              try {
+                _entityClient.deleteEntityReferences(assertionUrn, context.getAuthentication());
+              } catch (RemoteInvocationException e) {
+                log.error(String.format("Caught exception while attempting to clear all entity references for assertion with urn %s", assertionUrn), e);
+              }
+            });
+
             return true;
           } catch (Exception e) {
             throw new RuntimeException(String.format("Failed to perform delete against assertion with urn %s", assertionUrn), e);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java

@@ -1,19 +1,16 @@
 package com.linkedin.datahub.graphql.resolvers.domain;
 
-import com.google.common.collect.ImmutableList;
 import com.linkedin.data.template.SetMode;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
-import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup;
-import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.CreateDomainInput;
 import com.linkedin.domain.DomainProperties;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
-import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.key.DomainKey;
+import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetcher;
@@ -26,7 +23,7 @@
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
 
 /**
- * Resolver used for creating a new Domain on DataHub. Requires the MANAGE_DOMAINS privilege.
+ * Resolver used for creating a new Domain on DataHub. Requires the CREATE_DOMAINS or MANAGE_DOMAINS privilege.
  */
 @Slf4j
 @RequiredArgsConstructor
@@ -42,12 +39,10 @@ public CompletableFuture<String> get(DataFetchingEnvironment environment) throws
 
     return CompletableFuture.supplyAsync(() -> {
 
-      if (!isAuthorizedToCreateDomain(context)) {
+      if (!AuthorizationUtils.canCreateDomains(context)) {
         throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
       }
 
-      // TODO: Add exists check. Currently this can override previously created domains.
-
       try {
         // Create the Domain Key
         final DomainKey key = new DomainKey();
@@ -56,13 +51,18 @@ public CompletableFuture<String> get(DataFetchingEnvironment environment) throws
         final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString();
         key.setId(id);
 
+        if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.DOMAIN_ENTITY_NAME), context.getAuthentication())) {
+          throw new IllegalArgumentException("This Domain already exists!");
+        }
+
         // Create the MCP
         final MetadataChangeProposal proposal = new MetadataChangeProposal();
         proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key));
         proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME);
         proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME);
         proposal.setAspect(GenericRecordUtils.serializeAspect(mapDomainProperties(input)));
         proposal.setChangeType(ChangeType.UPSERT);
+
         return _entityClient.ingestProposal(proposal, context.getAuthentication());
       } catch (Exception e) {
         log.error("Failed to create Domain with id: {}, name: {}: {}", input.getId(), input.getName(), e.getMessage());
@@ -77,15 +77,4 @@ private DomainProperties mapDomainProperties(final CreateDomainInput input) {
     result.setDescription(input.getDescription(), SetMode.IGNORE_NULL);
     return result;
   }
-
-  private boolean isAuthorizedToCreateDomain(final QueryContext context) {
-    final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of(
-        new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType()))
-    ));
-
-    return AuthorizationUtils.isAuthorized(
-        context.getAuthorizer(),
-        context.getActorUrn(),
-        orPrivilegeGroups);
-  }
 }