Merge branch 'master' into security-fixes-12-02

.github/actions/docker-custom-build-and-push/action.yml

@@ -68,13 +68,13 @@ runs:
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
       if: ${{ inputs.publish == 'true' }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
       if: ${{ inputs.publish == 'true' }}
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}

.github/workflows/docker-ingestion-base.yml

@@ -24,11 +24,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}

.github/workflows/docker-ingestion.yml

@@ -72,12 +72,12 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
         if: ${{ needs.setup.outputs.publish == 'true' }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-postgres-setup.yml

@@ -64,7 +64,7 @@ jobs:
           tag-custom-only: true
       - name: Login to DockerHub
         if: ${{ needs.setup.outputs.publish == 'true' }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.ORG_DOCKER_PASSWORD }}

.github/workflows/publish-datahub-jars.yml

@@ -1,4 +1,4 @@
-name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf, Ranger)
+name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf, Auth API)
 
 on:
   push:
@@ -142,7 +142,7 @@ jobs:
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-protobuf:publish
           ./gradlew :metadata-integration:java:datahub-protobuf:closeAndReleaseRepository --info
-      - name: publish datahub-ranger-plugin snapshot jar
+      - name: publish datahub-auth-api snapshot jar
         if: ${{ github.event_name != 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
@@ -153,9 +153,9 @@ jobs:
           NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
-          ./gradlew :datahub-ranger-plugin:printVersion
-          ./gradlew :datahub-ranger-plugin:publish
-      - name: release datahub-ranger-plugin jar
+          ./gradlew :metadata-auth:auth-api:printVersion
+          ./gradlew :metadata-auth:auth-api:publish
+      - name: release datahub-auth-api jar
         if: ${{ github.event_name == 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
@@ -166,5 +166,5 @@ jobs:
           NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
-          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :datahub-ranger-plugin:publish
-          ./gradlew :datahub-ranger-plugin:closeAndReleaseRepository --info
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-auth:auth-api:publish
+          ./gradlew :metadata-auth:auth-api:closeAndReleaseRepository --info

.gitignore

@@ -75,3 +75,5 @@ datahub-frontend/public/**
 datahub-frontend/test/resources/public/**
 
 .remote*
+# Ignore runtime generated authenticatior/authorizer jar files
+metadata-service/plugin/src/test/resources/sample-plugins/**

build.gradle

@@ -60,6 +60,8 @@ project.ext.externalDependency = [
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
     'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
+    'awsRds':'software.amazon.awssdk:rds:2.18.24',
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
@@ -92,6 +94,8 @@ project.ext.externalDependency = [
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
     'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
     'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
+    'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion",
+    'jacksonDataPropertyFormat': "com.fasterxml.jackson.dataformat:jackson-dataformat-properties:$jacksonVersion",
     'jacksonCore': "com.fasterxml.jackson.core:jackson-core:$jacksonVersion",
     'jacksonDataBind': "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion.2",
     'jacksonDataFormatYaml': "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion",
@@ -196,7 +200,7 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"

datahub-graphql-core/build.gradle

@@ -6,6 +6,7 @@ apply plugin: 'java'
 dependencies {
     compile project(':metadata-service:restli-client')
     compile project(':metadata-service:auth-impl')
+    compile project(':metadata-service:auth-api')
     compile project(':metadata-io')
     compile project(':metadata-utils')
 
@@ -34,6 +35,7 @@ graphqlCodegen {
         "$projectDir/src/main/resources/auth.graphql".toString(),
         "$projectDir/src/main/resources/timeline.graphql".toString(),
         "$projectDir/src/main/resources/tests.graphql".toString(),
+        "$projectDir/src/main/resources/step.graphql".toString(),
     ]
     outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
     packageName = "com.linkedin.datahub.graphql.generated"

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -18,6 +18,7 @@ public class Constants {
     public static final String INGESTION_SCHEMA_FILE = "ingestion.graphql";
     public static final String TIMELINE_SCHEMA_FILE = "timeline.graphql";
     public static final String TESTS_SCHEMA_FILE = "tests.graphql";
+    public static final String STEPS_SCHEMA_FILE = "step.graphql";
     public static final String BROWSE_PATH_DELIMITER = "/";
     public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -3,11 +3,11 @@
 import com.datahub.authentication.AuthenticationConfiguration;
 import com.datahub.authentication.group.GroupService;
 import com.datahub.authentication.invite.InviteTokenService;
-import com.datahub.authentication.post.PostService;
 import com.datahub.authentication.token.StatefulTokenService;
 import com.datahub.authentication.user.NativeUserService;
 import com.datahub.authorization.AuthorizationConfiguration;
 import com.datahub.authorization.role.RoleService;
+import com.datahub.authentication.post.PostService;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.VersionedUrn;
 import com.linkedin.common.urn.Urn;
@@ -192,6 +192,8 @@
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
+import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
+import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver;
@@ -249,7 +251,7 @@
 import com.linkedin.datahub.graphql.types.tag.TagType;
 import com.linkedin.datahub.graphql.types.test.TestType;
 import com.linkedin.entity.client.EntityClient;
-import com.linkedin.metadata.config.DatahubConfiguration;
+import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
@@ -327,7 +329,7 @@ public class GmsGraphQLEngine {
     private final VisualConfiguration visualConfiguration;
     private final TelemetryConfiguration telemetryConfiguration;
     private final TestsConfiguration testsConfiguration;
-    private final DatahubConfiguration datahubConfiguration;
+    private final DataHubConfiguration datahubConfiguration;
 
     private final DatasetType datasetType;
     private final CorpUserType corpUserType;
@@ -393,7 +395,7 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph
         final AuthorizationConfiguration authorizationConfiguration, final GitVersion gitVersion,
         final TimelineService timelineService, final boolean supportsImpactAnalysis,
         final VisualConfiguration visualConfiguration, final TelemetryConfiguration telemetryConfiguration,
-        final TestsConfiguration testsConfiguration, final DatahubConfiguration datahubConfiguration,
+        final TestsConfiguration testsConfiguration, final DataHubConfiguration datahubConfiguration,
         final SiblingGraphService siblingGraphService, final GroupService groupService, final RoleService roleService,
         final InviteTokenService inviteTokenService, final PostService postService, final FeatureFlags featureFlags) {
 
@@ -557,6 +559,7 @@ public GraphQLEngine.Builder builder() {
             .addSchema(fileBasedSchema(INGESTION_SCHEMA_FILE))
             .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE))
             .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE))
+            .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE))
             .addDataLoaders(loaderSuppliers(loadableTypes))
             .addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
             .configureRuntimeWiring(this::configureRuntimeWiring);
@@ -691,6 +694,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("listRoles", new ListRolesResolver(this.entityClient))
             .dataFetcher("getInviteToken", new GetInviteTokenResolver(this.inviteTokenService))
             .dataFetcher("listPosts", new ListPostsResolver(this.entityClient))
+            .dataFetcher("batchGetStepStates", new BatchGetStepStatesResolver(this.entityClient))
         );
     }
 
@@ -814,6 +818,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("createInviteToken", new CreateInviteTokenResolver(this.inviteTokenService))
             .dataFetcher("acceptRole", new AcceptRoleResolver(this.roleService, this.inviteTokenService))
             .dataFetcher("createPost", new CreatePostResolver(this.postService))
+            .dataFetcher("batchUpdateStepStates", new BatchUpdateStepStatesResolver(this.entityClient))
         );
     }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.Authentication;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 
 
 /**
@@ -16,7 +16,7 @@ public interface QueryContext {
     boolean isAuthenticated();
 
     /**
-     * Returns the {@link com.datahub.authentication.Authentication} associated with the current query context.
+     * Returns the {@link Authentication} associated with the current query context.
      */
     Authentication getAuthentication();
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.datahub.authorization.ResourceSpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthUtils.java

@@ -7,7 +7,7 @@
 import java.util.Optional;
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 
 public class AuthUtils {
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -16,7 +16,7 @@
 import com.linkedin.datahub.graphql.generated.TelemetryConfig;
 import com.linkedin.datahub.graphql.generated.TestsConfig;
 import com.linkedin.datahub.graphql.generated.VisualConfig;
-import com.linkedin.metadata.config.DatahubConfiguration;
+import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.telemetry.TelemetryConfiguration;
@@ -42,7 +42,7 @@ public class AppConfigResolver implements DataFetcher<CompletableFuture<AppConfi
   private final VisualConfiguration _visualConfiguration;
   private final TelemetryConfiguration _telemetryConfiguration;
   private final TestsConfiguration _testsConfiguration;
-  private final DatahubConfiguration _datahubConfiguration;
+  private final DataHubConfiguration _datahubConfiguration;
 
   public AppConfigResolver(
       final GitVersion gitVersion,
@@ -54,7 +54,7 @@ public AppConfigResolver(
       final VisualConfiguration visualConfiguration,
       final TelemetryConfiguration telemetryConfiguration,
       final TestsConfiguration testsConfiguration,
-      final DatahubConfiguration datahubConfiguration) {
+      final DataHubConfiguration datahubConfiguration) {
     _gitVersion = gitVersion;
     _isAnalyticsEnabled = isAnalyticsEnabled;
     _ingestionConfiguration = ingestionConfiguration;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java

@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.ingest;
 
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.metadata.authorization.PoliciesConfig;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/PolicyAuthUtils.java

@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.policy;
 
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.metadata.authorization.PoliciesConfig;