Skip to content

Commit

Permalink
Merge branch 'master' into feat-clear-system-metadata-service-step
Browse files Browse the repository at this point in the history
  • Loading branch information
jjoyce0510 authored Jul 1, 2024
2 parents 44931e0 + 3056cd8 commit b8d314f
Show file tree
Hide file tree
Showing 437 changed files with 18,333 additions and 16,079 deletions.
17 changes: 8 additions & 9 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -32,29 +32,29 @@ buildscript {

ext.junitJupiterVersion = '5.6.1'
// Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
ext.pegasusVersion = '29.51.6'
ext.pegasusVersion = '29.57.0'
ext.mavenVersion = '3.6.3'
ext.springVersion = '6.1.4'
ext.springBootVersion = '3.2.3'
ext.springKafkaVersion = '3.1.2'
ext.springVersion = '6.1.5'
ext.springBootVersion = '3.2.6'
ext.springKafkaVersion = '3.1.6'
ext.openTelemetryVersion = '1.18.0'
ext.neo4jVersion = '5.14.0'
ext.neo4jTestVersion = '5.14.0'
ext.neo4jApocVersion = '5.14.0'
ext.testContainersVersion = '1.17.4'
ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
ext.jacksonVersion = '2.15.3'
ext.jettyVersion = '11.0.19'
ext.playVersion = '2.8.21'
ext.log4jVersion = '2.19.0'
ext.jettyVersion = '11.0.21'
ext.playVersion = '2.8.22'
ext.log4jVersion = '2.23.1'
ext.slf4jVersion = '1.7.36'
ext.logbackClassic = '1.4.14'
ext.hadoop3Version = '3.3.5'
ext.kafkaVersion = '5.5.15'
ext.hazelcastVersion = '5.3.6'
ext.ebeanVersion = '12.16.1'
ext.googleJavaFormatVersion = '1.18.1'
ext.openLineageVersion = '1.14.0'
ext.openLineageVersion = '1.16.0'
ext.logbackClassicJava8 = '1.2.12'

ext.docker_registry = 'acryldata'
Expand Down Expand Up @@ -149,7 +149,6 @@ project.ext.externalDependency = [
'hazelcastTest':"com.hazelcast:hazelcast:$hazelcastVersion:tests",
'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
'httpClient': 'org.apache.httpcomponents.client5:httpclient5:5.3',
'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion",
'jacksonDataPropertyFormat': "com.fasterxml.jackson.dataformat:jackson-dataformat-properties:$jacksonVersion",
Expand Down
2 changes: 1 addition & 1 deletion datahub-frontend/app/controllers/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
.setBody(
new InMemoryBodyWritable(
ByteString.fromByteBuffer(request.body().asBytes().asByteBuffer()),
"application/json"))
request.contentType().orElse("application/json")))
.setRequestTimeout(Duration.ofSeconds(120))
.execute()
.thenApply(
Expand Down
6 changes: 5 additions & 1 deletion datahub-frontend/conf/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,12 @@ jwt {
play.server.provider = server.CustomAkkaHttpServerProvider
play.http.server.akka.max-header-count = 64
play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT}
play.server.akka.max-header-size = 8k
# max-header-size is reportedly no longer used
play.server.akka.max-header-size = 32k
play.server.akka.max-header-size = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH}
# max header value length seems to impact the actual limit
play.server.akka.max-header-value-length = 32k
play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH}

# Update AUTH_COOKIE_SAME_SITE and AUTH_COOKIE_SECURE in order to change how authentication cookies
# are configured. If you wish cookies to be sent in first and third party contexts, set
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package com.linkedin.datahub.graphql;

import com.google.common.collect.ImmutableSet;
import java.util.Set;

/** Constants relating to GraphQL type system & execution. */
public class Constants {

Expand Down Expand Up @@ -28,4 +31,11 @@ private Constants() {}
public static final String BROWSE_PATH_V2_DELIMITER = "␟";
public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
public static final String ENTITY_FILTER_NAME = "_entityType";

public static final Set<String> DEFAULT_PERSONA_URNS =
ImmutableSet.of(
"urn:li:dataHubPersona:technicalUser",
"urn:li:dataHubPersona:businessUser",
"urn:li:dataHubPersona:dataLeader",
"urn:li:dataHubPersona:dataSteward");
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import com.linkedin.datahub.graphql.generated.CorpGroup;
import com.linkedin.datahub.graphql.generated.CorpGroupInfo;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.CorpUserEditableProperties;
import com.linkedin.datahub.graphql.generated.CorpUserInfo;
import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings;
import com.linkedin.datahub.graphql.generated.Dashboard;
Expand All @@ -53,6 +54,7 @@
import com.linkedin.datahub.graphql.generated.DataHubView;
import com.linkedin.datahub.graphql.generated.DataJob;
import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
import com.linkedin.datahub.graphql.generated.DataPlatform;
import com.linkedin.datahub.graphql.generated.DataPlatformInstance;
import com.linkedin.datahub.graphql.generated.DataQualityContract;
import com.linkedin.datahub.graphql.generated.Dataset;
Expand Down Expand Up @@ -121,6 +123,8 @@
import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver;
import com.linkedin.datahub.graphql.resolvers.assertion.DeleteAssertionResolver;
import com.linkedin.datahub.graphql.resolvers.assertion.EntityAssertionsResolver;
import com.linkedin.datahub.graphql.resolvers.assertion.ReportAssertionResultResolver;
import com.linkedin.datahub.graphql.resolvers.assertion.UpsertCustomAssertionResolver;
import com.linkedin.datahub.graphql.resolvers.auth.CreateAccessTokenResolver;
import com.linkedin.datahub.graphql.resolvers.auth.DebugAccessResolver;
import com.linkedin.datahub.graphql.resolvers.auth.GetAccessTokenMetadataResolver;
Expand Down Expand Up @@ -377,6 +381,7 @@
import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.query.filter.SortOrder;
import com.linkedin.metadata.recommendation.RecommendationsService;
import com.linkedin.metadata.service.AssertionService;
import com.linkedin.metadata.service.BusinessAttributeService;
import com.linkedin.metadata.service.DataProductService;
import com.linkedin.metadata.service.ERModelRelationshipService;
Expand Down Expand Up @@ -454,6 +459,7 @@ public class GmsGraphQLEngine {
private final FormService formService;
private final RestrictedService restrictedService;
private ConnectionService connectionService;
private AssertionService assertionService;

private final BusinessAttributeService businessAttributeService;
private final FeatureFlags featureFlags;
Expand Down Expand Up @@ -575,6 +581,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
this.formService = args.formService;
this.restrictedService = args.restrictedService;
this.connectionService = args.connectionService;
this.assertionService = args.assertionService;

this.businessAttributeService = args.businessAttributeService;
this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration);
Expand Down Expand Up @@ -1220,6 +1227,10 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
"createTestConnectionRequest",
new CreateTestConnectionRequestResolver(
this.entityClient, this.ingestionConfiguration))
.dataFetcher(
"upsertCustomAssertion", new UpsertCustomAssertionResolver(assertionService))
.dataFetcher(
"reportAssertionResult", new ReportAssertionResultResolver(assertionService))
.dataFetcher(
"deleteAssertion",
new DeleteAssertionResolver(this.entityClient, this.entityService))
Expand Down Expand Up @@ -1814,6 +1825,18 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) {
new LoadableTypeResolver<>(
corpUserType,
(env) -> ((CorpUserInfo) env.getSource()).getManager().getUrn())));
builder.type(
"CorpUserEditableProperties",
typeWiring ->
typeWiring.dataFetcher(
"platforms",
new LoadableTypeBatchResolver<>(
dataPlatformType,
(env) ->
((CorpUserEditableProperties) env.getSource())
.getPlatforms().stream()
.map(DataPlatform::getUrn)
.collect(Collectors.toList()))));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.linkedin.metadata.graph.SiblingGraphService;
import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.metadata.recommendation.RecommendationsService;
import com.linkedin.metadata.service.AssertionService;
import com.linkedin.metadata.service.BusinessAttributeService;
import com.linkedin.metadata.service.DataProductService;
import com.linkedin.metadata.service.ERModelRelationshipService;
Expand Down Expand Up @@ -86,6 +87,7 @@ public class GmsGraphQLEngineArgs {
boolean graphQLQueryIntrospectionEnabled;
BusinessAttributeService businessAttributeService;
ConnectionService connectionService;
AssertionService assertionService;

// any fork specific args should go below this line
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ public class FeatureFlags {
private boolean nestedDomainsEnabled = false;
private boolean schemaFieldEntityFetchEnabled = false;
private boolean businessAttributeEntityEnabled = false;
private boolean dataContractsEnabled = false;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package com.linkedin.datahub.graphql.resolvers.assertion;

import com.datahub.authorization.ConjunctivePrivilegeGroup;
import com.datahub.authorization.DisjunctivePrivilegeGroup;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
import com.linkedin.metadata.authorization.PoliciesConfig;

public class AssertionUtils {
public static boolean isAuthorizedToEditAssertionFromAssertee(
final QueryContext context, final Urn asserteeUrn) {
final DisjunctivePrivilegeGroup orPrivilegeGroups =
new DisjunctivePrivilegeGroup(
ImmutableList.of(
AuthorizationUtils.ALL_PRIVILEGES_GROUP,
new ConjunctivePrivilegeGroup(
ImmutableList.of(PoliciesConfig.EDIT_ENTITY_ASSERTIONS_PRIVILEGE.getType()))));
return AuthorizationUtils.isAuthorized(
context.getAuthorizer(),
context.getActorUrn(),
asserteeUrn.getEntityType(),
asserteeUrn.toString(),
orPrivilegeGroups);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package com.linkedin.datahub.graphql.resolvers.assertion;

import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;

import com.linkedin.assertion.AssertionResult;
import com.linkedin.assertion.AssertionResultError;
import com.linkedin.assertion.AssertionResultErrorType;
import com.linkedin.assertion.AssertionResultType;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.template.SetMode;
import com.linkedin.data.template.StringMap;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.exception.AuthorizationException;
import com.linkedin.datahub.graphql.generated.AssertionResultInput;
import com.linkedin.datahub.graphql.generated.StringMapEntryInput;
import com.linkedin.metadata.service.AssertionService;
import graphql.execution.DataFetcherExceptionHandler;
import graphql.execution.DataFetcherResult;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class ReportAssertionResultResolver implements DataFetcher<CompletableFuture<Boolean>> {

public static final String ERROR_MESSAGE_KEY = "message";
private final AssertionService _assertionService;

public ReportAssertionResultResolver(AssertionService assertionService) {
_assertionService = assertionService;
}

/**
* This is called by the graphql engine to fetch the value. The {@link DataFetchingEnvironment} is
* a composite context object that tells you all you need to know about how to fetch a data value
* in graphql type terms.
*
* @param environment this is the data fetching environment which contains all the context you
* need to fetch a value
* @return a value of type T. May be wrapped in a {@link DataFetcherResult}
* @throws Exception to relieve the implementations from having to wrap checked exceptions. Any
* exception thrown from a {@code DataFetcher} will eventually be handled by the registered
* {@link DataFetcherExceptionHandler} and the related field will have a value of {@code null}
* in the result.
*/
@Override
public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
final Urn assertionUrn = UrnUtils.getUrn(environment.getArgument("urn"));
final AssertionResultInput input =
bindArgument(environment.getArgument("result"), AssertionResultInput.class);

return CompletableFuture.supplyAsync(
() -> {
final Urn asserteeUrn =
_assertionService.getEntityUrnForAssertion(
context.getOperationContext(), assertionUrn);
if (asserteeUrn == null) {
throw new RuntimeException(
String.format(
"Failed to report Assertion Run Event. Assertion with urn %s does not exist or is not associated with any entity.",
assertionUrn));
}

// Check whether the current user is allowed to update the assertion.
if (AssertionUtils.isAuthorizedToEditAssertionFromAssertee(context, asserteeUrn)) {
AssertionResult assertionResult = mapAssertionResult(input);
_assertionService.addAssertionRunEvent(
context.getOperationContext(),
assertionUrn,
asserteeUrn,
input.getTimestampMillis() != null
? input.getTimestampMillis()
: System.currentTimeMillis(),
assertionResult);
return true;
}
throw new AuthorizationException(
"Unauthorized to perform this action. Please contact your DataHub administrator.");
});
}

private static StringMap mapContextParameters(List<StringMapEntryInput> input) {

if (input == null || input.isEmpty()) {
return null;
}
StringMap entries = new StringMap();
input.forEach(entry -> entries.put(entry.getKey(), entry.getValue()));
return entries;
}

private AssertionResult mapAssertionResult(AssertionResultInput input) {
AssertionResult assertionResult = new AssertionResult();
assertionResult.setType(AssertionResultType.valueOf(input.getType().toString()));
assertionResult.setExternalUrl(input.getExternalUrl(), SetMode.IGNORE_NULL);
if (assertionResult.getType() == AssertionResultType.ERROR && input.getError() != null) {
assertionResult.setError(mapAssertionResultError(input));
}
if (input.getProperties() != null) {
assertionResult.setNativeResults(mapContextParameters(input.getProperties()));
}
return assertionResult;
}

private static AssertionResultError mapAssertionResultError(AssertionResultInput input) {
AssertionResultError error = new AssertionResultError();
error.setType(AssertionResultErrorType.valueOf(input.getError().getType().toString()));
StringMap errorProperties = new StringMap();
errorProperties.put(ERROR_MESSAGE_KEY, input.getError().getMessage());
error.setProperties(errorProperties);
return error;
}
}
Loading

0 comments on commit b8d314f

Please sign in to comment.