Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(propagation): Add graphql API #11030

Merged
merged 2 commits into from
Jul 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private Constants() {}
public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
public static final String FORMS_SCHEMA_FILE = "forms.graphql";
public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql";
public static final String COMMON_SCHEMA_FILE = "common.graphql";
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

one thing that's missing is adding the new common.graphql file on the backend to our docs website auto-generation list. So we can add the following line to docs-website/graphql/generateGraphQLSchema.sh:

cat ../../datahub-graphql-core/src/main/resources/common.graphql >> combined.graphql

public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql";
public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
import com.linkedin.datahub.graphql.generated.MLPrimaryKey;
import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
import com.linkedin.datahub.graphql.generated.MatchedField;
import com.linkedin.datahub.graphql.generated.MetadataAttribution;
import com.linkedin.datahub.graphql.generated.Notebook;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
Expand Down Expand Up @@ -695,7 +696,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
businessAttributeType));
this.loadableTypes = new ArrayList<>(entityTypes);
// Extend loadable types with types from the plugins
// This allows us to offer search and browse capabilities out of the box for those types
// This allows us to offer search and browse capabilities out of the box for
// those types
for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
this.entityTypes.addAll(plugin.getEntityTypes());
Collection<? extends LoadableType<?, ?>> pluginLoadableTypes = plugin.getLoadableTypes();
Expand Down Expand Up @@ -790,6 +792,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
configureBusinessAttributeAssociationResolver(builder);
configureConnectionResolvers(builder);
configureDeprecationResolvers(builder);
configureMetadataAttributionResolver(builder);
}

private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
Expand Down Expand Up @@ -843,7 +846,8 @@ public GraphQLEngine.Builder builder() {
.addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(ASSERTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE));
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(COMMON_SCHEMA_FILE));

for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
List<String> pluginSchemaFiles = plugin.getSchemaFiles();
Expand Down Expand Up @@ -2823,7 +2827,8 @@ private void configureContractResolvers(final RuntimeWiring.Builder builder) {
}

private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type.
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the
// Policy type.
builder.type(
"ActorFilter",
typeWiring ->
Expand Down Expand Up @@ -3176,4 +3181,20 @@ private void configureDeprecationResolvers(final RuntimeWiring.Builder builder)
new EntityTypeResolver(
entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity())));
}

private void configureMetadataAttributionResolver(final RuntimeWiring.Builder builder) {
builder.type(
"MetadataAttribution",
typeWiring ->
typeWiring
.dataFetcher(
"actor",
new EntityTypeResolver(
entityTypes, (env) -> ((MetadataAttribution) env.getSource()).getActor()))
.dataFetcher(
"source",
new EntityTypeResolver(
entityTypes,
(env) -> ((MetadataAttribution) env.getSource()).getSource())));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.DataHubConnection;
import com.linkedin.datahub.graphql.generated.Documentation;
import com.linkedin.datahub.graphql.generated.DocumentationAssociation;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class DocumentationMapper
implements ModelMapper<com.linkedin.common.Documentation, Documentation> {

public static final DocumentationMapper INSTANCE = new DocumentationMapper();

public static Documentation map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.Documentation metadata) {
return INSTANCE.apply(context, metadata);
}

@Override
public Documentation apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.Documentation input) {
final Documentation result = new Documentation();
result.setDocumentations(
input.getDocumentations().stream()
.map(docAssociation -> mapDocAssociation(context, docAssociation))
.collect(Collectors.toList()));
return result;
}

private DocumentationAssociation mapDocAssociation(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DocumentationAssociation association) {
final DocumentationAssociation result = new DocumentationAssociation();
result.setDocumentation(association.getDocumentation());
if (association.getAttribution() != null) {
result.setAttribution(MetadataAttributionMapper.map(context, association.getAttribution()));
}
return result;
}

private DataHubConnection mapConnectionEntity(@Nonnull final Urn urn) {
DataHubConnection connection = new DataHubConnection();
connection.setUrn(urn.toString());
connection.setType(EntityType.DATAHUB_CONNECTION);
return connection;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.MetadataAttribution;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class MetadataAttributionMapper
implements ModelMapper<com.linkedin.common.MetadataAttribution, MetadataAttribution> {

public static final MetadataAttributionMapper INSTANCE = new MetadataAttributionMapper();

public static MetadataAttribution map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.MetadataAttribution metadata) {
return INSTANCE.apply(context, metadata);
}

@Override
public MetadataAttribution apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.MetadataAttribution input) {
final MetadataAttribution result = new MetadataAttribution();
result.setTime(input.getTime());
result.setActor(UrnToEntityMapper.map(context, input.getActor()));
if (input.getSource() != null) {
result.setSource(UrnToEntityMapper.map(context, input.getSource()));
}
if (input.getSourceDetail() != null) {
result.setSourceDetail(StringMapMapper.map(context, input.getSourceDetail()));
}
return result;
}
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package com.linkedin.datahub.graphql.types.schemafield;

import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT;
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
import static com.linkedin.metadata.Constants.*;

import com.linkedin.businessattribute.BusinessAttributes;
import com.linkedin.common.Documentation;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.SchemaFieldEntity;
import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DocumentationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
Expand Down Expand Up @@ -46,6 +47,10 @@ public SchemaFieldEntity apply(
(((schemaField, dataMap) ->
schemaField.setBusinessAttributes(
BusinessAttributesMapper.map(new BusinessAttributes(dataMap), entityUrn)))));
mappingHelper.mapToResult(
DOCUMENTATION_ASPECT_NAME,
(entity, dataMap) ->
entity.setDocumentation(DocumentationMapper.map(context, new Documentation(dataMap))));
return result;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package com.linkedin.datahub.graphql.types.schemafield;

import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT;
import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME;
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;
import static com.linkedin.metadata.Constants.*;

import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
Expand Down Expand Up @@ -32,7 +30,8 @@ public class SchemaFieldType
implements com.linkedin.datahub.graphql.types.EntityType<SchemaFieldEntity, String> {

public static final Set<String> ASPECTS_TO_FETCH =
ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME, BUSINESS_ATTRIBUTE_ASPECT);
ImmutableSet.of(
STRUCTURED_PROPERTIES_ASPECT_NAME, BUSINESS_ATTRIBUTE_ASPECT, DOCUMENTATION_ASPECT_NAME);

private final EntityClient _entityClient;
private final FeatureFlags _featureFlags;
Expand Down
49 changes: 49 additions & 0 deletions datahub-graphql-core/src/main/resources/common.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""
Object containing the documentation aspect for an entity
"""
type Documentation {
"""
Structured properties on this entity
"""
documentations: [DocumentationAssociation!]!
}

"""
Object containing the documentation aspect for an entity
"""
type DocumentationAssociation {
"""
Structured properties on this entity
"""
documentation: String!

"""
Information about who, why, and how this metadata was applied
"""
attribution: MetadataAttribution
}

"""
Information about who, why, and how this metadata was applied
"""
type MetadataAttribution {
"""
The time this metadata was applied
"""
time: Long!

"""
The actor responsible for this metadata application
"""
actor: Entity!

"""
The source of this metadata application. If propagated, this will be an action.
"""
source: Entity

"""
Extra details about how this metadata was applied
"""
sourceDetail: [StringMapEntry!]
}
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -3225,6 +3225,11 @@ type SchemaFieldEntity implements Entity {
Business Attribute associated with the field
"""
businessAttributes: BusinessAttributes

"""
Documentation aspect for this schema field
"""
documentation: Documentation
}

"""
Expand Down
4 changes: 2 additions & 2 deletions datahub-web-react/src/app/home/AcrylDemoBanner.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ export default function AcrylDemoBanner() {
>
Schedule a demo
</StyledLink>{' '}
of DataHub Cloud to see the advanced features that take it to the next level or purchase DataHub Cloud
on{' '}
of DataHub Cloud to see the advanced features that take it to the next level or purchase DataHub
Cloud on{' '}
<StyledLink
href="https://aws.amazon.com/marketplace/pp/prodview-ratzv4k453pck?sr=0-1&ref_=beagle&applicationId=AWSMPContessa"
target="_blank"
Expand Down
31 changes: 31 additions & 0 deletions datahub-web-react/src/graphql/fragments.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -736,6 +736,37 @@ fragment schemaFieldFields on SchemaField {
...businessAttribute
}
}
documentation {
documentations {
documentation
attribution {
time
actor {
urn
type
...entityDisplayNameFields
}
source {
urn
type
}
sourceDetail {
key
value
}
}
}
}
parent {
urn
type
...entityDisplayNameFields
... on Dataset {
platform {
...platformFields
}
}
}
}
}

Expand Down
6 changes: 6 additions & 0 deletions datahub-web-react/src/graphql/search.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -974,6 +974,12 @@ fragment entityField on SchemaFieldEntity {
parent {
urn
type
...entityDisplayNameFields
... on Dataset {
platform {
...platformFields
}
}
}
fieldPath
structuredProperties {
Expand Down
10 changes: 7 additions & 3 deletions li-utils/src/main/java/com/linkedin/metadata/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ public class Constants {
".", STRUCTURED_PROPERTY_MAPPING_FIELD, STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD, "");

// !!!!!!! IMPORTANT !!!!!!!
// This effectively sets the max aspect size to 16 MB. Used in deserialization of messages.
// This effectively sets the max aspect size to 16 MB. Used in deserialization
// of messages.
// Without this the limit is
// whatever Jackson is defaulting to (5 MB currently).
public static final String MAX_JACKSON_STRING_SIZE = "16000000";
Expand All @@ -37,10 +38,12 @@ public class Constants {
/** System Metadata */
public static final String DEFAULT_RUN_ID = "no-run-id-provided";

// Forces indexing for no-ops, enabled for restore indices calls. Only considered in the no-op
// Forces indexing for no-ops, enabled for restore indices calls. Only
// considered in the no-op
// case
public static final String FORCE_INDEXING_KEY = "forceIndexing";
// Indicates an event source from an application with hooks that have already been processed and
// Indicates an event source from an application with hooks that have already
// been processed and
// should not be reprocessed
public static final String APP_SOURCE = "appSource";

Expand Down Expand Up @@ -116,6 +119,7 @@ public class Constants {
public static final String INPUT_FIELDS_ASPECT_NAME = "inputFields";
public static final String EMBED_ASPECT_NAME = "embed";
public static final String INCIDENTS_SUMMARY_ASPECT_NAME = "incidentsSummary";
public static final String DOCUMENTATION_ASPECT_NAME = "documentation";

// User
public static final String CORP_USER_KEY_ASPECT_NAME = "corpUserKey";
Expand Down
Loading