diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java index 7e068d5d414e0..ca6b30b36f4cd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java @@ -1,26 +1,33 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; import com.linkedin.datahub.graphql.generated.Schema; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.SchemaMetadata; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.stream.Collectors; -public class SchemaMapper implements ModelMapper { +public class SchemaMapper { public static final SchemaMapper INSTANCE = new SchemaMapper(); public static Schema map(@Nonnull final SchemaMetadata metadata) { - return INSTANCE.apply(metadata); + return INSTANCE.apply(metadata, null); } - @Override - public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { + public static Schema map(@Nonnull final SchemaMetadata metadata, @Nullable final SystemMetadata systemMetadata) { + return INSTANCE.apply(metadata, systemMetadata); + } + + public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input, @Nullable final SystemMetadata systemMetadata) { final Schema result = new Schema(); if (input.getDataset() != null) { result.setDatasetUrn(input.getDataset().toString()); } + if (systemMetadata != null) { + result.setLastObserved(systemMetadata.getLastObserved()); + } result.setName(input.getSchemaName()); result.setPlatformUrn(input.getPlatform().toString()); result.setVersion(input.getVersion()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java index 5f9d4944e5ae2..1b0fb5c73326b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java @@ -31,6 +31,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.metadata.key.DatasetKey; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.EditableSchemaMetadata; import com.linkedin.schema.SchemaMetadata; import javax.annotation.Nonnull; @@ -61,12 +62,14 @@ public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) { EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + SystemMetadata schemaSystemMetadata = getSystemMetadata(aspectMap, SCHEMA_METADATA_ASPECT_NAME); + mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey); mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, this::mapDatasetProperties); mappingHelper.mapToResult(DATASET_DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> dataset.setDeprecation(DatasetDeprecationMapper.map(new DatasetDeprecation(dataMap)))); mappingHelper.mapToResult(SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> - dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap)))); + dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap), schemaSystemMetadata))); mappingHelper.mapToResult(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, this::mapEditableDatasetProperties); mappingHelper.mapToResult(VIEW_PROPERTIES_ASPECT_NAME, this::mapViewProperties); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) -> @@ -88,6 +91,13 @@ public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) { return mappingHelper.getResult(); } + private SystemMetadata getSystemMetadata(EnvelopedAspectMap aspectMap, String aspectName) { + if (aspectMap.containsKey(aspectName) && aspectMap.get(aspectName).hasSystemMetadata()) { + return aspectMap.get(aspectName).getSystemMetadata(); + } + return null; + } + private void mapDatasetKey(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { final DatasetKey gmsKey = new DatasetKey(dataMap); dataset.setName(gmsKey.getName()); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index f8941f681a8af..b0c7a9bd7b128 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -2132,6 +2132,11 @@ type Schema { The time at which the schema metadata information was created """ createdAt: Long + + """ + The time at which the schema metadata information was last ingested + """ + lastObserved: Long } """ diff --git a/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx b/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx new file mode 100644 index 0000000000000..c8bb5d8100f2a --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx @@ -0,0 +1,23 @@ +import { render } from '@testing-library/react'; +import React from 'react'; +import { toRelativeTimeString } from '../../../../../shared/time/timeUtils'; +import SchemaTimeStamps from '../../schema/components/SchemaTimeStamps'; + +describe('SchemaTimeStamps', () => { + it('should render last observed text if lastObserved is not null', () => { + const { getByText, queryByText } = render(); + expect(getByText(`Last observed ${toRelativeTimeString(123)}`)).toBeInTheDocument(); + expect(queryByText(`Reported ${toRelativeTimeString(123)}`)).toBeNull(); + }); + + it('should render last updated text if lastObserved is null', () => { + const { getByText, queryByText } = render(); + expect(queryByText(`Last observed ${toRelativeTimeString(123)}`)).toBeNull(); + expect(getByText(`Reported ${toRelativeTimeString(123)}`)).toBeInTheDocument(); + }); + + it('should return null if lastUpdated and lastObserved are both null', () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); +}); diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 2644dcffa77f0..fb992d0a0c341 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -10,6 +10,7 @@ import { toRelativeTimeString } from '../../../../../shared/time/timeUtils'; import { SchemaViewType } from '../utils/types'; import { ANTD_GRAY } from '../../../../shared/constants'; import { navigateToVersionedDatasetUrl } from '../../../../shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl'; +import SchemaTimeStamps from './SchemaTimeStamps'; const SchemaHeaderContainer = styled.div` display: flex; @@ -99,16 +100,6 @@ const BlameRadioButton = styled(Radio.Button)` } `; -const CurrentVersionTimestampText = styled(Typography.Text)` - &&& { - line-height: 22px; - margin-top: 10px; - margin-right: 10px; - color: ${ANTD_GRAY[7]}; - min-width: 220px; - } -`; - const StyledInfoCircleOutlined = styled(InfoCircleOutlined)` &&& { margin-top: 12px; @@ -134,7 +125,8 @@ type Props = { hasKeySchema: boolean; showKeySchema: boolean; setShowKeySchema: (show: boolean) => void; - lastUpdatedTimeString: string; + lastUpdated?: number | null; + lastObserved?: number | null; selectedVersion: string; versionList: Array; schemaView: SchemaViewType; @@ -152,7 +144,8 @@ export default function SchemaHeader({ hasKeySchema, showKeySchema, setShowKeySchema, - lastUpdatedTimeString, + lastUpdated, + lastObserved, selectedVersion, versionList, schemaView, @@ -233,7 +226,7 @@ export default function SchemaHeader({ ))} - {lastUpdatedTimeString} + Normal diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx new file mode 100644 index 0000000000000..42eae019b947a --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx @@ -0,0 +1,64 @@ +import { ClockCircleOutlined } from '@ant-design/icons'; +import { Popover, Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components/macro'; +import { toLocalDateTimeString, toRelativeTimeString } from '../../../../../shared/time/timeUtils'; +import { ANTD_GRAY } from '../../../../shared/constants'; + +const CurrentVersionTimestampText = styled(Typography.Text)` + &&& { + line-height: 22px; + margin-top: 10px; + margin-right: 10px; + color: ${ANTD_GRAY[7]}; + width: max-content; + } +`; + +const TimeStampWrapper = styled.div` + margin-bottom: 5px; +`; + +const StyledClockIcon = styled(ClockCircleOutlined)` + margin-right: 5px; +`; + +interface Props { + lastUpdated?: number | null; + lastObserved?: number | null; +} + +function SchemaTimeStamps(props: Props) { + const { lastUpdated, lastObserved } = props; + + if (!lastUpdated && !lastObserved) return null; + + return ( + + {lastObserved && ( + Last observed on {toLocalDateTimeString(lastObserved)}. + )} + {lastUpdated &&
First reported on {toLocalDateTimeString(lastUpdated)}.
} + + } + > + + {lastObserved && ( + + Last observed {toRelativeTimeString(lastObserved)} + + )} + {!lastObserved && lastUpdated && ( + + + Reported {toRelativeTimeString(lastUpdated)} + + )} + + + ); +} + +export default SchemaTimeStamps; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx index 7a69002b87b31..a3d9b48a73c17 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx @@ -14,7 +14,6 @@ import { groupByFieldPath } from '../../../../dataset/profile/schema/utils/utils import { ANTD_GRAY } from '../../../constants'; import { useBaseEntity, useEntityData } from '../../../EntityContext'; import { ChangeCategoryType, SchemaFieldBlame, SemanticVersionStruct } from '../../../../../../types.generated'; -import { toLocalDateTimeString } from '../../../../../shared/time/timeUtils'; import { SchemaViewType } from '../../../../dataset/profile/schema/utils/types'; import SchemaTable from './SchemaTable'; import useGetSemanticVersionFromUrlParams from './utils/useGetSemanticVersionFromUrlParams'; @@ -120,11 +119,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { return groupByFieldPath(schemaMetadata?.fields, { showKeySchema }); }, [schemaMetadata, showKeySchema]); - const lastUpdatedTimeString = `Reported at ${ - (getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp && - toLocalDateTimeString(getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp)) || - 'unknown' - }`; + const lastUpdated = getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp; + const lastObserved = versionedDatasetData.data?.versionedDataset?.schema?.lastObserved; const schemaFieldBlameList: Array = (getSchemaBlameData?.getSchemaBlame?.schemaFieldBlameList as Array) || []; @@ -139,7 +135,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { hasKeySchema={hasKeySchema} showKeySchema={showKeySchema} setShowKeySchema={setShowKeySchema} - lastUpdatedTimeString={lastUpdatedTimeString} + lastObserved={lastObserved} + lastUpdated={lastUpdated} selectedVersion={selectedVersion} versionList={versionList} schemaView={schemaViewMode} diff --git a/datahub-web-react/src/graphql/versionedDataset.graphql b/datahub-web-react/src/graphql/versionedDataset.graphql index b9f3002dc0999..d61139927e14b 100644 --- a/datahub-web-react/src/graphql/versionedDataset.graphql +++ b/datahub-web-react/src/graphql/versionedDataset.graphql @@ -11,6 +11,7 @@ query getVersionedDataset($urn: String!, $versionStamp: String) { recursive isPartOfKey } + lastObserved } editableSchemaMetadata { editableSchemaFieldInfo { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 737fe7eaf6ae9..0c20e3fea5194 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -1522,6 +1522,16 @@ private Map getEnvelopedAspects(final S // since nowhere else is using it should be safe for now at least envelopedAspect.setType(AspectType.VERSIONED); envelopedAspect.setValue(aspect); + + try { + if (currAspectEntry.getSystemMetadata() != null) { + final SystemMetadata systemMetadata = RecordUtils.toRecordTemplate(SystemMetadata.class, currAspectEntry.getSystemMetadata()); + envelopedAspect.setSystemMetadata(systemMetadata); + } + } catch (Exception e) { + log.warn("Exception encountered when setting system metadata on enveloped aspect {}. Error: {}", envelopedAspect.getName(), e); + } + envelopedAspect.setCreated(new AuditStamp() .setActor(UrnUtils.getUrn(currAspectEntry.getCreatedBy())) .setTime(currAspectEntry.getCreatedOn().getTime()) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 6eef5dfe9efb0..47fd051ed0454 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -307,7 +307,7 @@ private String batchGetSelect( outputParamsToValues.put(aspectArg, aspect); outputParamsToValues.put(versionArg, version); - return String.format("SELECT urn, aspect, version, metadata, createdOn, createdBy, createdFor " + return String.format("SELECT urn, aspect, version, metadata, systemMetadata, createdOn, createdBy, createdFor " + "FROM %s WHERE urn = :%s AND aspect = :%s AND version = :%s", EbeanAspectV2.class.getAnnotation(Table.class).name(), urnArg, aspectArg, versionArg); } diff --git a/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl index 28e73c38b0172..3942bf4262e51 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl @@ -37,4 +37,9 @@ record EnvelopedAspect { * The audit stamp detailing who the aspect was created by and when **/ created: AuditStamp + + /** + * The system metadata for this aspect + **/ + systemMetadata: optional SystemMetadata } diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl index 5f3c90e7f4a59..bf63933ff1f67 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl @@ -3,6 +3,7 @@ namespace com.linkedin.schema import com.linkedin.common.ChangeAuditStamps import com.linkedin.common.DatasetUrn import com.linkedin.dataset.SchemaFieldPath +import com.linkedin.mxe.SystemMetadata /** * SchemaMetadata to describe metadata related to store schema diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json index c428918748e25..73c355a23f747 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json @@ -93,12 +93,53 @@ "name" : "created", "type" : "com.linkedin.common.AuditStamp", "doc" : "The audit stamp detailing who the aspect was created by and when\n" + }, { + "name" : "systemMetadata", + "type" : { + "type" : "record", + "name" : "SystemMetadata", + "namespace" : "com.linkedin.mxe", + "doc" : "Metadata associated with each metadata change that is processed by the system", + "fields" : [ { + "name" : "lastObserved", + "type" : "long", + "doc" : "The timestamp the metadata was observed at", + "default" : 0, + "optional" : true + }, { + "name" : "runId", + "type" : "string", + "doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.", + "default" : "no-run-id-provided", + "optional" : true + }, { + "name" : "registryName", + "type" : "string", + "doc" : "The model registry name that was used to process this event", + "optional" : true + }, { + "name" : "registryVersion", + "type" : "string", + "doc" : "The model registry version that was used to process this event", + "optional" : true + }, { + "name" : "properties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Additional properties", + "optional" : true + } ] + }, + "doc" : "The system metadata for this aspect\n", + "optional" : true } ] } }, "doc" : "A map of aspect name to aspect\n" } ] - }, "com.linkedin.entity.EnvelopedAspect" ], + }, "com.linkedin.entity.EnvelopedAspect", "com.linkedin.mxe.SystemMetadata" ], "schema" : { "name" : "entitiesV2", "namespace" : "com.linkedin.entity", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json index 6e187fba11856..d9f36ddba4026 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json @@ -102,12 +102,53 @@ "name" : "created", "type" : "com.linkedin.common.AuditStamp", "doc" : "The audit stamp detailing who the aspect was created by and when\n" + }, { + "name" : "systemMetadata", + "type" : { + "type" : "record", + "name" : "SystemMetadata", + "namespace" : "com.linkedin.mxe", + "doc" : "Metadata associated with each metadata change that is processed by the system", + "fields" : [ { + "name" : "lastObserved", + "type" : "long", + "doc" : "The timestamp the metadata was observed at", + "default" : 0, + "optional" : true + }, { + "name" : "runId", + "type" : "string", + "doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.", + "default" : "no-run-id-provided", + "optional" : true + }, { + "name" : "registryName", + "type" : "string", + "doc" : "The model registry name that was used to process this event", + "optional" : true + }, { + "name" : "registryVersion", + "type" : "string", + "doc" : "The model registry version that was used to process this event", + "optional" : true + }, { + "name" : "properties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Additional properties", + "optional" : true + } ] + }, + "doc" : "The system metadata for this aspect\n", + "optional" : true } ] } }, "doc" : "A map of aspect name to aspect\n" } ] - }, "com.linkedin.entity.EnvelopedAspect" ], + }, "com.linkedin.entity.EnvelopedAspect", "com.linkedin.mxe.SystemMetadata" ], "schema" : { "name" : "entitiesVersionedV2", "namespace" : "com.linkedin.entity",