Skip to content

Commit

Permalink
feat(schema) Show last observed timestamp in the schema tab (#5348)
Browse files Browse the repository at this point in the history
  • Loading branch information
chriscollins3456 authored Jul 7, 2022
1 parent 7872a74 commit 68fb0d8
Show file tree
Hide file tree
Showing 14 changed files with 227 additions and 29 deletions.
Original file line number Diff line number Diff line change
@@ -1,26 +1,33 @@
package com.linkedin.datahub.graphql.types.dataset.mappers;

import com.linkedin.datahub.graphql.generated.Schema;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.mxe.SystemMetadata;
import com.linkedin.schema.SchemaMetadata;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.stream.Collectors;

public class SchemaMapper implements ModelMapper<SchemaMetadata, Schema> {
public class SchemaMapper {

public static final SchemaMapper INSTANCE = new SchemaMapper();

public static Schema map(@Nonnull final SchemaMetadata metadata) {
return INSTANCE.apply(metadata);
return INSTANCE.apply(metadata, null);
}

@Override
public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) {
public static Schema map(@Nonnull final SchemaMetadata metadata, @Nullable final SystemMetadata systemMetadata) {
return INSTANCE.apply(metadata, systemMetadata);
}

public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input, @Nullable final SystemMetadata systemMetadata) {
final Schema result = new Schema();
if (input.getDataset() != null) {
result.setDatasetUrn(input.getDataset().toString());
}
if (systemMetadata != null) {
result.setLastObserved(systemMetadata.getLastObserved());
}
result.setName(input.getSchemaName());
result.setPlatformUrn(input.getPlatform().toString());
result.setVersion(input.getVersion());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.metadata.key.DatasetKey;
import com.linkedin.mxe.SystemMetadata;
import com.linkedin.schema.EditableSchemaMetadata;
import com.linkedin.schema.SchemaMetadata;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -61,12 +62,14 @@ public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) {

EnvelopedAspectMap aspectMap = entityResponse.getAspects();
MappingHelper<VersionedDataset> mappingHelper = new MappingHelper<>(aspectMap, result);
SystemMetadata schemaSystemMetadata = getSystemMetadata(aspectMap, SCHEMA_METADATA_ASPECT_NAME);

mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey);
mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, this::mapDatasetProperties);
mappingHelper.mapToResult(DATASET_DEPRECATION_ASPECT_NAME, (dataset, dataMap) ->
dataset.setDeprecation(DatasetDeprecationMapper.map(new DatasetDeprecation(dataMap))));
mappingHelper.mapToResult(SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) ->
dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap))));
dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap), schemaSystemMetadata)));
mappingHelper.mapToResult(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, this::mapEditableDatasetProperties);
mappingHelper.mapToResult(VIEW_PROPERTIES_ASPECT_NAME, this::mapViewProperties);
mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) ->
Expand All @@ -88,6 +91,13 @@ public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) {
return mappingHelper.getResult();
}

private SystemMetadata getSystemMetadata(EnvelopedAspectMap aspectMap, String aspectName) {
if (aspectMap.containsKey(aspectName) && aspectMap.get(aspectName).hasSystemMetadata()) {
return aspectMap.get(aspectName).getSystemMetadata();
}
return null;
}

private void mapDatasetKey(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) {
final DatasetKey gmsKey = new DatasetKey(dataMap);
dataset.setName(gmsKey.getName());
Expand Down
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -2132,6 +2132,11 @@ type Schema {
The time at which the schema metadata information was created
"""
createdAt: Long

"""
The time at which the schema metadata information was last ingested
"""
lastObserved: Long
}

"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { render } from '@testing-library/react';
import React from 'react';
import { toRelativeTimeString } from '../../../../../shared/time/timeUtils';
import SchemaTimeStamps from '../../schema/components/SchemaTimeStamps';

describe('SchemaTimeStamps', () => {
it('should render last observed text if lastObserved is not null', () => {
const { getByText, queryByText } = render(<SchemaTimeStamps lastUpdated={123} lastObserved={123} />);
expect(getByText(`Last observed ${toRelativeTimeString(123)}`)).toBeInTheDocument();
expect(queryByText(`Reported ${toRelativeTimeString(123)}`)).toBeNull();
});

it('should render last updated text if lastObserved is null', () => {
const { getByText, queryByText } = render(<SchemaTimeStamps lastUpdated={123} lastObserved={null} />);
expect(queryByText(`Last observed ${toRelativeTimeString(123)}`)).toBeNull();
expect(getByText(`Reported ${toRelativeTimeString(123)}`)).toBeInTheDocument();
});

it('should return null if lastUpdated and lastObserved are both null', () => {
const { container } = render(<SchemaTimeStamps lastUpdated={null} lastObserved={null} />);
expect(container.firstChild).toBeNull();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { toRelativeTimeString } from '../../../../../shared/time/timeUtils';
import { SchemaViewType } from '../utils/types';
import { ANTD_GRAY } from '../../../../shared/constants';
import { navigateToVersionedDatasetUrl } from '../../../../shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl';
import SchemaTimeStamps from './SchemaTimeStamps';

const SchemaHeaderContainer = styled.div`
display: flex;
Expand Down Expand Up @@ -99,16 +100,6 @@ const BlameRadioButton = styled(Radio.Button)`
}
`;

const CurrentVersionTimestampText = styled(Typography.Text)`
&&& {
line-height: 22px;
margin-top: 10px;
margin-right: 10px;
color: ${ANTD_GRAY[7]};
min-width: 220px;
}
`;

const StyledInfoCircleOutlined = styled(InfoCircleOutlined)`
&&& {
margin-top: 12px;
Expand All @@ -134,7 +125,8 @@ type Props = {
hasKeySchema: boolean;
showKeySchema: boolean;
setShowKeySchema: (show: boolean) => void;
lastUpdatedTimeString: string;
lastUpdated?: number | null;
lastObserved?: number | null;
selectedVersion: string;
versionList: Array<SemanticVersionStruct>;
schemaView: SchemaViewType;
Expand All @@ -152,7 +144,8 @@ export default function SchemaHeader({
hasKeySchema,
showKeySchema,
setShowKeySchema,
lastUpdatedTimeString,
lastUpdated,
lastObserved,
selectedVersion,
versionList,
schemaView,
Expand Down Expand Up @@ -233,7 +226,7 @@ export default function SchemaHeader({
))}
</LeftButtonsGroup>
<RightButtonsGroup>
<CurrentVersionTimestampText>{lastUpdatedTimeString}</CurrentVersionTimestampText>
<SchemaTimeStamps lastObserved={lastObserved} lastUpdated={lastUpdated} />
<BlameRadio value={schemaView} onChange={onSchemaViewToggle}>
<BlameRadioButton value={SchemaViewType.NORMAL} data-testid="schema-normal-button">
Normal
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { ClockCircleOutlined } from '@ant-design/icons';
import { Popover, Typography } from 'antd';
import React from 'react';
import styled from 'styled-components/macro';
import { toLocalDateTimeString, toRelativeTimeString } from '../../../../../shared/time/timeUtils';
import { ANTD_GRAY } from '../../../../shared/constants';

const CurrentVersionTimestampText = styled(Typography.Text)`
&&& {
line-height: 22px;
margin-top: 10px;
margin-right: 10px;
color: ${ANTD_GRAY[7]};
width: max-content;
}
`;

const TimeStampWrapper = styled.div`
margin-bottom: 5px;
`;

const StyledClockIcon = styled(ClockCircleOutlined)`
margin-right: 5px;
`;

interface Props {
lastUpdated?: number | null;
lastObserved?: number | null;
}

function SchemaTimeStamps(props: Props) {
const { lastUpdated, lastObserved } = props;

if (!lastUpdated && !lastObserved) return null;

return (
<Popover
content={
<>
{lastObserved && (
<TimeStampWrapper>Last observed on {toLocalDateTimeString(lastObserved)}.</TimeStampWrapper>
)}
{lastUpdated && <div>First reported on {toLocalDateTimeString(lastUpdated)}.</div>}
</>
}
>
<CurrentVersionTimestampText>
{lastObserved && (
<span>
<StyledClockIcon /> Last observed {toRelativeTimeString(lastObserved)}
</span>
)}
{!lastObserved && lastUpdated && (
<span>
<StyledClockIcon />
Reported {toRelativeTimeString(lastUpdated)}
</span>
)}
</CurrentVersionTimestampText>
</Popover>
);
}

export default SchemaTimeStamps;
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import { groupByFieldPath } from '../../../../dataset/profile/schema/utils/utils
import { ANTD_GRAY } from '../../../constants';
import { useBaseEntity, useEntityData } from '../../../EntityContext';
import { ChangeCategoryType, SchemaFieldBlame, SemanticVersionStruct } from '../../../../../../types.generated';
import { toLocalDateTimeString } from '../../../../../shared/time/timeUtils';
import { SchemaViewType } from '../../../../dataset/profile/schema/utils/types';
import SchemaTable from './SchemaTable';
import useGetSemanticVersionFromUrlParams from './utils/useGetSemanticVersionFromUrlParams';
Expand Down Expand Up @@ -120,11 +119,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => {
return groupByFieldPath(schemaMetadata?.fields, { showKeySchema });
}, [schemaMetadata, showKeySchema]);

const lastUpdatedTimeString = `Reported at ${
(getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp &&
toLocalDateTimeString(getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp)) ||
'unknown'
}`;
const lastUpdated = getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp;
const lastObserved = versionedDatasetData.data?.versionedDataset?.schema?.lastObserved;

const schemaFieldBlameList: Array<SchemaFieldBlame> =
(getSchemaBlameData?.getSchemaBlame?.schemaFieldBlameList as Array<SchemaFieldBlame>) || [];
Expand All @@ -139,7 +135,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => {
hasKeySchema={hasKeySchema}
showKeySchema={showKeySchema}
setShowKeySchema={setShowKeySchema}
lastUpdatedTimeString={lastUpdatedTimeString}
lastObserved={lastObserved}
lastUpdated={lastUpdated}
selectedVersion={selectedVersion}
versionList={versionList}
schemaView={schemaViewMode}
Expand Down
1 change: 1 addition & 0 deletions datahub-web-react/src/graphql/versionedDataset.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ query getVersionedDataset($urn: String!, $versionStamp: String) {
recursive
isPartOfKey
}
lastObserved
}
editableSchemaMetadata {
editableSchemaFieldInfo {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1522,6 +1522,16 @@ private Map<EntityAspectIdentifier, EnvelopedAspect> getEnvelopedAspects(final S
// since nowhere else is using it should be safe for now at least
envelopedAspect.setType(AspectType.VERSIONED);
envelopedAspect.setValue(aspect);

try {
if (currAspectEntry.getSystemMetadata() != null) {
final SystemMetadata systemMetadata = RecordUtils.toRecordTemplate(SystemMetadata.class, currAspectEntry.getSystemMetadata());
envelopedAspect.setSystemMetadata(systemMetadata);
}
} catch (Exception e) {
log.warn("Exception encountered when setting system metadata on enveloped aspect {}. Error: {}", envelopedAspect.getName(), e);
}

envelopedAspect.setCreated(new AuditStamp()
.setActor(UrnUtils.getUrn(currAspectEntry.getCreatedBy()))
.setTime(currAspectEntry.getCreatedOn().getTime())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ private String batchGetSelect(
outputParamsToValues.put(aspectArg, aspect);
outputParamsToValues.put(versionArg, version);

return String.format("SELECT urn, aspect, version, metadata, createdOn, createdBy, createdFor "
return String.format("SELECT urn, aspect, version, metadata, systemMetadata, createdOn, createdBy, createdFor "
+ "FROM %s WHERE urn = :%s AND aspect = :%s AND version = :%s",
EbeanAspectV2.class.getAnnotation(Table.class).name(), urnArg, aspectArg, versionArg);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,9 @@ record EnvelopedAspect {
* The audit stamp detailing who the aspect was created by and when
**/
created: AuditStamp

/**
* The system metadata for this aspect
**/
systemMetadata: optional SystemMetadata
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ namespace com.linkedin.schema
import com.linkedin.common.ChangeAuditStamps
import com.linkedin.common.DatasetUrn
import com.linkedin.dataset.SchemaFieldPath
import com.linkedin.mxe.SystemMetadata

/**
* SchemaMetadata to describe metadata related to store schema
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,53 @@
"name" : "created",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "The audit stamp detailing who the aspect was created by and when\n"
}, {
"name" : "systemMetadata",
"type" : {
"type" : "record",
"name" : "SystemMetadata",
"namespace" : "com.linkedin.mxe",
"doc" : "Metadata associated with each metadata change that is processed by the system",
"fields" : [ {
"name" : "lastObserved",
"type" : "long",
"doc" : "The timestamp the metadata was observed at",
"default" : 0,
"optional" : true
}, {
"name" : "runId",
"type" : "string",
"doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.",
"default" : "no-run-id-provided",
"optional" : true
}, {
"name" : "registryName",
"type" : "string",
"doc" : "The model registry name that was used to process this event",
"optional" : true
}, {
"name" : "registryVersion",
"type" : "string",
"doc" : "The model registry version that was used to process this event",
"optional" : true
}, {
"name" : "properties",
"type" : {
"type" : "map",
"values" : "string"
},
"doc" : "Additional properties",
"optional" : true
} ]
},
"doc" : "The system metadata for this aspect\n",
"optional" : true
} ]
}
},
"doc" : "A map of aspect name to aspect\n"
} ]
}, "com.linkedin.entity.EnvelopedAspect" ],
}, "com.linkedin.entity.EnvelopedAspect", "com.linkedin.mxe.SystemMetadata" ],
"schema" : {
"name" : "entitiesV2",
"namespace" : "com.linkedin.entity",
Expand Down
Loading

0 comments on commit 68fb0d8

Please sign in to comment.