From 32bf848c2d411c57901330605d1f23852ffc3ecd Mon Sep 17 00:00:00 2001 From: Masterchen09 <13187726+Masterchen09@users.noreply.github.com> Date: Thu, 19 May 2022 21:47:29 +0200 Subject: [PATCH] feat(dashboards): add datasets field to DashboardInfo aspect --- .../app/entity/dashboard/DashboardEntity.tsx | 13 ++++++++++++- .../tabs/Entity/DashboardDatasetsTab.tsx | 19 +++++++++++++++++++ .../src/graphql/dashboard.graphql | 3 +++ .../integration/looker/expected_output.json | 1 + .../looker/golden_test_allow_ingest.json | 1 + .../looker/golden_test_ingest.json | 1 + .../looker/golden_test_ingest_joins.json | 1 + .../golden_test_ingest_unaliased_joins.json | 1 + .../metabase/metabase_mces_golden.json | 1 + .../integration/mode/mode_mces_golden.json | 1 + .../powerbi/golden_test_ingest.json | 2 +- .../tableau/tableau_mces_golden.json | 4 ++++ .../tests/unit/test_redash_source.py | 1 + .../com/linkedin/dashboard/DashboardInfo.pdl | 13 +++++++++++++ .../com.linkedin.entity.aspects.snapshot.json | 15 +++++++++++++++ ...com.linkedin.entity.entities.snapshot.json | 15 +++++++++++++++ .../com.linkedin.entity.runs.snapshot.json | 15 +++++++++++++++ ...m.linkedin.platform.platform.snapshot.json | 15 +++++++++++++++ 18 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index 7a90826b711cf..923263863d810 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -13,6 +13,7 @@ import { SidebarAboutSection } from '../shared/containers/profile/sidebar/Sideba import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; import { DashboardChartsTab } from '../shared/tabs/Entity/DashboardChartsTab'; +import { DashboardDatasetsTab } from '../shared/tabs/Entity/DashboardDatasetsTab'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { GenericEntityProperties } from '../shared/types'; import { DashboardPreview } from './preview/DashboardPreview'; @@ -100,10 +101,20 @@ export class DashboardEntity implements Entity { name: 'Charts', component: DashboardChartsTab, display: { - visible: (_, _1) => true, + visible: (_, dashboard: GetDashboardQuery) => + (dashboard?.dashboard?.charts?.total || 0) > 0 || + (dashboard?.dashboard?.datasets?.total || 0) === 0, enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.charts?.total || 0) > 0, }, }, + { + name: 'Datasets', + component: DashboardDatasetsTab, + display: { + visible: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0, + enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0, + }, + }, ]} sidebarSections={[ { diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx new file mode 100644 index 0000000000000..62d612b0df4df --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx @@ -0,0 +1,19 @@ +import React from 'react'; +import { useBaseEntity } from '../../EntityContext'; +import { EntityType } from '../../../../../types.generated'; +import { EntityList } from './components/EntityList'; +import { useEntityRegistry } from '../../../../useEntityRegistry'; + +export const DashboardDatasetsTab = () => { + const entity = useBaseEntity() as any; + const dashboard = entity && entity.dashboard; + const datasets = dashboard?.datasets?.relationships.map((relationship) => relationship.entity); + const entityRegistry = useEntityRegistry(); + const totalDatasets = dashboard?.datasets?.total || 0; + const title = `Consumes ${totalDatasets} ${ + totalDatasets === 1 + ? entityRegistry.getEntityName(EntityType.Dataset) + : entityRegistry.getCollectionName(EntityType.Dataset) + }`; + return ; +}; diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql index e9e1412f5eeab..6da6264f14df8 100644 --- a/datahub-web-react/src/graphql/dashboard.graphql +++ b/datahub-web-react/src/graphql/dashboard.graphql @@ -4,6 +4,9 @@ query getDashboard($urn: String!) { charts: relationships(input: { types: ["Contains"], direction: OUTGOING, start: 0, count: 100 }) { ...fullRelationshipResults } + datasets: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { + ...fullRelationshipResults + } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { ...partialLineageResults } diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json index f4e272e04ceac..96393ce4b531a 100644 --- a/metadata-ingestion/tests/integration/looker/expected_output.json +++ b/metadata-ingestion/tests/integration/looker/expected_output.json @@ -56,6 +56,7 @@ "charts": [ "urn:li:chart:(looker,dashboard_elements.2)" ], + "datasets": [], "lastModified": { "created": { "time": 0, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index 4f031c7010a1d..df0190b8e7298 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -12,6 +12,7 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { "time": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index 4f031c7010a1d..df0190b8e7298 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -12,6 +12,7 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { "time": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index baaa64100bdb8..35bf69572d2a1 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -12,6 +12,7 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { "time": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index af3b7ab3eb48e..4438c07bb0ad4 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -12,6 +12,7 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { "time": 1586847600000, diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json index 127988ba381d7..3029f815e58bb 100644 --- a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json +++ b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json @@ -15,6 +15,7 @@ "urn:li:chart:(metabase,1)", "urn:li:chart:(metabase,2)" ], + "datasets": [], "lastModified": { "created": { "time": 1639417721742, diff --git a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json index 16e9bd0db5622..003a74ed0a6d1 100644 --- a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json +++ b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json @@ -14,6 +14,7 @@ "charts": [ "urn:li:chart:(mode,f622b9ee725b)" ], + "datasets": [], "lastModified": { "created": { "time": 1639169724316, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index f7f8f285c1c4c..de37604b3ebbe 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -311,7 +311,7 @@ "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { - "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"foo\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", + "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"foo\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index f19c9432bcf07..db69d5cbdc365 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -514,6 +514,7 @@ "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" ], + "datasets": [], "lastModified": { "created": { "time": 1640200234000, @@ -3283,6 +3284,7 @@ "charts": [ "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" ], + "datasets": [], "lastModified": { "created": { "time": 1639773866000, @@ -3368,6 +3370,7 @@ "title": "Story 1", "description": "", "charts": [], + "datasets": [], "lastModified": { "created": { "time": 1639773866000, @@ -7311,6 +7314,7 @@ "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" ], + "datasets": [], "lastModified": { "created": { "time": 1639768450000, diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py index 95d368a9498db..70e6f816949c4 100644 --- a/metadata-ingestion/tests/unit/test_redash_source.py +++ b/metadata-ingestion/tests/unit/test_redash_source.py @@ -487,6 +487,7 @@ def test_get_dashboard_snapshot(): "urn:li:chart:(redash,9)", "urn:li:chart:(redash,8)", ], + datasets=[], lastModified=ChangeAuditStamps( created=AuditStamp( time=1628882055288, actor="urn:li:corpuser:unknown" diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl index 10549227213c4..bf498f5af6244 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl @@ -5,6 +5,7 @@ import com.linkedin.common.ChangeAuditStamps import com.linkedin.common.ChartUrn import com.linkedin.common.Time import com.linkedin.common.Url +import com.linkedin.common.Urn import com.linkedin.common.CustomProperties import com.linkedin.common.ExternalReference @@ -47,6 +48,18 @@ record DashboardInfo includes CustomProperties, ExternalReference { } charts: array[ChartUrn] = [ ] + /** + * Datasets consumed by a dashboard + */ + @Relationship = { + "/*": { + "name": "Consumes", + "entityTypes": [ "dataset" ], + "isLineage": true + } + } + datasets: array[Urn] = [ ] + /** * Captures information about who created/last modified/deleted this dashboard and when */ diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index d815eaa2d77b1..927dfabaa12ea 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -1200,6 +1200,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 3a7dee437cfc6..365a1d1a597a2 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -1217,6 +1217,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index f3d81d99fe4f8..82c14d3cb3fe6 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -960,6 +960,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index e787f5bd4551d..96cc658d20172 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -1217,6 +1217,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps",