feat(dashboards): add datasets field to DashboardInfo aspect (#5188)

Co-authored-by: John Joyce <[email protected]>
datahub-project · Jul 14, 2022 · a030344 · a030344
1 parent 8d4b7cf
commit a030344
Show file tree

Hide file tree

Showing 18 changed files with 120 additions and 2 deletions.
diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
@@ -13,6 +13,7 @@ import { SidebarAboutSection } from '../shared/containers/profile/sidebar/Sideba
 import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
 import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab';
 import { DashboardChartsTab } from '../shared/tabs/Entity/DashboardChartsTab';
+import { DashboardDatasetsTab } from '../shared/tabs/Entity/DashboardDatasetsTab';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
 import { GenericEntityProperties } from '../shared/types';
 import { DashboardPreview } from './preview/DashboardPreview';
@@ -100,10 +101,20 @@ export class DashboardEntity implements Entity<Dashboard> {
                     name: 'Charts',
                     component: DashboardChartsTab,
                     display: {
-                        visible: (_, _1) => true,
+                        visible: (_, dashboard: GetDashboardQuery) =>
+                            (dashboard?.dashboard?.charts?.total || 0) > 0 ||
+                            (dashboard?.dashboard?.datasets?.total || 0) === 0,
                         enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.charts?.total || 0) > 0,
                     },
                 },
+                {
+                    name: 'Datasets',
+                    component: DashboardDatasetsTab,
+                    display: {
+                        visible: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0,
+                        enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0,
+                    },
+                },
             ]}
             sidebarSections={[
                 {

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx
@@ -0,0 +1,19 @@
+import React from 'react';
+import { useBaseEntity } from '../../EntityContext';
+import { EntityType } from '../../../../../types.generated';
+import { EntityList } from './components/EntityList';
+import { useEntityRegistry } from '../../../../useEntityRegistry';
+
+export const DashboardDatasetsTab = () => {
+    const entity = useBaseEntity() as any;
+    const dashboard = entity && entity.dashboard;
+    const datasets = dashboard?.datasets?.relationships.map((relationship) => relationship.entity);
+    const entityRegistry = useEntityRegistry();
+    const totalDatasets = dashboard?.datasets?.total || 0;
+    const title = `Consumes ${totalDatasets} ${
+        totalDatasets === 1
+            ? entityRegistry.getEntityName(EntityType.Dataset)
+            : entityRegistry.getCollectionName(EntityType.Dataset)
+    }`;
+    return <EntityList title={title} type={EntityType.Dataset} entities={datasets || []} />;
+};
diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql
@@ -4,6 +4,9 @@ query getDashboard($urn: String!) {
         charts: relationships(input: { types: ["Contains"], direction: OUTGOING, start: 0, count: 100 }) {
             ...fullRelationshipResults
         }
+        datasets: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) {
+            ...fullRelationshipResults
+        }
         upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) {
             ...partialLineageResults
         }

diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json
@@ -56,6 +56,7 @@
                         "charts": [
                             "urn:li:chart:(looker,dashboard_elements.2)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 0,

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
@@ -12,6 +12,7 @@
                         "title": "foo",
                         "description": "lorem ipsum",
                         "charts": [],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json
@@ -12,6 +12,7 @@
                         "title": "foo",
                         "description": "lorem ipsum",
                         "charts": [],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
@@ -12,6 +12,7 @@
                         "title": "foo",
                         "description": "lorem ipsum",
                         "charts": [],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,

diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
@@ -12,6 +12,7 @@
                         "title": "foo",
                         "description": "lorem ipsum",
                         "charts": [],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1586847600000,

diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json
@@ -15,6 +15,7 @@
                             "urn:li:chart:(metabase,1)",
                             "urn:li:chart:(metabase,2)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639417721742,

diff --git a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json
@@ -14,6 +14,7 @@
                         "charts": [
                             "urn:li:chart:(mode,f622b9ee725b)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639169724316,

diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
@@ -311,7 +311,7 @@
         "changeType": "UPSERT",
         "aspectName": "dashboardInfo",
         "aspect": {
-            "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"foo\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}",
+            "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"foo\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}",
             "contentType": "application/json"
         },
         "systemMetadata": {

diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json
@@ -514,6 +514,7 @@
                             "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)",
                             "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1640200234000,
@@ -3283,6 +3284,7 @@
                         "charts": [
                             "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639773866000,
@@ -3368,6 +3370,7 @@
                         "title": "Story 1",
                         "description": "",
                         "charts": [],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639773866000,
@@ -7311,6 +7314,7 @@
                             "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)",
                             "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)"
                         ],
+                        "datasets": [],
                         "lastModified": {
                             "created": {
                                 "time": 1639768450000,

diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py
@@ -487,6 +487,7 @@ def test_get_dashboard_snapshot():
                     "urn:li:chart:(redash,9)",
                     "urn:li:chart:(redash,8)",
                 ],
+                datasets=[],
                 lastModified=ChangeAuditStamps(
                     created=AuditStamp(
                         time=1628882055288, actor="urn:li:corpuser:unknown"

diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl
@@ -5,6 +5,7 @@ import com.linkedin.common.ChangeAuditStamps
 import com.linkedin.common.ChartUrn
 import com.linkedin.common.Time
 import com.linkedin.common.Url
+import com.linkedin.common.Urn
 import com.linkedin.common.CustomProperties
 import com.linkedin.common.ExternalReference
 
@@ -47,6 +48,18 @@ record DashboardInfo includes CustomProperties, ExternalReference {
   }
   charts: array[ChartUrn] = [ ]
 
+  /**
+   * Datasets consumed by a dashboard
+   */
+  @Relationship = {
+    "/*": {
+      "name": "Consumes",
+      "entityTypes": [ "dataset" ],
+      "isLineage": true
+    }
+  }
+  datasets: array[Urn] = [ ]
+
   /**
    * Captures information about who created/last modified/deleted this dashboard and when
    */

diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
@@ -1200,6 +1200,21 @@
           "name" : "Contains"
         }
       }
+    }, {
+      "name" : "datasets",
+      "type" : {
+        "type" : "array",
+        "items" : "com.linkedin.common.Urn"
+      },
+      "doc" : "Datasets consumed by a dashboard",
+      "default" : [ ],
+      "Relationship" : {
+        "/*" : {
+          "entityTypes" : [ "dataset" ],
+          "isLineage" : true,
+          "name" : "Consumes"
+        }
+      }
     }, {
       "name" : "lastModified",
       "type" : "com.linkedin.common.ChangeAuditStamps",

diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
@@ -1217,6 +1217,21 @@
           "name" : "Contains"
         }
       }
+    }, {
+      "name" : "datasets",
+      "type" : {
+        "type" : "array",
+        "items" : "com.linkedin.common.Urn"
+      },
+      "doc" : "Datasets consumed by a dashboard",
+      "default" : [ ],
+      "Relationship" : {
+        "/*" : {
+          "entityTypes" : [ "dataset" ],
+          "isLineage" : true,
+          "name" : "Consumes"
+        }
+      }
     }, {
       "name" : "lastModified",
       "type" : "com.linkedin.common.ChangeAuditStamps",

diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
@@ -960,6 +960,21 @@
           "name" : "Contains"
         }
       }
+    }, {
+      "name" : "datasets",
+      "type" : {
+        "type" : "array",
+        "items" : "com.linkedin.common.Urn"
+      },
+      "doc" : "Datasets consumed by a dashboard",
+      "default" : [ ],
+      "Relationship" : {
+        "/*" : {
+          "entityTypes" : [ "dataset" ],
+          "isLineage" : true,
+          "name" : "Consumes"
+        }
+      }
     }, {
       "name" : "lastModified",
       "type" : "com.linkedin.common.ChangeAuditStamps",

diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json
@@ -1217,6 +1217,21 @@
           "name" : "Contains"
         }
       }
+    }, {
+      "name" : "datasets",
+      "type" : {
+        "type" : "array",
+        "items" : "com.linkedin.common.Urn"
+      },
+      "doc" : "Datasets consumed by a dashboard",
+      "default" : [ ],
+      "Relationship" : {
+        "/*" : {
+          "entityTypes" : [ "dataset" ],
+          "isLineage" : true,
+          "name" : "Consumes"
+        }
+      }
     }, {
       "name" : "lastModified",
       "type" : "com.linkedin.common.ChangeAuditStamps",