From 6d04511949f9563268fc7ddf64f1c684cf8ab6fe Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 27 Mar 2023 05:06:12 +0530 Subject: [PATCH] fix(ingest/looker): correct looker/lookml capability reports (#7683) --- .../ingestion/source/looker/looker_source.py | 15 +++++---------- .../ingestion/source/looker/lookml_source.py | 18 +++++++++--------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 3185c319d6c7a..0603e64997b31 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -24,7 +24,7 @@ import datahub.emitter.mce_builder as builder from datahub.configuration.common import AllowDenyPattern, ConfigurationError -from datahub.configuration.source_common import DatasetSourceConfigMixin +from datahub.configuration.source_common import DatasetSourceConfigMixin, EnvConfigMixin from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import create_embed_mcp @@ -106,7 +106,7 @@ class LookerDashboardSourceConfig( LookerAPIConfig, LookerCommonConfig, StatefulIngestionConfigBase, - DatasetSourceConfigMixin, + EnvConfigMixin, ): _removed_github_info = pydantic_removed_field("github_info") @@ -168,23 +168,18 @@ def external_url_defaults_to_api_config_base_url( ) -> Optional[str]: return v or values.get("base_url") - @validator("platform_instance") - def platform_instance_not_supported(cls, v: Optional[str]) -> Optional[str]: - if v is not None: - raise ConfigurationError("Looker Source doesn't support platform instances") - return v - @platform_name("Looker") @support_status(SupportStatus.CERTIFIED) @config_class(LookerDashboardSourceConfig) @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") -@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.PLATFORM_INSTANCE, "Not supported", supported=False) @capability( SourceCapability.OWNERSHIP, "Enabled by default, configured using `extract_owners`" ) @capability( - SourceCapability.USAGE_STATS, "Can be enabled using `extract_usage_history`" + SourceCapability.USAGE_STATS, + "Enabled by default, configured using `extract_usage_history`", ) class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase): """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 30e4f533f0791..c216c6d286606 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -27,11 +27,13 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, + capability, config_class, platform_name, support_status, ) from datahub.ingestion.api.registry import import_path +from datahub.ingestion.api.source import SourceCapability from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.git.git_import import GitClone @@ -175,7 +177,7 @@ def from_looker_connection( class LookMLSourceConfig( - LookerCommonConfig, StatefulIngestionConfigBase, DatasetSourceConfigMixin + LookerCommonConfig, StatefulIngestionConfigBase, EnvConfigMixin ): git_info: Optional[GitInfo] = Field( None, @@ -237,14 +239,6 @@ class LookMLSourceConfig( default=None, description="" ) - @validator("platform_instance") - def platform_instance_not_supported(cls, v: Optional[str]) -> Optional[str]: - if v is not None: - raise ConfigurationError( - "LookML Source doesn't support platform instance at the top level. However connection-specific platform instances are supported for generating lineage edges. Read the documentation to find out more." - ) - return v - @validator("connection_to_platform_map", pre=True) def convert_string_to_connection_def(cls, conn_map): # Previous version of config supported strings in connection map. This upconverts strings to ConnectionMap @@ -1094,6 +1088,12 @@ class LookerManifest: @platform_name("Looker") @config_class(LookMLSourceConfig) @support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.PLATFORM_INSTANCE, "Not supported", supported=False) +@capability(SourceCapability.LINEAGE_COARSE, "Supported by default") +@capability( + SourceCapability.LINEAGE_FINE, + "Enabled by default, configured using `extract_column_level_lineage`", +) class LookMLSource(StatefulIngestionSourceBase): """ This plugin extracts the following: