From c68de1066d3179d8db74875156279a74fbfd4e85 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 23 Sep 2019 17:28:13 -0400 Subject: [PATCH 1/5] Support external table config --- core/dbt/contracts/graph/parsed.py | 4 +++- core/dbt/contracts/graph/unparsed.py | 26 ++++++++++++++++++++++++++ core/dbt/parser/schemas.py | 10 +++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 52c05c3403e..ad93acfaaf5 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -12,7 +12,7 @@ import dbt.flags from dbt.contracts.graph.unparsed import ( UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, - UnparsedBaseNode, FreshnessThreshold + UnparsedBaseNode, FreshnessThreshold#, ExternalTable ) from dbt.contracts.util import Replaceable from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -139,6 +139,7 @@ def __len__(self): class ColumnInfo(JsonSchemaMixin, Replaceable): name: str description: str = '' + data_type: str = '' # Docrefs are not quite like regular references, as they indicate what they @@ -469,6 +470,7 @@ class ParsedSourceDefinition( quoting: Quoting = field(default_factory=Quoting) loaded_at_field: Optional[str] = None freshness: Optional[FreshnessThreshold] = None + external: Optional[Dict[str, Any]] = None docrefs: List[Docref] = field(default_factory=list) description: str = '' columns: Dict[str, ColumnInfo] = field(default_factory=dict) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 1df73a70185..cfb1b5a4fe6 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -47,6 +47,7 @@ class UnparsedRunHook(UnparsedNode): class NamedTested(JsonSchemaMixin, Replaceable): name: str description: str = '' + data_type: str = '' tests: Optional[List[Union[Dict[str, Any], str]]] = None def __post_init__(self): @@ -117,6 +118,30 @@ def status(self, age: float) -> FreshnessStatus: def __bool__(self): return self.warn_after is not None or self.error_after is not None +#@dataclass +#class ExternalPartition(JsonSchemaMixin, Replaceable): +# name: str +# description: str = '' +# data_type: str = '' +# expression: Optional[str] = None +# path_macro: Optional[str] = None +# vals: Optional[List, Dict[str, Any]] = None +# +#@dataclass +#class ExternalTable(JsonSchemaMixin, Mergeable): +# location: Optional[str] = None +# file_format: Optional[str] = None +# row_format: Optional[str] = None +# tbl_properties: Optional[str] = None +# auto_refresh: Optional[str] = None +# partitions: Optional[List[ExternalPartition]] = field(default_factory=list) +# +# def __bool__(self): +# return self.location is not None +# +# def __post_init__(self): +# if self.partitions is None: +# self.partitions = [] @dataclass class Quoting(JsonSchemaMixin, Mergeable): @@ -133,6 +158,7 @@ class UnparsedSourceTableDefinition(ColumnDescription, NodeDescription): freshness: Optional[FreshnessThreshold] = field( default_factory=FreshnessThreshold ) + external: Optional[Dict[str, Any]] = None def __post_init__(self): NodeDescription.__post_init__(self) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index a94030554bf..d9809b5e7d5 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -62,9 +62,10 @@ def __init__(self): self.column_info: Dict[str, ColumnInfo] = {} self.docrefs: List[Docref] = [] - def add(self, column_name, description): + def add(self, column_name, description, data_type): self.column_info[column_name] = ColumnInfo(name=column_name, - description=description) + description=description, + data_type=data_type) def collect_docrefs( @@ -216,9 +217,10 @@ def parse_column( ) -> None: column_name = column.name description = column.description + data_type = column.data_type collect_docrefs(block.target, refs, column_name, description) - refs.add(column_name, description) + refs.add(column_name, description, data_type) if not column.tests: return @@ -327,6 +329,7 @@ def generate_source_node( NodeType.Source, self.project.project_name, source.name, table.name ]) description = table.description or '' + external = table.external or '' source_description = source.description or '' collect_docrefs(source, refs, None, description, source_description) @@ -348,6 +351,7 @@ def generate_source_node( unique_id=unique_id, name=table.name, description=description, + external=external, source_name=source.name, source_description=source_description, loader=source.loader, From b93563bfeb8d6a6ae813674bb06b1500dbb7fdd0 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 24 Sep 2019 15:50:59 -0400 Subject: [PATCH 2/5] Create AdditionalPropertiesAllowed subdataclass --- core/dbt/contracts/graph/parsed.py | 26 ++-------- core/dbt/contracts/graph/unparsed.py | 72 ++++++++++++++++++---------- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index ad93acfaaf5..9c75b50a526 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -12,7 +12,8 @@ import dbt.flags from dbt.contracts.graph.unparsed import ( UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, - UnparsedBaseNode, FreshnessThreshold#, ExternalTable + UnparsedBaseNode, FreshnessThreshold, ExternalTable, + AdditionalPropertiesAllowed ) from dbt.contracts.util import Replaceable from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -56,7 +57,7 @@ def insensitive_patterns(*patterns: str): @dataclass class NodeConfig( - ExtensibleJsonSchemaMixin, Replaceable, MutableMapping[str, Any] + AdditionalPropertiesAllowed, Replaceable, MutableMapping[str, Any] ): enabled: bool = True materialized: str = 'view' @@ -73,25 +74,6 @@ class NodeConfig( def extra(self): return self._extra - @classmethod - def from_dict(cls, data, validate=True): - self = super().from_dict(data=data, validate=validate) - keys = self.to_dict(validate=False, omit_none=False) - for key, value in data.items(): - if key not in keys: - self._extra[key] = value - return self - - def to_dict(self, omit_none=True, validate=False): - data = super().to_dict(omit_none=omit_none, validate=validate) - data.update(self._extra) - return data - - def replace(self, **kwargs): - dct = self.to_dict(omit_none=False, validate=False) - dct.update(kwargs) - return self.from_dict(dct) - @classmethod def field_mapping(cls): return {'post_hook': 'post-hook', 'pre_hook': 'pre-hook'} @@ -470,7 +452,7 @@ class ParsedSourceDefinition( quoting: Quoting = field(default_factory=Quoting) loaded_at_field: Optional[str] = None freshness: Optional[FreshnessThreshold] = None - external: Optional[Dict[str, Any]] = None + external: Optional[ExternalTable] = None docrefs: List[Docref] = field(default_factory=list) description: str = '' columns: Dict[str, ColumnInfo] = field(default_factory=dict) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index cfb1b5a4fe6..88be7b527b6 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -2,7 +2,7 @@ from dbt.contracts.util import Replaceable, Mergeable from hologram import JsonSchemaMixin -from hologram.helpers import StrEnum +from hologram.helpers import StrEnum, ExtensibleJsonSchemaMixin from dataclasses import dataclass, field from datetime import timedelta @@ -118,30 +118,50 @@ def status(self, age: float) -> FreshnessStatus: def __bool__(self): return self.warn_after is not None or self.error_after is not None -#@dataclass -#class ExternalPartition(JsonSchemaMixin, Replaceable): -# name: str -# description: str = '' -# data_type: str = '' -# expression: Optional[str] = None -# path_macro: Optional[str] = None -# vals: Optional[List, Dict[str, Any]] = None -# -#@dataclass -#class ExternalTable(JsonSchemaMixin, Mergeable): -# location: Optional[str] = None -# file_format: Optional[str] = None -# row_format: Optional[str] = None -# tbl_properties: Optional[str] = None -# auto_refresh: Optional[str] = None -# partitions: Optional[List[ExternalPartition]] = field(default_factory=list) -# -# def __bool__(self): -# return self.location is not None -# -# def __post_init__(self): -# if self.partitions is None: -# self.partitions = [] +@dataclass +class AdditionalPropertiesAllowed(ExtensibleJsonSchemaMixin): + + @classmethod + def from_dict(cls, data, validate=True): + self = super().from_dict(data=data, validate=validate) + keys = self.to_dict(validate=False, omit_none=False) + for key, value in data.items(): + if key not in keys: + self._extra[key] = value + return self + + def to_dict(self, omit_none=True, validate=False): + data = super().to_dict(omit_none=omit_none, validate=validate) + data.update(self._extra) + return data + + def replace(self, **kwargs): + dct = self.to_dict(omit_none=False, validate=False) + dct.update(kwargs) + return self.from_dict(dct) + +@dataclass +class ExternalPartition(AdditionalPropertiesAllowed, Replaceable): + name: str + description: str = '' + data_type: str = '' + _extra: Dict[str, Any] = field(default_factory=dict) + +@dataclass +class ExternalTable(AdditionalPropertiesAllowed, Mergeable): + location: Optional[str] = None + file_format: Optional[str] = None + row_format: Optional[str] = None + tbl_properties: Optional[str] = None + partitions: Optional[List[ExternalPartition]] = field(default_factory=list) + _extra: Dict[str, Any] = field(default_factory=dict) + + def __bool__(self): + return self.location is not None + + def __post_init__(self): + if self.partitions is None: + self.partitions = [] @dataclass class Quoting(JsonSchemaMixin, Mergeable): @@ -158,7 +178,7 @@ class UnparsedSourceTableDefinition(ColumnDescription, NodeDescription): freshness: Optional[FreshnessThreshold] = field( default_factory=FreshnessThreshold ) - external: Optional[Dict[str, Any]] = None + external: Optional[ExternalTable] = None def __post_init__(self): NodeDescription.__post_init__(self) From 01be41392769dca77a68b54cdc1c79ca95d484ff Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 26 Sep 2019 15:24:21 -0400 Subject: [PATCH 3/5] Optional data_type. More rigorous subdataclass --- core/dbt/contracts/graph/parsed.py | 7 +------ core/dbt/contracts/graph/unparsed.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 9c75b50a526..cb168349b77 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -68,11 +68,6 @@ class NodeConfig( quoting: Dict[str, Any] = field(default_factory=dict) column_types: Dict[str, Any] = field(default_factory=dict) tags: Union[List[str], str] = field(default_factory=list) - _extra: Dict[str, Any] = field(default_factory=dict) - - @property - def extra(self): - return self._extra @classmethod def field_mapping(cls): @@ -121,7 +116,7 @@ def __len__(self): class ColumnInfo(JsonSchemaMixin, Replaceable): name: str description: str = '' - data_type: str = '' + data_type: Optional[str] = None # Docrefs are not quite like regular references, as they indicate what they diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 88be7b527b6..ea19cecf2f7 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -1,5 +1,6 @@ from dbt.node_types import UnparsedNodeType, NodeType, OperationType, MacroType from dbt.contracts.util import Replaceable, Mergeable +from dbt.exceptions import CompilationException from hologram import JsonSchemaMixin from hologram.helpers import StrEnum, ExtensibleJsonSchemaMixin @@ -47,7 +48,7 @@ class UnparsedRunHook(UnparsedNode): class NamedTested(JsonSchemaMixin, Replaceable): name: str description: str = '' - data_type: str = '' + data_type: Optional[str] = None tests: Optional[List[Union[Dict[str, Any], str]]] = None def __post_init__(self): @@ -120,6 +121,11 @@ def __bool__(self): @dataclass class AdditionalPropertiesAllowed(ExtensibleJsonSchemaMixin): + _extra: Dict[str, Any] = field(default_factory=dict) + + @property + def extra(self): + return self._extra @classmethod def from_dict(cls, data, validate=True): @@ -142,10 +148,15 @@ def replace(self, **kwargs): @dataclass class ExternalPartition(AdditionalPropertiesAllowed, Replaceable): - name: str + name: str = '' description: str = '' data_type: str = '' - _extra: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + if self.name == '' or self.data_type == '': + raise CompilationException ( + 'External partition columns must have names and data types' + ) @dataclass class ExternalTable(AdditionalPropertiesAllowed, Mergeable): @@ -154,7 +165,6 @@ class ExternalTable(AdditionalPropertiesAllowed, Mergeable): row_format: Optional[str] = None tbl_properties: Optional[str] = None partitions: Optional[List[ExternalPartition]] = field(default_factory=list) - _extra: Dict[str, Any] = field(default_factory=dict) def __bool__(self): return self.location is not None From 2ab21ed710aaad41d0f260cec695cbf0a9049933 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 26 Sep 2019 16:53:28 -0400 Subject: [PATCH 4/5] Address failing tests --- core/dbt/contracts/graph/parsed.py | 4 ++-- core/dbt/contracts/graph/unparsed.py | 20 +++++++++++--------- core/dbt/parser/schemas.py | 3 +-- test/unit/test_contracts_graph_unparsed.py | 2 ++ test/unit/test_parser.py | 3 ++- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index cb168349b77..3d938936d5a 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -5,14 +5,14 @@ from hologram import JsonSchemaMixin from hologram.helpers import ( - StrEnum, register_pattern, ExtensibleJsonSchemaMixin + StrEnum, register_pattern ) import dbt.clients.jinja import dbt.flags from dbt.contracts.graph.unparsed import ( UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, - UnparsedBaseNode, FreshnessThreshold, ExternalTable, + UnparsedBaseNode, FreshnessThreshold, ExternalTable, AdditionalPropertiesAllowed ) from dbt.contracts.util import Replaceable diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index ea19cecf2f7..e51b22fe141 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -119,14 +119,15 @@ def status(self, age: float) -> FreshnessStatus: def __bool__(self): return self.warn_after is not None or self.error_after is not None + @dataclass class AdditionalPropertiesAllowed(ExtensibleJsonSchemaMixin): _extra: Dict[str, Any] = field(default_factory=dict) - + @property def extra(self): return self._extra - + @classmethod def from_dict(cls, data, validate=True): self = super().from_dict(data=data, validate=validate) @@ -146,32 +147,31 @@ def replace(self, **kwargs): dct.update(kwargs) return self.from_dict(dct) + @dataclass class ExternalPartition(AdditionalPropertiesAllowed, Replaceable): name: str = '' description: str = '' data_type: str = '' - + def __post_init__(self): if self.name == '' or self.data_type == '': - raise CompilationException ( + raise CompilationException( 'External partition columns must have names and data types' ) + @dataclass class ExternalTable(AdditionalPropertiesAllowed, Mergeable): location: Optional[str] = None file_format: Optional[str] = None row_format: Optional[str] = None tbl_properties: Optional[str] = None - partitions: Optional[List[ExternalPartition]] = field(default_factory=list) + partitions: Optional[List[ExternalPartition]] = None def __bool__(self): return self.location is not None - def __post_init__(self): - if self.partitions is None: - self.partitions = [] @dataclass class Quoting(JsonSchemaMixin, Mergeable): @@ -188,7 +188,9 @@ class UnparsedSourceTableDefinition(ColumnDescription, NodeDescription): freshness: Optional[FreshnessThreshold] = field( default_factory=FreshnessThreshold ) - external: Optional[ExternalTable] = None + external: Optional[ExternalTable] = field( + default_factory=ExternalTable + ) def __post_init__(self): NodeDescription.__post_init__(self) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index d9809b5e7d5..4ce60fe353d 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -329,7 +329,6 @@ def generate_source_node( NodeType.Source, self.project.project_name, source.name, table.name ]) description = table.description or '' - external = table.external or '' source_description = source.description or '' collect_docrefs(source, refs, None, description, source_description) @@ -351,7 +350,7 @@ def generate_source_node( unique_id=unique_id, name=table.name, description=description, - external=external, + external=table.external, source_name=source.name, source_description=source_description, loader=source.loader, diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index 792e6be109c..e2200d523b0 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -300,6 +300,7 @@ def test_table_defaults(self): 'tests': [], 'columns': [], 'quoting': {}, + 'external': {}, 'freshness': {}, }, { @@ -308,6 +309,7 @@ def test_table_defaults(self): 'tests': [], 'columns': [], 'quoting': {'database': True}, + 'external': {}, 'freshness': {}, }, ], diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index f22988d1198..f193d61e6b0 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -26,7 +26,7 @@ ParsedSnapshotNode, TimestampSnapshotConfig, TimestampStrategy, ParsedAnalysisNode ) -from dbt.contracts.graph.unparsed import FreshnessThreshold +from dbt.contracts.graph.unparsed import FreshnessThreshold, ExternalTable from .utils import config_from_parts_or_dicts, normalize @@ -213,6 +213,7 @@ def test__parse_basic_source(self): name='my_table', loader='', freshness=FreshnessThreshold(), + external=ExternalTable(), source_description='', identifier='my_table', fqn=['snowplow', 'my_source', 'my_table'], From fcd86e3298efdc08fe56245a751b40872f9f0e91 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 27 Sep 2019 13:22:07 -0400 Subject: [PATCH 5/5] Address failing docs int tests --- .../test_docs_generate.py | 86 +++++++++++++------ .../035_docs_blocks/test_docs_blocks.py | 10 ++- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 69107fcf221..3d6ac0ccc2c 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -844,22 +844,27 @@ def expected_seeded_manifest(self, model_database=None): 'id': { 'name': 'id', 'description': 'The user ID number', + 'data_type': None, }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", + 'data_type': None, }, 'email': { 'name': 'email', 'description': "The user's email", + 'data_type': None, }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", + 'data_type': None, }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", + 'data_type': None, }, }, 'patch_path': schema_yml_path, @@ -1161,11 +1166,13 @@ def expected_postgres_references_manifest(self, model_database=None): 'columns': { 'first_name': { 'description': 'The first name being summarized', - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'ct': { 'description': 'The number of instances of the first name', - 'name': 'ct' + 'name': 'ct', + 'data_type': None }, }, 'config': { @@ -1227,11 +1234,13 @@ def expected_postgres_references_manifest(self, model_database=None): 'columns': { 'first_name': { 'description': 'The first name being summarized', - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'ct': { 'description': 'The number of instances of the first name', - 'name': 'ct' + 'name': 'ct', + 'data_type': None }, }, 'config': { @@ -1325,7 +1334,8 @@ def expected_postgres_references_manifest(self, model_database=None): 'columns': { 'id': { 'description': 'An ID field', - 'name': 'id' + 'name': 'id', + 'data_type': None } }, 'quoting': { @@ -1352,6 +1362,10 @@ def expected_postgres_references_manifest(self, model_database=None): 'documentation_package': '', }, ], + 'external': { + 'file_format': None, 'location': None, 'partitions': None, + 'row_format': None, 'tbl_properties': None + }, 'freshness': {'error_after': None, 'warn_after': None, 'filter': None}, 'identifier': 'seed', 'loaded_at_field': None, @@ -1589,23 +1603,28 @@ def expected_bigquery_complex_manifest(self): 'columns': { 'email': { 'description': "The user's email", - 'name': 'email' + 'name': 'email', + 'data_type': None }, 'first_name': { 'description': "The user's name", - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'id': { 'description': 'The user id', - 'name': 'id' + 'name': 'id', + 'data_type': None }, 'ip_address': { 'description': "The user's IP address", - 'name': 'ip_address' + 'name': 'ip_address', + 'data_type': None }, 'updated_at': { 'description': 'When the user was updated', - 'name': 'updated_at' + 'name': 'updated_at', + 'data_type': None }, }, 'description': 'A clustered and partitioned copy of the test model', @@ -1646,23 +1665,28 @@ def expected_bigquery_complex_manifest(self): 'columns': { 'email': { 'description': "The user's email", - 'name': 'email' + 'name': 'email', + 'data_type': None }, 'first_name': { 'description': "The user's name", - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'id': { 'description': 'The user id', - 'name': 'id' + 'name': 'id', + 'data_type': None }, 'ip_address': { 'description': "The user's IP address", - 'name': 'ip_address' + 'name': 'ip_address', + 'data_type': None }, 'updated_at': { 'description': 'When the user was updated', - 'name': 'updated_at' + 'name': 'updated_at', + 'data_type': None }, }, 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', @@ -1705,22 +1729,27 @@ def expected_bigquery_complex_manifest(self): 'field_1': { 'name': 'field_1', 'description': 'The first field', + 'data_type': None, }, 'field_2': { 'name': 'field_2', 'description': 'The second field', + 'data_type': None, }, 'field_3': { 'name': 'field_3', 'description': 'The third field', + 'data_type': None, }, 'nested_field.field_4': { 'name': 'nested_field.field_4', 'description': 'The first nested field', + 'data_type': None, }, 'nested_field.field_5': { 'name': 'nested_field.field_5', 'description': 'The second nested field', + 'data_type': None, }, }, 'description': 'The test model', @@ -1963,22 +1992,27 @@ def expected_redshift_incremental_view_manifest(self): 'id': { 'name': 'id', 'description': 'The user ID number', + 'data_type': None, }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", + 'data_type': None, }, 'email': { 'name': 'email', 'description': "The user's email", + 'data_type': None, }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", + 'data_type': None, }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", + 'data_type': None, }, }, 'patch_path': self.dir('rs_models/schema.yml'), @@ -2179,12 +2213,12 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'target/compiled/test/model.sql' ), 'columns': { - 'id': {'description': 'The user ID number', 'name': 'id'}, - 'first_name': {'description': "The user's first name", 'name': 'first_name'}, - 'email': {'description': "The user's email", 'name': 'email'}, - 'ip_address': {'description': "The user's IP address", 'name': 'ip_address'}, + 'id': {'description': 'The user ID number', 'name': 'id', 'data_type': None}, + 'first_name': {'description': "The user's first name", 'name': 'first_name', 'data_type': None}, + 'email': {'description': "The user's email", 'name': 'email', 'data_type': None}, + 'ip_address': {'description': "The user's IP address", 'name': 'ip_address', 'data_type': None}, 'updated_at': {'description': "The last time this user's email was updated", - 'name': 'updated_at'} + 'name': 'updated_at', 'data_type': None} }, 'compiled': True, 'compiled_sql': compiled_sql, @@ -2466,11 +2500,13 @@ def expected_postgres_references_run_results(self): 'columns': { 'first_name': { 'description': 'The first name being summarized', - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'ct': { 'description': 'The number of instances of the first name', - 'name': 'ct' + 'name': 'ct', + 'data_type': None }, }, 'compiled': True, @@ -2556,11 +2592,13 @@ def expected_postgres_references_run_results(self): 'columns': { 'first_name': { 'description': 'The first name being summarized', - 'name': 'first_name' + 'name': 'first_name', + 'data_type': None }, 'ct': { 'description': 'The number of instances of the first name', - 'name': 'ct' + 'name': 'ct', + 'data_type': None }, }, 'compiled': True, diff --git a/test/integration/035_docs_blocks/test_docs_blocks.py b/test/integration/035_docs_blocks/test_docs_blocks.py index 48f14410e9e..d6d7b4c5631 100644 --- a/test/integration/035_docs_blocks/test_docs_blocks.py +++ b/test/integration/035_docs_blocks/test_docs_blocks.py @@ -36,7 +36,8 @@ def test_postgres_valid_doc_ref(self): self.assertEqual( { 'name': 'id', - 'description': 'The user ID number' + 'description': 'The user ID number', + 'data_type': None, }, model_data['columns']['id'] ) @@ -44,6 +45,7 @@ def test_postgres_valid_doc_ref(self): { 'name': 'first_name', 'description': "The user's first name", + 'data_type': None, }, model_data['columns']['first_name'] ) @@ -52,6 +54,7 @@ def test_postgres_valid_doc_ref(self): { 'name': 'last_name', 'description': "The user's last name", + 'data_type': None, }, model_data['columns']['last_name'] ) @@ -75,7 +78,8 @@ def test_postgres_alternative_docs_path(self): self.assertEqual( { 'name': 'id', - 'description': 'The user ID number with alternative text' + 'description': 'The user ID number with alternative text', + 'data_type': None, }, model_data['columns']['id'] ) @@ -83,6 +87,7 @@ def test_postgres_alternative_docs_path(self): { 'name': 'first_name', 'description': "The user's first name", + 'data_type': None, }, model_data['columns']['first_name'] ) @@ -91,6 +96,7 @@ def test_postgres_alternative_docs_path(self): { 'name': 'last_name', 'description': "The user's last name in this other file", + 'data_type': None, }, model_data['columns']['last_name'] )