Skip to content

Commit

Permalink
Fix empty tag in case of custom parser (#1100)
Browse files Browse the repository at this point in the history
## Description
As of version 1.5.1, using a custom parser to parse a dbt project
hardcodes the tags value to an empty list [] when initializing 
the DbtNode object. This creates issues for users utilizing 
the select option in the DAG with render_config, 
resulting in distorted DAG rendering, as shown below.

<img width="1435" alt="Screenshot 2024-07-16 at 10 51 00 PM"
src="https://github.com/user-attachments/assets/f6da8ce0-cd80-479f-a0ad-a537a5f2c776">

This PR parses the tag values from the project and uses them to
initialize the DbtNode object. Post this change rendering appears like
<img width="1473" alt="Screenshot 2024-07-17 at 7 01 53 PM"
src="https://github.com/user-attachments/assets/8d15d4f0-7e4d-4808-b00d-2ac3286db5a9">


## Related Issue(s)

closes: #943 
closes: #693

## Breaking Change?

No

## Checklist

- [x] I have made corresponding changes to the documentation (if
required)
- [x] I have added tests that prove my fix is effective or that my
feature works
  • Loading branch information
pankajastro authored Jul 19, 2024
1 parent fd264e3 commit b25d5ea
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 5 deletions.
3 changes: 2 additions & 1 deletion cosmos/dbt/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,7 @@ def load_via_custom_parser(self) -> None:
)
for model_name, model in models:
config = {item.split(":")[0]: item.split(":")[-1] for item in model.config.config_selectors}
tags = [selector for selector in model.config.config_selectors if selector.startswith("tags:")]
node = DbtNode(
unique_id=f"{model.type.value}.{self.project.project_name}.{model_name}",
resource_type=DbtResourceType(model.type.value),
Expand All @@ -554,7 +555,7 @@ def load_via_custom_parser(self) -> None:
self.render_config.project_path.as_posix(), self.execution_config.project_path.as_posix()
)
),
tags=[],
tags=tags or [],
config=config,
)
nodes[model_name] = node
Expand Down
2 changes: 2 additions & 0 deletions dev/dags/dbt/jaffle_shop/models/customers.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
{{ config(tags=["customers"]) }}

with customers as (

select * from {{ ref('stg_customers') }}
Expand Down
4 changes: 2 additions & 2 deletions tests/dbt/parser/test_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@ def test_dbtmodelconfig___repr__():
dbt_model = DbtModel(name="some_name", type=DbtModelType.DBT_MODEL, path=SAMPLE_MODEL_SQL_PATH)
expected_start = (
"DbtModel(name='some_name', type='DbtModelType.DBT_MODEL', "
"path='dev/dags/dbt/jaffle_shop/models/customers.sql', config=DbtModelConfig(config_selectors=set(), "
"upstream_models={'stg_"
"path='dev/dags/dbt/jaffle_shop/models/customers.sql', config=DbtModelConfig("
"config_selectors={'tags:customers'}, upstream_models={'stg_"
)
assert str(dbt_model).startswith(expected_start)

Expand Down
30 changes: 28 additions & 2 deletions tests/dbt/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,32 @@ def test_load_via_load_via_custom_parser(project_name):
assert len(dbt_graph.nodes) == 28


def test_load_via_load_via_custom_parser_select_rendering_config():
project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / "jaffle_shop")
execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME)
render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, select=["customers"])
profile_config = ProfileConfig(
profile_name="test",
target_name="test",
profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml",
)
dbt_graph = DbtGraph(
project=project_config,
profile_config=profile_config,
render_config=render_config,
execution_config=execution_config,
)

dbt_graph.load_via_custom_parser()

assert len(dbt_graph.filtered_nodes) == 1
for model_name in dbt_graph.filtered_nodes:
assert model_name == "customers"
filter_node = dbt_graph.filtered_nodes.get(model_name)
assert filter_node.file_path == DBT_PROJECTS_ROOT_DIR / "jaffle_shop/models/customers.sql"
assert filter_node.tags == ["tags:customers"]


@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency", return_value=None)
def test_update_node_dependency_called(mock_update_node_dependency):
project_config = ProjectConfig(
Expand Down Expand Up @@ -1387,9 +1413,9 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir
hash_dir, hash_args = version.split(",")
assert hash_args == "d41d8cd98f00b204e9800998ecf8427e"
if sys.platform == "darwin":
assert hash_dir == "cdc6f0bec00f4edc616f3aa755a34330"
assert hash_dir == "465fc0735d8bef08a0d375b2315069bb"
else:
assert hash_dir == "77d08d6da374330ac1b49438ff2873f7"
assert hash_dir == "6c662da10b64a8390c469c884af88321"


@pytest.mark.integration
Expand Down

0 comments on commit b25d5ea

Please sign in to comment.