Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add selected_resources to the Jinja context #5001

Merged
merged 7 commits into from
Apr 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changes/unreleased/Features-20220404-190439.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
kind: Features
body: Add a variable called selected_resources in the Jinja context containing a list
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gshank could there be situations where selected_resources is referenced before it got set by get_graph_queue ? I think there isn't but want to double-check.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the selected_resources will be empty unless it's execution time, which should be okay. I suppose the list could be really big for a run command that executes the whole project, but I think that should be okay too; it doesn't do much except reference the list of nodes.

of all the resources matching the nodes for the --select, --exclude and/or --selector
parameters.
time: 2022-04-04T19:04:39.347479+02:00
custom:
Author: b-per
Issue: "3471"
PR: "5001"
11 changes: 11 additions & 0 deletions core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@

from dbt.utils import merge, AttrDict, MultiDict

from dbt import selected_resources

import agate


Expand Down Expand Up @@ -1143,6 +1145,15 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
msg = f"Env var required but not provided: '{var}'"
raise_parsing_error(msg)

@contextproperty
def selected_resources(self) -> List[str]:
"""The `selected_resources` variable contains a list of the resources
selected based on the parameters provided to the dbt command.
Currently, is not populated for the command `run-operation` that
doesn't support `--select`.
"""
return selected_resources.SELECTED_RESOURCES


class MacroContext(ProviderContext):
"""Internally, macros can be executed like nodes, with some restrictions:
Expand Down
3 changes: 3 additions & 0 deletions core/dbt/graph/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.state import PreviousState

from dbt import selected_resources


def get_package_names(nodes):
return set([node.split(".")[1] for node in nodes])
Expand Down Expand Up @@ -269,6 +271,7 @@ def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
dependecies.
"""
selected_nodes = self.get_selected(spec)
selected_resources.set_selected_resources(selected_nodes)
new_graph = self.full_graph.get_subset_graph(selected_nodes)
# should we give a way here for consumers to mutate the graph?
return GraphQueue(new_graph.graph, self.manifest, selected_nodes)
Expand Down
6 changes: 6 additions & 0 deletions core/dbt/selected_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
SELECTED_RESOURCES = []


def set_selected_resources(selected_resources):
global SELECTED_RESOURCES
SELECTED_RESOURCES = list(selected_resources)
1 change: 1 addition & 0 deletions test/unit/test_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ def assert_has_keys(
'sql',
'sql_now',
'adapter_macro',
'selected_resources'
}
REQUIRED_MODEL_KEYS = REQUIRED_MACRO_KEYS | {'this'}
MAYBE_KEYS = frozenset({'debug'})
Expand Down
35 changes: 35 additions & 0 deletions tests/functional/selected_resources/fixtures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
on_run_start_macro_assert_selected_models_expected_list = """
{% macro assert_selected_models_expected_list(expected_list) %}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really like the way you use this macro to test the functionality!


{% if execute and (expected_list is not none) %}

{% set sorted_selected_resources = selected_resources | sort %}
{% set sorted_expected_list = expected_list | sort %}

{% if sorted_selected_resources != sorted_expected_list %}
{{ exceptions.raise_compiler_error("FAIL: sorted_selected_resources" ~ sorted_selected_resources ~ " is different from " ~ sorted_expected_list) }}
{% endif %}

{% endif %}

{% endmacro %}
"""


my_model1 = """
select 1 as id
"""

my_model2 = """
select * from {{ ref('model1') }}
"""

my_snapshot = """
{% snapshot cc_all_snapshot %}
{{ config(
check_cols='all', unique_key='id', strategy='check',
target_database=database, target_schema=schema
) }}
select * from {{ ref('model2') }}
{% endsnapshot %}
"""
104 changes: 104 additions & 0 deletions tests/functional/selected_resources/test_selected_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import pytest
from dbt.tests.util import run_dbt
from tests.functional.selected_resources.fixtures import (
on_run_start_macro_assert_selected_models_expected_list,
my_model1,
my_model2,
my_snapshot,
)


@pytest.fixture(scope="class")
def macros():
return {
"assert_selected_models_expected_list.sql": on_run_start_macro_assert_selected_models_expected_list,
}


@pytest.fixture(scope="class")
def models():
return {"model1.sql": my_model1, "model2.sql": my_model2}


@pytest.fixture(scope="class")
def snapshots():
return {
"my_snapshot.sql": my_snapshot,
}


@pytest.fixture(scope="class")
def project_config_update():
return {
"on-run-start": "{{ assert_selected_models_expected_list(var('expected_list',None)) }}",
}


@pytest.fixture
def build_all(project):
run_dbt(["build"])


@pytest.mark.usefixtures("build_all")
class TestSelectedResources:
def test_selected_resources_build_selector(self, project):
results = run_dbt(
[
"build",
"--select",
"model1+",
"--vars",
'{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}',
]
)
assert results[0].status == "success"

def test_selected_resources_build_selector_subgraph(self, project):
results = run_dbt(
[
"build",
"--select",
"model2+",
"--vars",
'{"expected_list": ["model.test.model2", "snapshot.test.cc_all_snapshot"]}',
]
)
assert results[0].status == "success"

def test_selected_resources_run(self, project):
results = run_dbt(
[
"run",
"--select",
"model1+",
"--vars",
'{"expected_list": ["model.test.model2", "model.test.model1"]}',
]
)
assert results[0].status == "success"

def test_selected_resources_build_no_selector(self, project):
results = run_dbt(
[
"build",
"--vars",
'{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}',
]
)
assert results[0].status == "success"

def test_selected_resources_build_no_model(self, project):
results = run_dbt(
[
"build",
"--select",
"model_that_does_not_exist",
"--vars",
'{"expected_list": []}',
]
)
assert not results

def test_selected_resources_test_no_model(self, project):
results = run_dbt(["test", "--select", "model1+", "--vars", '{"expected_list": []}'])
assert not results