Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add selected_resources to the Jinja context #5001

Merged
merged 7 commits into from
Apr 12, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changes/unreleased/Features-20220404-190439.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
kind: Features
body: Add a variable called selected_resources in the Jinja context containing a list
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gshank could there be situations where selected_resources is referenced before it got set by get_graph_queue ? I think there isn't but want to double-check.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the selected_resources will be empty unless it's execution time, which should be okay. I suppose the list could be really big for a run command that executes the whole project, but I think that should be okay too; it doesn't do much except reference the list of nodes.

of all the resources matching the nodes for the --select, --exclude and/or --selector
parameters.
time: 2022-04-04T19:04:39.347479+02:00
custom:
Author: b-per
Issue: "3471"
PR: "5001"
11 changes: 11 additions & 0 deletions core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@

from dbt.utils import merge, AttrDict, MultiDict

from dbt import selected_resources

import agate


Expand Down Expand Up @@ -1241,6 +1243,15 @@ def this(self) -> Optional[RelationProxy]:
return None
return self.db_wrapper.Relation.create_from(self.config, self.model)

@contextproperty
def selected_resources(self) -> Any:
"""The `selected_resources` variable contains a list of the resources
selected based on the parameters provided to the dbt command.
Currently, is not populated for the command `run-operation` that
doesn't support `--select`.
"""
return selected_resources.SELECTED_RESOURCES


# This is called by '_context_for', used in 'render_with_context'
def generate_parser_model_context(
Expand Down
3 changes: 3 additions & 0 deletions core/dbt/graph/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.state import PreviousState

from dbt import selected_resources


def get_package_names(nodes):
return set([node.split(".")[1] for node in nodes])
Expand Down Expand Up @@ -269,6 +271,7 @@ def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
dependecies.
"""
selected_nodes = self.get_selected(spec)
selected_resources.set_selected_resources(selected_nodes)
new_graph = self.full_graph.get_subset_graph(selected_nodes)
# should we give a way here for consumers to mutate the graph?
return GraphQueue(new_graph.graph, self.manifest, selected_nodes)
Expand Down
6 changes: 6 additions & 0 deletions core/dbt/selected_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
SELECTED_RESOURCES = []


def set_selected_resources(selected_resources):
global SELECTED_RESOURCES
SELECTED_RESOURCES = list(selected_resources)
35 changes: 35 additions & 0 deletions tests/functional/selected_resources/fixtures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
on_run_start_macro_assert_selected_models_expected_list = """
{% macro assert_selected_models_expected_list(expected_list) %}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really like the way you use this macro to test the functionality!


{% if execute %}

{% set sorted_selected_resources = selected_resources | sort %}
{% set sorted_expected_list = expected_list | sort %}

{% if sorted_selected_resources != sorted_expected_list %}
{{ exceptions.raise_compiler_error("FAIL: sorted_selected_resources" ~ sorted_selected_resources ~ " is different from " ~ sorted_expected_list) }}
{% endif %}

{% endif %}

{% endmacro %}
"""


my_model1 = """
select 1 as id
"""

my_model2 = """
select * from {{ ref('model1') }}
"""

my_snapshot = """
{% snapshot cc_all_snapshot %}
{{ config(
check_cols='all', unique_key='id', strategy='check',
target_database=database, target_schema=schema
) }}
select * from {{ ref('model2') }}
{% endsnapshot %}
"""
81 changes: 81 additions & 0 deletions tests/functional/selected_resources/test_selected_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import pytest
from dbt.tests.util import run_dbt
from tests.functional.selected_resources.fixtures import (
on_run_start_macro_assert_selected_models_expected_list,
my_model1,
my_model2,
my_snapshot,
)


@pytest.fixture(scope="class")
def macros():
return {
"assert_selected_models_expected_list.sql": on_run_start_macro_assert_selected_models_expected_list,
}


@pytest.fixture(scope="class")
def models():
return {"model1.sql": my_model1, "model2.sql": my_model2}


@pytest.fixture(scope="class")
def snapshots():
return {
"my_snapshot.sql": my_snapshot,
}


@pytest.fixture(scope="class")
def project_config_update():
return {
"on-run-start": "{{ assert_selected_models_expected_list(var('expected_list',[])) }}",
}


def test_selected_resources_build(project):
results = run_dbt(
[
"build",
"--select",
"model1+",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we do a model2+ as one of the test cases? since the result of this selection is the same as the no select specified test below.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just modified a test to be model2+

It was failing at first (because model1 was not built) so I moved all the tests to a Class that uses the fixture build_all, meaning that a full dbt build is run before each test (which allows us to select nodes that are depending on other ones)

"--vars",
'{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}',
]
)
assert results[0].status == "success"


def test_selected_resources_run(project):
results = run_dbt(
[
"run",
"--select",
"model1+",
"--vars",
'{"expected_list": ["model.test.model2", "model.test.model1"]}',
]
)
assert results[0].status == "success"


def test_selected_resources_build_all(project):
results = run_dbt(
[
"build",
"--vars",
'{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}',
]
)
assert results[0].status == "success"


def test_selected_resources_build_no_model(project):
results = run_dbt(["build", "--select", "model_that_does_not_exist"])
assert not results


def test_selected_resources_test_no_model(project):
results = run_dbt(["test", "--select", "model1+", "--vars", '{"expected_list": []}'])
assert not results