From 4e38f7b22b6f288857cfc2b00823fd963f30d287 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 8 Jun 2023 19:09:37 +0100 Subject: [PATCH] Refactor tests to be separated from application code (#313) Closes: #302 --------- Co-authored-by: Julian LaNeve --- .coveragerc | 3 +- cosmos/providers/dbt/dataset.py | 3 ++ cosmos/providers/dbt/tests/test_dataset.py | 51 ------------------- pyproject.toml | 2 +- {cosmos/tests => tests}/__init__.py | 0 {cosmos => tests}/conftest.py | 0 tests/core/__init__.py | 0 .../core/tests => tests/core}/test_airflow.py | 0 tests/providers/__init__.py | 0 tests/providers/dbt/__init__.py | 0 tests/providers/dbt/core/__init__.py | 0 .../providers/dbt/core/operators/__init__.py | 0 .../dbt/core/operators}/test_docker.py | 0 .../dbt/core/operators}/test_kubernetes.py | 0 .../dbt/core/operators}/test_local.py | 0 .../dbt/core/operators}/test_virtualenv.py | 0 tests/providers/dbt/core/profiles/__init__.py | 0 .../bigquery}/test_bq_service_account_file.py | 0 .../profiles/databricks}/test_dbr_token.py | 0 .../profiles/exasol}/test_exasol_user_pass.py | 0 .../profiles/postgres}/test_pg_user_pass.py | 0 .../redshift}/test_redshift_user_pass.py | 0 .../snowflake}/test_snowflake_user_pass.py | 0 .../profiles/spark/tests/test_spark_thrift.py | 0 .../core/profiles/trino}/test_trino_base.py | 0 .../profiles/trino}/test_trino_certificate.py | 0 .../core/profiles/trino}/test_trino_jwt.py | 0 .../core/profiles/trino}/test_trino_ldap.py | 0 tests/providers/dbt/core/utils/__init__.py | 0 .../core/utils}/test_data_aware_scheduling.py | 0 .../dbt/core/utils}/test_warn_parsing.py | 0 tests/providers/dbt/parser/__init__.py | 0 .../providers/dbt/parser}/test_project.py | 0 tests/providers/dbt/test_dataset.py | 16 ++++++ .../providers/dbt}/test_export.py | 0 .../providers/dbt}/test_render.py | 0 {cosmos/tests => tests}/test-connections.yaml | 0 {cosmos/tests => tests}/test_example_dags.py | 2 +- {cosmos/tests => tests}/test_version.py | 0 {cosmos/tests => tests}/utils.py | 0 40 files changed, 22 insertions(+), 55 deletions(-) delete mode 100644 cosmos/providers/dbt/tests/test_dataset.py rename {cosmos/tests => tests}/__init__.py (100%) rename {cosmos => tests}/conftest.py (100%) create mode 100644 tests/core/__init__.py rename {cosmos/core/tests => tests/core}/test_airflow.py (100%) create mode 100644 tests/providers/__init__.py create mode 100644 tests/providers/dbt/__init__.py create mode 100644 tests/providers/dbt/core/__init__.py create mode 100644 tests/providers/dbt/core/operators/__init__.py rename {cosmos/providers/dbt/core/operators/tests => tests/providers/dbt/core/operators}/test_docker.py (100%) rename {cosmos/providers/dbt/core/operators/tests => tests/providers/dbt/core/operators}/test_kubernetes.py (100%) rename {cosmos/providers/dbt/core/operators/tests => tests/providers/dbt/core/operators}/test_local.py (100%) rename {cosmos/providers/dbt/core/operators/tests => tests/providers/dbt/core/operators}/test_virtualenv.py (100%) create mode 100644 tests/providers/dbt/core/profiles/__init__.py rename {cosmos/providers/dbt/core/profiles/bigquery/tests => tests/providers/dbt/core/profiles/bigquery}/test_bq_service_account_file.py (100%) rename {cosmos/providers/dbt/core/profiles/databricks/tests => tests/providers/dbt/core/profiles/databricks}/test_dbr_token.py (100%) rename {cosmos/providers/dbt/core/profiles/exasol/tests => tests/providers/dbt/core/profiles/exasol}/test_exasol_user_pass.py (100%) rename {cosmos/providers/dbt/core/profiles/postgres/tests => tests/providers/dbt/core/profiles/postgres}/test_pg_user_pass.py (100%) rename {cosmos/providers/dbt/core/profiles/redshift/tests => tests/providers/dbt/core/profiles/redshift}/test_redshift_user_pass.py (100%) rename {cosmos/providers/dbt/core/profiles/snowflake/tests => tests/providers/dbt/core/profiles/snowflake}/test_snowflake_user_pass.py (100%) rename {cosmos => tests}/providers/dbt/core/profiles/spark/tests/test_spark_thrift.py (100%) rename {cosmos/providers/dbt/core/profiles/trino/tests => tests/providers/dbt/core/profiles/trino}/test_trino_base.py (100%) rename {cosmos/providers/dbt/core/profiles/trino/tests => tests/providers/dbt/core/profiles/trino}/test_trino_certificate.py (100%) rename {cosmos/providers/dbt/core/profiles/trino/tests => tests/providers/dbt/core/profiles/trino}/test_trino_jwt.py (100%) rename {cosmos/providers/dbt/core/profiles/trino/tests => tests/providers/dbt/core/profiles/trino}/test_trino_ldap.py (100%) create mode 100644 tests/providers/dbt/core/utils/__init__.py rename {cosmos/providers/dbt/core/utils/tests => tests/providers/dbt/core/utils}/test_data_aware_scheduling.py (100%) rename {cosmos/providers/dbt/core/utils/tests => tests/providers/dbt/core/utils}/test_warn_parsing.py (100%) create mode 100644 tests/providers/dbt/parser/__init__.py rename {cosmos/providers/dbt/parser/tests => tests/providers/dbt/parser}/test_project.py (100%) create mode 100644 tests/providers/dbt/test_dataset.py rename {cosmos/providers/dbt/tests => tests/providers/dbt}/test_export.py (100%) rename {cosmos/providers/dbt/tests => tests/providers/dbt}/test_render.py (100%) rename {cosmos/tests => tests}/test-connections.yaml (100%) rename {cosmos/tests => tests}/test_example_dags.py (96%) rename {cosmos/tests => tests}/test_version.py (100%) rename {cosmos/tests => tests}/utils.py (100%) diff --git a/.coveragerc b/.coveragerc index 7bf9333d2..b24d4a469 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,3 @@ [run] omit = - */test_* - cosmos/tests/utils.py + tests/* diff --git a/cosmos/providers/dbt/dataset.py b/cosmos/providers/dbt/dataset.py index 9a5686c33..6c4da09f0 100644 --- a/cosmos/providers/dbt/dataset.py +++ b/cosmos/providers/dbt/dataset.py @@ -14,3 +14,6 @@ def __init__(self, id: str, *args, **kwargs): def __eq__(self, other) -> bool: return self.id == other.id + + +__all__ = ["Dataset"] diff --git a/cosmos/providers/dbt/tests/test_dataset.py b/cosmos/providers/dbt/tests/test_dataset.py deleted file mode 100644 index b4f3055b7..000000000 --- a/cosmos/providers/dbt/tests/test_dataset.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Tests Dataset export override -""" -import sys -from unittest import mock - -import airflow -import pytest - - -@pytest.fixture(autouse=False) -def reset_dataset_import() -> None: - """ - Remove all dataset imports from sys.modules. - """ - for key in list(sys.modules.keys()): - if "dataset" in key.lower(): - del sys.modules[key] - - -def test_failed_dataset_import(reset_dataset_import: None) -> None: - """ - Test that you can still import the Dataset class even if Airflow <= 2.5 is installed. - """ - with mock.patch.dict(sys.modules, {"airflow.datasets": None}): - from cosmos.providers.dbt.dataset import Dataset - - assert Dataset.cosmos_override is True - assert Dataset.__module__ == "cosmos.providers.dbt.dataset" - - dataset = Dataset("my_dataset") - assert dataset.id == "my_dataset" - - dataset_two = Dataset("my_dataset") - assert dataset_two.id == "my_dataset" - - assert dataset == dataset_two - - -@pytest.mark.skipif( - airflow.__version__ < "2.5.0", - reason="This test only applies to Airflow 2.5.0 and above", -) -def test_successful_dataset_import() -> None: - """ - Test that you can still import the Dataset class even if Airflow <= 2.5 is installed. - """ - from cosmos.providers.dbt.dataset import Dataset - - # this should be the airflow.datasets.Dataset class - assert Dataset.__module__ == "airflow.datasets" diff --git a/pyproject.toml b/pyproject.toml index f5564afeb..3189ac3bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,7 +155,7 @@ matrix.airflow.dependencies = [ [tool.hatch.envs.tests.scripts] freeze = "pip freeze" test = 'pytest -vv --durations=0 . -m "not integration"' -test-cov = 'pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 . -m "not integration"' +test-cov = 'pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m "not integration"' test-integration = "pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m integration" [tool.pytest.ini_options] diff --git a/cosmos/tests/__init__.py b/tests/__init__.py similarity index 100% rename from cosmos/tests/__init__.py rename to tests/__init__.py diff --git a/cosmos/conftest.py b/tests/conftest.py similarity index 100% rename from cosmos/conftest.py rename to tests/conftest.py diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cosmos/core/tests/test_airflow.py b/tests/core/test_airflow.py similarity index 100% rename from cosmos/core/tests/test_airflow.py rename to tests/core/test_airflow.py diff --git a/tests/providers/__init__.py b/tests/providers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/providers/dbt/__init__.py b/tests/providers/dbt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/providers/dbt/core/__init__.py b/tests/providers/dbt/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/providers/dbt/core/operators/__init__.py b/tests/providers/dbt/core/operators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cosmos/providers/dbt/core/operators/tests/test_docker.py b/tests/providers/dbt/core/operators/test_docker.py similarity index 100% rename from cosmos/providers/dbt/core/operators/tests/test_docker.py rename to tests/providers/dbt/core/operators/test_docker.py diff --git a/cosmos/providers/dbt/core/operators/tests/test_kubernetes.py b/tests/providers/dbt/core/operators/test_kubernetes.py similarity index 100% rename from cosmos/providers/dbt/core/operators/tests/test_kubernetes.py rename to tests/providers/dbt/core/operators/test_kubernetes.py diff --git a/cosmos/providers/dbt/core/operators/tests/test_local.py b/tests/providers/dbt/core/operators/test_local.py similarity index 100% rename from cosmos/providers/dbt/core/operators/tests/test_local.py rename to tests/providers/dbt/core/operators/test_local.py diff --git a/cosmos/providers/dbt/core/operators/tests/test_virtualenv.py b/tests/providers/dbt/core/operators/test_virtualenv.py similarity index 100% rename from cosmos/providers/dbt/core/operators/tests/test_virtualenv.py rename to tests/providers/dbt/core/operators/test_virtualenv.py diff --git a/tests/providers/dbt/core/profiles/__init__.py b/tests/providers/dbt/core/profiles/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cosmos/providers/dbt/core/profiles/bigquery/tests/test_bq_service_account_file.py b/tests/providers/dbt/core/profiles/bigquery/test_bq_service_account_file.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/bigquery/tests/test_bq_service_account_file.py rename to tests/providers/dbt/core/profiles/bigquery/test_bq_service_account_file.py diff --git a/cosmos/providers/dbt/core/profiles/databricks/tests/test_dbr_token.py b/tests/providers/dbt/core/profiles/databricks/test_dbr_token.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/databricks/tests/test_dbr_token.py rename to tests/providers/dbt/core/profiles/databricks/test_dbr_token.py diff --git a/cosmos/providers/dbt/core/profiles/exasol/tests/test_exasol_user_pass.py b/tests/providers/dbt/core/profiles/exasol/test_exasol_user_pass.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/exasol/tests/test_exasol_user_pass.py rename to tests/providers/dbt/core/profiles/exasol/test_exasol_user_pass.py diff --git a/cosmos/providers/dbt/core/profiles/postgres/tests/test_pg_user_pass.py b/tests/providers/dbt/core/profiles/postgres/test_pg_user_pass.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/postgres/tests/test_pg_user_pass.py rename to tests/providers/dbt/core/profiles/postgres/test_pg_user_pass.py diff --git a/cosmos/providers/dbt/core/profiles/redshift/tests/test_redshift_user_pass.py b/tests/providers/dbt/core/profiles/redshift/test_redshift_user_pass.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/redshift/tests/test_redshift_user_pass.py rename to tests/providers/dbt/core/profiles/redshift/test_redshift_user_pass.py diff --git a/cosmos/providers/dbt/core/profiles/snowflake/tests/test_snowflake_user_pass.py b/tests/providers/dbt/core/profiles/snowflake/test_snowflake_user_pass.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/snowflake/tests/test_snowflake_user_pass.py rename to tests/providers/dbt/core/profiles/snowflake/test_snowflake_user_pass.py diff --git a/cosmos/providers/dbt/core/profiles/spark/tests/test_spark_thrift.py b/tests/providers/dbt/core/profiles/spark/tests/test_spark_thrift.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/spark/tests/test_spark_thrift.py rename to tests/providers/dbt/core/profiles/spark/tests/test_spark_thrift.py diff --git a/cosmos/providers/dbt/core/profiles/trino/tests/test_trino_base.py b/tests/providers/dbt/core/profiles/trino/test_trino_base.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/trino/tests/test_trino_base.py rename to tests/providers/dbt/core/profiles/trino/test_trino_base.py diff --git a/cosmos/providers/dbt/core/profiles/trino/tests/test_trino_certificate.py b/tests/providers/dbt/core/profiles/trino/test_trino_certificate.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/trino/tests/test_trino_certificate.py rename to tests/providers/dbt/core/profiles/trino/test_trino_certificate.py diff --git a/cosmos/providers/dbt/core/profiles/trino/tests/test_trino_jwt.py b/tests/providers/dbt/core/profiles/trino/test_trino_jwt.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/trino/tests/test_trino_jwt.py rename to tests/providers/dbt/core/profiles/trino/test_trino_jwt.py diff --git a/cosmos/providers/dbt/core/profiles/trino/tests/test_trino_ldap.py b/tests/providers/dbt/core/profiles/trino/test_trino_ldap.py similarity index 100% rename from cosmos/providers/dbt/core/profiles/trino/tests/test_trino_ldap.py rename to tests/providers/dbt/core/profiles/trino/test_trino_ldap.py diff --git a/tests/providers/dbt/core/utils/__init__.py b/tests/providers/dbt/core/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cosmos/providers/dbt/core/utils/tests/test_data_aware_scheduling.py b/tests/providers/dbt/core/utils/test_data_aware_scheduling.py similarity index 100% rename from cosmos/providers/dbt/core/utils/tests/test_data_aware_scheduling.py rename to tests/providers/dbt/core/utils/test_data_aware_scheduling.py diff --git a/cosmos/providers/dbt/core/utils/tests/test_warn_parsing.py b/tests/providers/dbt/core/utils/test_warn_parsing.py similarity index 100% rename from cosmos/providers/dbt/core/utils/tests/test_warn_parsing.py rename to tests/providers/dbt/core/utils/test_warn_parsing.py diff --git a/tests/providers/dbt/parser/__init__.py b/tests/providers/dbt/parser/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cosmos/providers/dbt/parser/tests/test_project.py b/tests/providers/dbt/parser/test_project.py similarity index 100% rename from cosmos/providers/dbt/parser/tests/test_project.py rename to tests/providers/dbt/parser/test_project.py diff --git a/tests/providers/dbt/test_dataset.py b/tests/providers/dbt/test_dataset.py new file mode 100644 index 000000000..74fc75ef6 --- /dev/null +++ b/tests/providers/dbt/test_dataset.py @@ -0,0 +1,16 @@ +""" +Tests Dataset export override +""" + +import airflow + +from cosmos.providers.dbt.dataset import Dataset + + +def test_import() -> None: + "Tests that we can import the Dataset object in all versions of Airflow." + + if airflow.__version__ >= "2.4.0": + assert Dataset.__module__ == "airflow.datasets" + else: + assert Dataset.__module__ == "cosmos.providers.dbt.dataset" diff --git a/cosmos/providers/dbt/tests/test_export.py b/tests/providers/dbt/test_export.py similarity index 100% rename from cosmos/providers/dbt/tests/test_export.py rename to tests/providers/dbt/test_export.py diff --git a/cosmos/providers/dbt/tests/test_render.py b/tests/providers/dbt/test_render.py similarity index 100% rename from cosmos/providers/dbt/tests/test_render.py rename to tests/providers/dbt/test_render.py diff --git a/cosmos/tests/test-connections.yaml b/tests/test-connections.yaml similarity index 100% rename from cosmos/tests/test-connections.yaml rename to tests/test-connections.yaml diff --git a/cosmos/tests/test_example_dags.py b/tests/test_example_dags.py similarity index 96% rename from cosmos/tests/test_example_dags.py rename to tests/test_example_dags.py index 0250ec5ff..174b940dc 100644 --- a/cosmos/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -11,7 +11,7 @@ from . import utils as test_utils -EXAMPLE_DAGS_DIR = Path(__file__).parent.parent.parent / "dev/dags" +EXAMPLE_DAGS_DIR = Path(__file__).parent.parent / "dev/dags" EXAMPLE_CONN_FILE = Path(__file__).parent / "test-connections.yaml" AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" diff --git a/cosmos/tests/test_version.py b/tests/test_version.py similarity index 100% rename from cosmos/tests/test_version.py rename to tests/test_version.py diff --git a/cosmos/tests/utils.py b/tests/utils.py similarity index 100% rename from cosmos/tests/utils.py rename to tests/utils.py