[#3940] Use flags.PROFILES_DIR in a few more places

CHANGELOG.md

@@ -1,13 +1,15 @@
 ## dbt 1.0.0 (Release TBD)
 
 ### Features
+- Normalize global CLI arguments/flags ([#2990](https://github.com/dbt-labs/dbt/issues/2990), [#3839](https://github.com/dbt-labs/dbt/pull/3839))
 
 ### Fixes
 
 ### Under the hood
 
 - Enact deprecation for `materialization-return` and replace deprecation warning with an exception. ([#3896](https://github.com/dbt-labs/dbt/issues/3896))
 
+
 ## dbt 0.21.0 (Release TBD)
 
 ## dbt 0.21.0rc1 (September 20, 2021)
@@ -26,7 +28,6 @@
 - Added timing and thread information to sources.json artifact ([#3804](https://github.com/dbt-labs/dbt/issues/3804), [#3894](https://github.com/dbt-labs/dbt/pull/3894))
 - Update cli and rpc flags for the `build` task to align with other commands (`--resource-type`, `--store-failures`) ([#3596](https://github.com/dbt-labs/dbt/issues/3596), [#3884](https://github.com/dbt-labs/dbt/pull/3884))
 - Log tests that are not indirectly selected. Add `--greedy` flag to `test`, `list`, `build` and `greedy` property in yaml selectors ([#3723](https://github.com/dbt-labs/dbt/pull/3723), [#3833](https://github.com/dbt-labs/dbt/pull/3833))
-- Normalize global CLI arguments/flags ([#2990](https://github.com/dbt-labs/dbt/issues/2990), [#3839](https://github.com/dbt-labs/dbt/pull/3839))
 
 ### Fixes
 

core/dbt/config/profile.py

@@ -4,6 +4,7 @@
 
 from dbt.dataclass_schema import ValidationError
 
+from dbt import flags
 from dbt.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import Credentials, HasCredentials
@@ -433,7 +434,7 @@ def render_from_args(
         """
         threads_override = getattr(args, 'threads', None)
         target_override = getattr(args, 'target', None)
-        raw_profiles = read_profile(args.profiles_dir)
+        raw_profiles = read_profile(flags.PROFILES_DIR)
         profile_name = cls.pick_profile_name(getattr(args, 'profile', None),
                                              project_profile_name)
         return cls.from_raw_profiles(

core/dbt/flags.py

@@ -7,6 +7,8 @@
 from typing import Optional
 
 # PROFILES_DIR must be set before the other flags
+# It also gets set in main.py and in set_from_args because the rpc server
+# doesn't go through exactly the same main arg processing.
 DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt')
 PROFILES_DIR = os.path.expanduser(
     os.getenv('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR)
@@ -107,6 +109,7 @@ def set_from_args(args, user_config):
     WRITE_JSON = get_flag_value('WRITE_JSON', args, user_config)
     PARTIAL_PARSE = get_flag_value('PARTIAL_PARSE', args, user_config)
     USE_COLORS = get_flag_value('USE_COLORS', args, user_config)
+    PROFILES_DIR = get_flag_value('PROFILES_DIR', args, user_config)
     DEBUG = get_flag_value('DEBUG', args, user_config)
     LOG_FORMAT = get_flag_value('LOG_FORMAT', args, user_config)
     VERSION_CHECK = get_flag_value('VERSION_CHECK', args, user_config)
@@ -125,7 +128,7 @@ def get_flag_value(flag, args, user_config):
         if env_value is not None and env_value != '':
             env_value = env_value.lower()
             # non Boolean values
-            if flag in ['LOG_FORMAT', 'PRINTER_WIDTH']:
+            if flag in ['LOG_FORMAT', 'PRINTER_WIDTH', 'PROFILES_DIR']:
                 flag_value = env_value
             else:
                 flag_value = env_set_bool(env_value)
@@ -135,6 +138,8 @@ def get_flag_value(flag, args, user_config):
             flag_value = flag_defaults[flag]
     if flag == 'PRINTER_WIDTH':  # printer_width must be an int or it hangs
         flag_value = int(flag_value)
+    if flag == 'PROFILES_DIR':
+        flag_value = os.path.abspath(flag_value)
 
     return flag_value
 

core/dbt/main.py

@@ -174,7 +174,7 @@ def handle_and_check(args):
         parsed = parse_args(args)
 
         # Set flags from args, user config, and env vars
-        user_config = read_user_config(parsed.profiles_dir)  # This is read again later
+        user_config = read_user_config(flags.PROFILES_DIR)  # This is read again later
         flags.set_from_args(parsed, user_config)
         dbt.tracking.initialize_from_flags()
         # Set log_format from flags
@@ -1160,11 +1160,14 @@ def parse_args(args, cls=DBTArgumentParser):
         if parsed.sub_profiles_dir is not None:
             parsed.profiles_dir = parsed.sub_profiles_dir
         delattr(parsed, 'sub_profiles_dir')
-    if hasattr(parsed, 'profiles_dir') and parsed.profiles_dir is not None:
-        parsed.profiles_dir = os.path.abspath(parsed.profiles_dir)
-        # needs to be set before the other flags, because it's needed to
-        # read the profile that contains them
-        flags.PROFILES_DIR = parsed.profiles_dir
+    if hasattr(parsed, 'profiles_dir'):
+        if parsed.profiles_dir is None:
+            parsed.profiles_dir = flags.PROFILES_DIR
+        else:
+            parsed.profiles_dir = os.path.abspath(parsed.profiles_dir)
+            # needs to be set before the other flags, because it's needed to
+            # read the profile that contains them
+            flags.PROFILES_DIR = parsed.profiles_dir
 
     # version_check is set before subcommands and after, so normalize
     if hasattr(parsed, 'sub_version_check'):

core/dbt/parser/manifest.py

@@ -607,7 +607,7 @@ def build_manifest_state_check(self):
             ])
         )
 
-        profile_path = os.path.join(config.args.profiles_dir, 'profiles.yml')
+        profile_path = os.path.join(flags.PROFILES_DIR, 'profiles.yml')
         with open(profile_path) as fp:
             profile_hash = FileHash.from_contents(fp.read())
 

core/dbt/task/base.py

@@ -93,7 +93,7 @@ def from_args(cls, args):
             logger.error("Encountered an error while reading profiles:")
             logger.error("  ERROR {}".format(str(exc)))
 
-            all_profiles = read_profiles(args.profiles_dir).keys()
+            all_profiles = read_profiles(flags.PROFILES_DIR).keys()
 
             if len(all_profiles) > 0:
                 logger.info("Defined profiles:")

test/integration/100_rpc_test/test_rpc.py

@@ -27,6 +27,7 @@ def __init__(self, port, profiles_dir, cli_vars=None):
             '--port', str(self.port),
             '--profiles-dir', profiles_dir
         ]
+        dbt.flags.PROFILES_DIR = profiles_dir
         if cli_vars:
             handle_and_check_args.extend(['--vars', cli_vars])
         super().__init__(

test/integration/base.py

@@ -438,6 +438,7 @@ def use_profile(self, adapter_type):
         if not os.path.exists(self.test_root_dir):
             os.makedirs(self.test_root_dir)
 
+        flags.PROFILES_DIR = self.test_root_dir
         profiles_path = os.path.join(self.test_root_dir, 'profiles.yml')
         with open(profiles_path, 'w') as f:
             yaml.safe_dump(profile_config, f, default_flow_style=True)

test/rpc/conftest.py

@@ -7,6 +7,7 @@
 
 import yaml
 
+from dbt import flags
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -145,6 +146,7 @@ def dbt_profile_data(unique_schema, pytestconfig):
 
 @pytest.fixture
 def dbt_profile(profiles_root, dbt_profile_data) -> Dict[str, Any]:
+    flags.PROFILES_DIR = profiles_root
     path = os.path.join(profiles_root, 'profiles.yml')
     with open(path, 'w') as fp:
         fp.write(yaml.safe_dump(dbt_profile_data))

test/rpc/util.py

@@ -47,6 +47,7 @@ def __init__(
             '--port', str(self.port),
             '--profiles-dir', profiles_dir
         ]
+        dbt.flags.PROFILES_DIR = profiles_dir
         if cli_vars:
             handle_and_check_args.extend(['--vars', cli_vars])
         if target is not None:

test/unit/test_config.py

@@ -12,6 +12,7 @@
 
 import dbt.config
 import dbt.exceptions
+from dbt import flags
 from dbt.adapters.factory import load_plugin
 from dbt.adapters.postgres import PostgresCredentials
 from dbt.adapters.redshift import RedshiftCredentials
@@ -93,6 +94,7 @@ def __init__(self, profiles_dir=None, threads=None, profile=None,
             self.threads = threads
         if profiles_dir is not None:
             self.profiles_dir = profiles_dir
+            flags.PROFILES_DIR = profiles_dir
         if cli_vars is not None:
             self.vars = cli_vars
         if version_check is not None: