From e25b613c1a1d7360d73d94562d8a71ea6eb43d20 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 24 Aug 2021 11:30:58 -0400 Subject: [PATCH] [#2990] Normalize global CLI args/flags automatic commit by git-black, original commits: 779c789a649f8a91a52d396de1837729c23bb23e --- core/dbt/config/profile.py | 16 ++++---- core/dbt/contracts/project.py | 2 +- core/dbt/flags.py | 34 ++++++++--------- core/dbt/main.py | 72 ++++++++++++++++++----------------- core/dbt/task/base.py | 2 +- 5 files changed, 65 insertions(+), 61 deletions(-) diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index 65704db6b40..3c1f7d49f20 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -43,7 +43,9 @@ defined in your profiles.yml file. You can find profiles.yml here: {profiles_file}/profiles.yml -""".format(profiles_file=DEFAULT_PROFILES_DIR) +""".format( + profiles_file=DEFAULT_PROFILES_DIR +) def read_profile(profiles_dir: str) -> Dict[str, Any]: @@ -73,7 +75,7 @@ def read_user_config(directory: str) -> UserConfig: try: profile = read_profile(directory) if profile: - user_config = coerce_dict_str(profile.get('config', {})) + user_config = coerce_dict_str(profile.get("config", {})) if user_config is not None: UserConfig.validate(user_config) return UserConfig.from_dict(user_config) @@ -126,12 +128,12 @@ def to_profile_info( result = { "profile_name": self.profile_name, "target_name": self.target_name, - 'user_config': self.user_config, + "user_config": self.user_config, "threads": self.threads, "credentials": self.credentials, } if serialize_credentials: - result['user_config'] = self.user_config.to_dict(omit_none=True) + result["user_config"] = self.user_config.to_dict(omit_none=True) result['credentials'] = self.credentials.to_dict(omit_none=True) return result @@ -242,7 +244,7 @@ def from_credentials( threads: int, profile_name: str, target_name: str, - user_config: Optional[Dict[str, Any]] = None + user_config: Optional[Dict[str, Any]] = None, ) -> 'Profile': """Create a profile from an existing set of Credentials and the remaining information. @@ -336,7 +338,7 @@ def from_raw_profile_info( """ # user_config is not rendered. if user_config is None: - user_config = raw_profile.get('config') + user_config = raw_profile.get("config") # TODO: should it be, and the values coerced to bool? target_name, profile_data = cls.render_profile( raw_profile, profile_name, target_override, renderer @@ -357,7 +359,7 @@ def from_raw_profile_info( profile_name=profile_name, target_name=target_name, threads=threads, - user_config=user_config + user_config=user_config, ) @classmethod diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 8e475114777..b6b00cb542b 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -261,7 +261,7 @@ class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract): class ProfileConfig(HyphenatedDbtClassMixin, Replaceable): profile_name: str = field(metadata={"preserve_underscore": True}) target_name: str = field(metadata={"preserve_underscore": True}) - user_config: UserConfig = field(metadata={'preserve_underscore': True}) + user_config: UserConfig = field(metadata={"preserve_underscore": True}) threads: int # TODO: make this a dynamic union of some kind? credentials: Optional[Dict[str, Any]] diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 540d0ce66d2..83bb5756c28 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -9,10 +9,8 @@ # PROFILES_DIR must be set before the other flags # It also gets set in main.py and in set_from_args because the rpc server # doesn't go through exactly the same main arg processing. -DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt') -PROFILES_DIR = os.path.expanduser( - os.getenv('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR) -) +DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser("~"), ".dbt") +PROFILES_DIR = os.path.expanduser(os.getenv("DBT_PROFILES_DIR", DEFAULT_PROFILES_DIR)) STRICT_MODE = False # Only here for backwards compatibility FULL_REFRESH = False # subcommand @@ -70,7 +68,7 @@ def env_set_truthy(key: str) -> Optional[str]: def env_set_bool(env_value): - if env_value in ('1', 't', 'true', 'y', 'yes'): + if env_value in ("1", "t", "true", "y", "yes"): return True return False @@ -99,7 +97,7 @@ def _get_context(): def set_from_args(args, user_config): - global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \ + # N.B. Multiple `globals` are purely for line length. USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \ USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, INDIRECT_SELECTION, \ VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \ @@ -112,19 +110,19 @@ def set_from_args(args, user_config): WHICH = getattr(args, 'which', WHICH) # global cli flags with env var and user_config alternatives - USE_EXPERIMENTAL_PARSER = get_flag_value('USE_EXPERIMENTAL_PARSER', args, user_config) + USE_EXPERIMENTAL_PARSER = get_flag_value("USE_EXPERIMENTAL_PARSER", args, user_config) STATIC_PARSER = get_flag_value('STATIC_PARSER', args, user_config) - WARN_ERROR = get_flag_value('WARN_ERROR', args, user_config) - WRITE_JSON = get_flag_value('WRITE_JSON', args, user_config) - PARTIAL_PARSE = get_flag_value('PARTIAL_PARSE', args, user_config) - USE_COLORS = get_flag_value('USE_COLORS', args, user_config) + WARN_ERROR = get_flag_value("WARN_ERROR", args, user_config) + WRITE_JSON = get_flag_value("WRITE_JSON", args, user_config) + PARTIAL_PARSE = get_flag_value("PARTIAL_PARSE", args, user_config) + USE_COLORS = get_flag_value("USE_COLORS", args, user_config) PROFILES_DIR = get_flag_value('PROFILES_DIR', args, user_config) - DEBUG = get_flag_value('DEBUG', args, user_config) - LOG_FORMAT = get_flag_value('LOG_FORMAT', args, user_config) - VERSION_CHECK = get_flag_value('VERSION_CHECK', args, user_config) - FAIL_FAST = get_flag_value('FAIL_FAST', args, user_config) - SEND_ANONYMOUS_USAGE_STATS = get_flag_value('SEND_ANONYMOUS_USAGE_STATS', args, user_config) - PRINTER_WIDTH = get_flag_value('PRINTER_WIDTH', args, user_config) + DEBUG = get_flag_value("DEBUG", args, user_config) + LOG_FORMAT = get_flag_value("LOG_FORMAT", args, user_config) + VERSION_CHECK = get_flag_value("VERSION_CHECK", args, user_config) + FAIL_FAST = get_flag_value("FAIL_FAST", args, user_config) + SEND_ANONYMOUS_USAGE_STATS = get_flag_value("SEND_ANONYMOUS_USAGE_STATS", args, user_config) + PRINTER_WIDTH = get_flag_value("PRINTER_WIDTH", args, user_config) INDIRECT_SELECTION = get_flag_value('INDIRECT_SELECTION', args, user_config) LOG_CACHE_EVENTS = get_flag_value('LOG_CACHE_EVENTS', args, user_config) EVENT_BUFFER_SIZE = get_flag_value('EVENT_BUFFER_SIZE', args, user_config) @@ -137,7 +135,7 @@ def get_flag_value(flag, args, user_config): # Environment variables use pattern 'DBT_{flag name}' env_flag = f"DBT_{flag}" env_value = os.getenv(env_flag) - if env_value is not None and env_value != '': + if env_value is not None and env_value != "": env_value = env_value.lower() # non Boolean values if flag in [ diff --git a/core/dbt/main.py b/core/dbt/main.py index 0bb289a1dfa..37b38bf2588 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -263,11 +263,13 @@ def _build_base_subparser(): base_subparser.add_argument( '--profiles-dir', default=None, - dest='sub_profiles_dir', # Main cli arg precedes subcommand + dest="sub_profiles_dir", # Main cli arg precedes subcommand type=str, help=''' Which directory to look in for the profiles.yml file. Default = {} - '''.format(DEFAULT_PROFILES_DIR) + """.format( + DEFAULT_PROFILES_DIR + ), ) base_subparser.add_argument( @@ -375,7 +377,7 @@ def _build_build_subparser(subparsers, base_subparser): sub.add_argument( '-x', '--fail-fast', - dest='sub_fail_fast', + dest="sub_fail_fast", action='store_true', help=''' Stop execution upon a first failure. @@ -516,7 +518,7 @@ def _build_run_subparser(subparsers, base_subparser): run_sub.add_argument( '-x', '--fail-fast', - dest='sub_fail_fast', + dest="sub_fail_fast", action='store_true', help=''' Stop execution upon a first failure. @@ -640,7 +642,7 @@ def _add_table_mutability_arguments(*subparsers): def _add_version_check(sub): sub.add_argument( '--no-version-check', - dest='sub_version_check', # main cli arg precedes subcommands + dest="sub_version_check", # main cli arg precedes subcommands action='store_false', default=None, help=''' @@ -722,7 +724,7 @@ def _build_test_subparser(subparsers, base_subparser): sub.add_argument( '-x', '--fail-fast', - dest='sub_fail_fast', + dest="sub_fail_fast", action='store_true', help=''' Stop execution upon a first test failure. @@ -970,8 +972,8 @@ def parse_args(args, cls=DBTArgumentParser): ) p.add_argument( - '--printer-width', - dest='printer_width', + "--printer-width", + dest="printer_width", help=''' Sets the width of terminal output ''' @@ -990,14 +992,14 @@ def parse_args(args, cls=DBTArgumentParser): ) p.add_argument( - '--no-version-check', - dest='version_check', - action='store_false', + "--no-version-check", + dest="version_check", + action="store_false", default=None, - help=''' + help=""" If set, skip ensuring dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version') - ''' + """, ) p.add_optional_argument_inverse( @@ -1046,34 +1048,36 @@ def parse_args(args, cls=DBTArgumentParser): ) p.add_argument( - '--profiles-dir', + "--profiles-dir", default=None, - dest='profiles_dir', + dest="profiles_dir", type=str, - help=''' + help=""" Which directory to look in for the profiles.yml file. Default = {} - '''.format(DEFAULT_PROFILES_DIR) + """.format( + DEFAULT_PROFILES_DIR + ), ) p.add_argument( - '--no-anonymous-usage-stats', - action='store_false', + "--no-anonymous-usage-stats", + action="store_false", default=None, - dest='send_anonymous_usage_stats', - help=''' + dest="send_anonymous_usage_stats", + help=""" Do not send anonymous usage stat to dbt Labs - ''' + """, ) p.add_argument( - '-x', - '--fail-fast', - dest='fail_fast', - action='store_true', + "-x", + "--fail-fast", + dest="fail_fast", + action="store_true", default=None, - help=''' + help=""" Stop execution upon a first failure. - ''' + """, ) p.add_argument( @@ -1131,10 +1135,10 @@ def parse_args(args, cls=DBTArgumentParser): parsed = p.parse_args(args) # profiles_dir is set before subcommands and after, so normalize - if hasattr(parsed, 'sub_profiles_dir'): + if hasattr(parsed, "sub_profiles_dir"): if parsed.sub_profiles_dir is not None: parsed.profiles_dir = parsed.sub_profiles_dir - delattr(parsed, 'sub_profiles_dir') + delattr(parsed, "sub_profiles_dir") if hasattr(parsed, 'profiles_dir'): if parsed.profiles_dir is None: parsed.profiles_dir = flags.PROFILES_DIR @@ -1145,16 +1149,16 @@ def parse_args(args, cls=DBTArgumentParser): flags.PROFILES_DIR = parsed.profiles_dir # version_check is set before subcommands and after, so normalize - if hasattr(parsed, 'sub_version_check'): + if hasattr(parsed, "sub_version_check"): if parsed.sub_version_check is False: parsed.version_check = False - delattr(parsed, 'sub_version_check') + delattr(parsed, "sub_version_check") # fail_fast is set before subcommands and after, so normalize - if hasattr(parsed, 'sub_fail_fast'): + if hasattr(parsed, "sub_fail_fast"): if parsed.sub_fail_fast is True: parsed.fail_fast = True - delattr(parsed, 'sub_fail_fast') + delattr(parsed, "sub_fail_fast") if getattr(parsed, 'project_dir', None) is not None: expanded_user = os.path.expanduser(parsed.project_dir) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index ac937b3879f..4de95403d8f 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -76,7 +76,7 @@ def pre_init_hook(cls, args): @classmethod def set_log_format(cls): - if flags.LOG_FORMAT == 'json': + if flags.LOG_FORMAT == "json": log_manager.format_json() # we're mutating the initialized, but not-yet-configured event logger # because it's being configured too late -- bad! TODO refactor!