diff --git a/airflow/configuration.py b/airflow/configuration.py index 124ef0fd54a2d..032da2e963053 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -184,6 +184,12 @@ def _get_env_var_option(self, section, key): env_var = self._env_var_name(section, key) if env_var in os.environ: return expand_env_var(os.environ[env_var]) + # alternatively AIRFLOW__{SECTION}__{KEY}_CMD (for a command) + env_var_cmd = env_var + '_CMD' + if env_var_cmd in os.environ: + # if this is a valid command key... + if (section, key) in self.as_command_stdout: + return run_command(os.environ[env_var_cmd]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' diff --git a/docs/howto/set-config.rst b/docs/howto/set-config.rst index 1311a9d6e73a0..155a10cf4de17 100644 --- a/docs/howto/set-config.rst +++ b/docs/howto/set-config.rst @@ -58,11 +58,19 @@ The following config options support this ``_cmd`` version: * ``bind_password`` in ``[ldap]`` section * ``git_password`` in ``[kubernetes]`` section +The ``_cmd`` config options can also be set using a corresponding environment variable +the same way the usual config options can. For example: + +.. code-block:: bash + + export AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD=bash_command_to_run + The idea behind this is to not store passwords on boxes in plain text files. The universal order of precedence for all configuration options is as follows: #. set as an environment variable +#. set as a command environment variable #. set in ``airflow.cfg`` #. command in ``airflow.cfg`` #. Airflow's built in defaults diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 8a199a85d67e1..10058f50d8da2 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -31,7 +31,9 @@ @unittest.mock.patch.dict('os.environ', { 'AIRFLOW__TESTSECTION__TESTKEY': 'testvalue', - 'AIRFLOW__TESTSECTION__TESTPERCENT': 'with%percent' + 'AIRFLOW__TESTSECTION__TESTPERCENT': 'with%percent', + 'AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD': 'echo -n "OK"', + 'AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD': 'echo -n "NOT OK"' }) class TestConf(unittest.TestCase): @@ -421,3 +423,21 @@ def test_deprecated_funcs(self): with mock.patch('airflow.configuration.{}'.format(func)): with self.assertWarns(DeprecationWarning): getattr(configuration, func)() + + def test_command_from_env(self): + TEST_CMDENV_CONFIG = '''[testcmdenv] +itsacommand = NOT OK +notacommand = OK +''' + test_cmdenv_conf = AirflowConfigParser() + test_cmdenv_conf.read_string(TEST_CMDENV_CONFIG) + test_cmdenv_conf.as_command_stdout.add(('testcmdenv', 'itsacommand')) + with unittest.mock.patch.dict('os.environ'): + # AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD maps to ('testcmdenv', 'itsacommand') in + # as_command_stdout and therefore should return 'OK' from the environment variable's + # echo command, and must not return 'NOT OK' from the configuration + self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'itsacommand'), 'OK') + # AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in as_command_stdout and therefore + # the option should return 'OK' from the configuration, and must not return 'NOT OK' from + # the environement variable's echo command + self.assertEqual(test_cmdenv_conf.get('testcmdenv', 'notacommand'), 'OK')