[AIRFLOW-4846] Allow kube git-sync mode to use existing secret for gi…

…t credentials (#5475)
apache · Aug 19, 2019 · 1b19b0c · 1b19b0c
1 parent 0661118
commit 1b19b0c
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 2 deletions.
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
@@ -553,8 +553,8 @@ basedn = dc=example,dc=com
 cacert = /etc/ca/ldap_ca.crt
 search_scope = LEVEL
 
-# This setting allows the use of LDAP servers that either return a 
-# broken schema, or do not return a schema. 
+# This setting allows the use of LDAP servers that either return a
+# broken schema, or do not return a schema.
 ignore_malformed_schema = False
 
 [kerberos]
@@ -688,6 +688,20 @@ git_dags_folder_mount_point =
 git_ssh_key_secret_name =
 git_ssh_known_hosts_configmap_name =
 
+# To give the git_sync init container credentials via a secret, create a secret
+# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and
+# add `git_sync_credentials_secret = <secret_name>` to your airflow config under the kubernetes section
+#
+# Secret Example:
+# apiVersion: v1
+# kind: Secret
+# metadata:
+#   name: git-credentials
+# data:
+#   GIT_SYNC_USERNAME: <base64_encoded_git_username>
+#   GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+git_sync_credentials_secret =
+
 # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
 git_sync_container_repository = k8s.gcr.io/git-sync
 git_sync_container_tag = v3.1.1

diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
@@ -189,6 +189,8 @@ def __init__(self):
         self.git_ssh_key_secret_name = conf.get(self.kubernetes_section, 'git_ssh_key_secret_name')
         self.git_ssh_known_hosts_configmap_name = conf.get(self.kubernetes_section,
                                                            'git_ssh_known_hosts_configmap_name')
+        self.git_sync_credentials_secret = conf.get(self.kubernetes_section,
+                                                    'git_sync_credentials_secret')
 
         # NOTE: The user may optionally use a volume claim to mount a PV containing
         # DAGs directly

diff --git a/airflow/kubernetes/worker_configuration.py b/airflow/kubernetes/worker_configuration.py
@@ -79,6 +79,28 @@ def _get_init_containers(self):
                 'value': self.kube_config.git_password
             })
 
+        if self.kube_config.git_sync_credentials_secret:
+            init_environment.extend([
+                {
+                    'name': 'GIT_SYNC_USERNAME',
+                    'valueFrom': {
+                        'secretKeyRef': {
+                            'name': self.kube_config.git_sync_credentials_secret,
+                            'key': 'GIT_SYNC_USERNAME'
+                        }
+                    }
+                },
+                {
+                    'name': 'GIT_SYNC_PASSWORD',
+                    'valueFrom': {
+                        'secretKeyRef': {
+                            'name': self.kube_config.git_sync_credentials_secret,
+                            'key': 'GIT_SYNC_PASSWORD'
+                        }
+                    }
+                }
+            ])
+
         volume_mounts = [{
             'mountPath': self.kube_config.git_sync_root,
             'name': self.dags_volume_name,

diff --git a/tests/executors/test_kubernetes_executor.py b/tests/executors/test_kubernetes_executor.py
@@ -376,6 +376,48 @@ def test_init_environment_using_git_sync_user_with_known_hosts(self):
                         'value': '/etc/git-secret/known_hosts'} in env)
         self.assertFalse({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
 
+    def test_make_pod_git_sync_credentials_secret(self):
+        # Tests the pod created with git_sync_credentials_secret will get into the init container
+        self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+        self.kube_config.worker_fs_group = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        kube_executor_config = KubernetesExecutorConfig(annotations=[],
+                                                        volumes=[],
+                                                        volume_mounts=[])
+
+        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
+                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
+                                     kube_executor_config)
+
+        username_env = {
+            'name': 'GIT_SYNC_USERNAME',
+            'valueFrom': {
+                'secretKeyRef': {
+                    'name': self.kube_config.git_sync_credentials_secret,
+                    'key': 'GIT_SYNC_USERNAME'
+                }
+            }
+        }
+        password_env = {
+            'name': 'GIT_SYNC_PASSWORD',
+            'valueFrom': {
+                'secretKeyRef': {
+                    'name': self.kube_config.git_sync_credentials_secret,
+                    'key': 'GIT_SYNC_PASSWORD'
+                }
+            }
+        }
+
+        self.assertIn(username_env, pod.init_containers[0]["env"],
+                      'The username env for git credentials did not get into the init container')
+
+        self.assertIn(password_env, pod.init_containers[0]["env"],
+                      'The password env for git credentials did not get into the init container')
+
     def test_init_environment_using_git_sync_run_as_user_empty(self):
         # Tests if git_syn_run_as_user is none, then no securityContext created in init container