From f81abd6145d20305148e0a91e8a6ff5026927b70 Mon Sep 17 00:00:00 2001 From: rom sharon <33751805+romsharon98@users.noreply.github.com> Date: Wed, 19 Jun 2024 09:00:41 +0300 Subject: [PATCH] add git-sync-ssh secret template (#39936) --- chart/templates/_helpers.yaml | 11 +++-- .../scheduler/scheduler-deployment.yaml | 2 +- .../templates/secrets/git-ssh-key-secret.yaml | 34 +++++++++++++++ .../triggerer/triggerer-deployment.yaml | 4 +- .../webserver/webserver-deployment.yaml | 2 +- .../templates/workers/worker-deployment.yaml | 2 +- chart/values.schema.json | 8 ++++ chart/values.yaml | 6 +++ helm_tests/other/test_git_ssh_key_secret.py | 42 +++++++++++++++++++ helm_tests/other/test_git_sync_scheduler.py | 30 +++++++++++++ helm_tests/other/test_git_sync_triggerer.py | 36 ++++++++++++++++ helm_tests/other/test_git_sync_webserver.py | 20 +++++++++ helm_tests/other/test_git_sync_worker.py | 36 ++++++++++++++++ 13 files changed, 225 insertions(+), 8 deletions(-) create mode 100644 chart/templates/secrets/git-ssh-key-secret.yaml create mode 100644 helm_tests/other/test_git_ssh_key_secret.py diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index dfa23fba2f069b..641c0f77b64ef7 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -188,7 +188,7 @@ If release name contains chart name it will be used as a full name. {{- define "git_sync_ssh_key_volume" }} - name: git-sync-ssh-key secret: - secretName: {{ .Values.dags.gitSync.sshKeySecret }} + secretName: {{ template "git_sync_ssh_key" . }} defaultMode: 288 {{- end }} @@ -199,7 +199,7 @@ If release name contains chart name it will be used as a full name. imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }} securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }} env: - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} - name: GIT_SSH_KEY_FILE value: "/etc/git-secret/ssh" - name: GITSYNC_SSH_KEY_FILE @@ -294,7 +294,7 @@ If release name contains chart name it will be used as a full name. volumeMounts: - name: dags mountPath: /git - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} - name: git-sync-ssh-key mountPath: /etc/git-secret/ssh readOnly: true @@ -427,6 +427,11 @@ If release name contains chart name it will be used as a full name. {{- printf "%s/%s" .Values.kerberos.ccacheMountPath .Values.kerberos.ccacheFileName }} {{- end }} +{{/* Create the name of the git sync ssh secret to use */}} +{{- define "git_sync_ssh_key" -}} + {{- default (printf "%s-ssh-secret" (include "airflow.fullname" .)) .Values.dags.gitSync.sshKeySecret }} +{{- end }} + {{- define "celery_executor_namespace" -}} {{- if semverCompare ">=2.7.0" .Values.airflowVersion }} {{- print "airflow.providers.celery.executors.celery_executor.app" -}} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index d21f60202bdc5e..a19c6cbd93023f 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -301,7 +301,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} diff --git a/chart/templates/secrets/git-ssh-key-secret.yaml b/chart/templates/secrets/git-ssh-key-secret.yaml new file mode 100644 index 00000000000000..8df2077ed672e7 --- /dev/null +++ b/chart/templates/secrets/git-ssh-key-secret.yaml @@ -0,0 +1,34 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +{{- if and .Values.dags.gitSync.sshKey .Values.dags.gitSync.enabled}} +apiVersion: v1 +kind: Secret +metadata: + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "git_sync_ssh_key" . }} +data: + gitSshKey: {{ .Values.dags.gitSync.sshKey | b64enc | quote }} +{{- end }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 828fd0e516d9e8..bb067de196e74c 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -156,7 +156,7 @@ spec: {{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }} {{- end }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if and (or .Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} {{- end }} {{- if .Values.triggerer.extraInitContainers }} @@ -273,7 +273,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | nindent 8 }} {{- end }} {{- end }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index cd40a7ba47ad91..45095a28e65241 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -276,7 +276,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 0a2f7c6720ef63..0cf3ecb9f6e2f7 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -412,7 +412,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} - {{- if .Values.dags.gitSync.sshKeySecret }} + {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey}} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 7ee05eecfce82c..2ef829b490f7ae 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -8426,6 +8426,14 @@ ], "default": null }, + "sshKey": { + "description": "SSH private key", + "type": [ + "string", + "null" + ], + "default": null + }, "sshKeySecret": { "description": "Name of a Secret containing the repo `sshKeySecret`.", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index f8b03a5ef6428c..eb50c4087b9eac 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2609,6 +2609,12 @@ dags: # and specify the name of the secret below # sshKeySecret: airflow-ssh-secret # + # Or set sshKeySecret with your key + # sshKey: |- + # -----BEGIN {OPENSSH PRIVATE KEY}----- + # ... + # -----END {OPENSSH PRIVATE KEY}----- + # # If you are using an ssh private key, you can additionally # specify the content of your known_hosts file, example: # diff --git a/helm_tests/other/test_git_ssh_key_secret.py b/helm_tests/other/test_git_ssh_key_secret.py new file mode 100644 index 00000000000000..73f796a05c0c8b --- /dev/null +++ b/helm_tests/other/test_git_ssh_key_secret.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import jmespath + +from tests.charts.helm_template_generator import render_chart + + +class TestGitSSHKeySecret: + """Tests git-ssh secret.""" + + def test_create_git_ssh_key_secret(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "sshKey": "cm9tIGlzIHRoZSBraW5n", + }, + "persistence": {"enabled": True}, + } + }, + show_only=["templates/secrets/git-ssh-key-secret.yaml"], + ) + + assert "release-name-ssh-secret" == jmespath.search("metadata.name", docs[0]) + assert "Y205dElHbHpJSFJvWlNCcmFXNW4=" == jmespath.search("data.gitSshKey", docs[0]) diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 7bf7b83ae55724..02289bd2deffc4 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -211,6 +211,36 @@ def test_validate_if_ssh_params_are_added(self): "secret": {"secretName": "ssh-secret", "defaultMode": 288}, } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "sshKey": "dummy-ssh-key", + } + } + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + + assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert { + "name": "git-sync-ssh-key", + "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) + def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): docs = render_chart( values={ diff --git a/helm_tests/other/test_git_sync_triggerer.py b/helm_tests/other/test_git_sync_triggerer.py index 96f7092666ebe2..f24b1b7b34a91c 100644 --- a/helm_tests/other/test_git_sync_triggerer.py +++ b/helm_tests/other/test_git_sync_triggerer.py @@ -41,3 +41,39 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): show_only=["templates/triggerer/triggerer-deployment.yaml"], ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "sshKey": "dummy-ssh-key", + } + } + }, + show_only=["templates/triggerer/triggerer-deployment.yaml"], + ) + + assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert { + "name": "git-sync-ssh-key", + "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) diff --git a/helm_tests/other/test_git_sync_webserver.py b/helm_tests/other/test_git_sync_webserver.py index 01bf49a65b83b3..8ecde2348cee4c 100644 --- a/helm_tests/other/test_git_sync_webserver.py +++ b/helm_tests/other/test_git_sync_webserver.py @@ -172,3 +172,23 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): show_only=["templates/webserver/webserver-deployment.yaml"], ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): + docs = render_chart( + values={ + "airflowVersion": "1.10.14", + "dags": { + "gitSync": { + "enabled": True, + "sshKey": "dummy-ssh-key", + }, + "persistence": {"enabled": False}, + }, + }, + show_only=["templates/webserver/webserver-deployment.yaml"], + ) + + assert { + "name": "git-sync-ssh-key", + "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) diff --git a/helm_tests/other/test_git_sync_worker.py b/helm_tests/other/test_git_sync_worker.py index 31589a78f3ba77..9d0007b7594c08 100644 --- a/helm_tests/other/test_git_sync_worker.py +++ b/helm_tests/other/test_git_sync_worker.py @@ -132,3 +132,39 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_validate_if_ssh_params_are_added_with_git_ssh_key(self): + docs = render_chart( + values={ + "dags": { + "gitSync": { + "enabled": True, + "sshKey": "dummy-ssh-key", + } + } + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + assert { + "name": "git-sync-ssh-key", + "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288}, + } in jmespath.search("spec.template.spec.volumes", docs[0])