Skip to content

Commit

Permalink
feat(chart): allow valueFrom in env config of components (apache#40095)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dada-engineer
dada-engineer committed Jun 10, 2024
1 parent 27579cb commit 2055bb4
Show file tree
Hide file tree
Showing 10 changed files with 486 additions and 24 deletions.
26 changes: 26 additions & 0 deletions chart/templates/_helpers.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1015,10 +1015,36 @@ capabilities:
{{- $env := index . 1 -}}
{{- range $i, $config := $env }}
- name: {{ $config.name }}
{{- if $config.value }}
value: {{ $config.value | quote }}
{{- else if $config.valueFrom }}
valueFrom:
{{- if $config.valueFrom.secretKeyRef }}
secretKeyRef:
name: {{ $config.valueFrom.secretKeyRef.name }}
key: {{ $config.valueFrom.secretKeyRef.key }}
{{- else if $config.valueFrom.configMapKeyRef }}
configMapKeyRef:
name: {{ $config.valueFrom.configMapKeyRef.name }}
key: {{ $config.valueFrom.configMapKeyRef.key }}
{{- end }}
{{- end }}
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
{{- if $config.value }}
value: {{ $config.value | quote }}
{{- else if $config.valueFrom }}
valueFrom:
{{- if $config.valueFrom.secretKeyRef }}
secretKeyRef:
name: {{ $config.valueFrom.secretKeyRef.name }}
key: {{ $config.valueFrom.secretKeyRef.key }}
{{- else if $config.valueFrom.configMapKeyRef }}
configMapKeyRef:
name: {{ $config.valueFrom.configMapKeyRef.name }}
key: {{ $config.valueFrom.configMapKeyRef.key }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
Expand Down
320 changes: 304 additions & 16 deletions chart/values.schema.json
View file

Large diffs are not rendered by default.

20 changes: 19 additions & 1 deletion helm_tests/airflow_aux/test_create_user_job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,17 @@ def test_should_add_extraEnvs(self):
docs = render_chart(
values={
"createUserJob": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
},
},
show_only=["templates/jobs/create-user-job.yaml"],
Expand All @@ -237,6 +247,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_enable_custom_env(self):
docs = render_chart(
Expand Down
24 changes: 23 additions & 1 deletion helm_tests/airflow_aux/test_pod_template_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -767,14 +767,36 @@ def test_should_add_pod_labels(self):

def test_should_add_extraEnvs(self):
docs = render_chart(
values={"workers": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}},
values={
"workers": {
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
]
}
},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)

assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.containers[0].env", docs[0])

def test_should_add_component_specific_labels(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/airflow_core/test_dag_processor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,17 @@ def test_should_add_extraEnvs(self):
values={
"dagProcessor": {
"enabled": True,
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
},
},
show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
Expand All @@ -212,6 +222,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_extraEnvs_to_wait_for_migration_container(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/airflow_core/test_scheduler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,17 @@ def test_should_add_extraEnvs(self):
docs = render_chart(
values={
"scheduler": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
},
},
show_only=["templates/scheduler/scheduler-deployment.yaml"],
Expand All @@ -178,6 +188,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_extraEnvs_to_wait_for_migration_container(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/airflow_core/test_triggerer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,17 @@ def test_should_add_extraEnvs(self):
docs = render_chart(
values={
"triggerer": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
}
},
show_only=["templates/triggerer/triggerer-deployment.yaml"],
Expand All @@ -200,6 +210,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_extraEnvs_to_wait_for_migration_container(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/airflow_core/test_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,17 @@ def test_should_add_extraEnvs(self):
docs = render_chart(
values={
"workers": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
},
},
show_only=["templates/workers/worker-deployment.yaml"],
Expand All @@ -191,6 +201,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_extraEnvs_to_wait_for_migration_container(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/other/test_flower.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,17 @@ def test_should_add_extraEnvs(self):
values={
"flower": {
"enabled": True,
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
}
},
show_only=["templates/flower/flower-deployment.yaml"],
Expand All @@ -314,6 +324,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_component_specific_labels(self):
docs = render_chart(
Expand Down
20 changes: 19 additions & 1 deletion helm_tests/webserver/test_webserver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,17 @@ def test_should_add_extraEnvs(self):
docs = render_chart(
values={
"webserver": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
"env": [
{"name": "TEST_ENV_1", "value": "test_env_1"},
{
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
},
{
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
},
],
},
},
show_only=["templates/webserver/webserver-deployment.yaml"],
Expand All @@ -228,6 +238,14 @@ def test_should_add_extraEnvs(self):
assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search(
"spec.template.spec.containers[0].env", docs[0]
)
assert {
"name": "TEST_ENV_2",
"valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])
assert {
"name": "TEST_ENV_3",
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}},
} in jmespath.search("spec.template.spec.containers[0].env", docs[0])

def test_should_add_extra_volume_and_extra_volume_mount(self):
docs = render_chart(
Expand Down

0 comments on commit 2055bb4

Please sign in to comment.