Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support mysql backend as keda trigger #36167

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion chart/templates/_helpers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ If release name contains chart name it will be used as a full name.
name: {{ template "airflow_metadata_secret" . }}
key: connection
{{- end }}
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
{{- if and .Values.workers.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer))) }}
- name: KEDA_DB_CONN
valueFrom:
secretKeyRef:
Expand Down
4 changes: 4 additions & 0 deletions chart/templates/secrets/metadata-connection-secret.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,9 @@ data:
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }}
{{- end }}
{{- else if and .Values.workers.keda.enabled (eq .Values.data.metadataConnection.protocol "mysql") }}
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}
11 changes: 10 additions & 1 deletion chart/templates/workers/worker-kedaautoscaler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,14 @@ spec:
advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }}
{{- end }}
triggers:
- type: postgresql
{{- if eq .Values.data.metadataConnection.protocol "mysql" }}
- type: "mysql"
metadata:
queryValue: "1"
connectionStringFromEnv: KEDA_DB_CONN
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- else }}
- type: "postgresql"
metadata:
targetQueryValue: "1"
{{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
Expand All @@ -56,4 +63,6 @@ spec:
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
{{- end }}
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- end }}

{{- end }}
36 changes: 36 additions & 0 deletions helm_tests/other/test_keda.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,3 +275,39 @@ def test_default_keda_db_connection_pgbouncer_enabled_usePgbouncer_false(self):
"spec.triggers[0].metadata.connectionFromEnv", keda_autoscaler
)
assert autoscaler_connection_env_var == "KEDA_DB_CONN"

def test_mysql_keda_db_connection(self):
"""Verify keda db connection when pgbouncer is enabled."""
docs = render_chart(
values={
"data": {"metadataConnection": {"protocol": "mysql", "port": 3306}},
"workers": {"keda": {"enabled": True}},
"executor": "CeleryExecutor",
},
show_only=[
"templates/workers/worker-deployment.yaml",
"templates/workers/worker-kedaautoscaler.yaml",
"templates/secrets/metadata-connection-secret.yaml",
],
)
worker_deployment = docs[0]
keda_autoscaler = docs[1]
metadata_connection_secret = docs[2]

worker_container_env_vars = jmespath.search(
"spec.template.spec.containers[?name=='worker'].env[].name", worker_deployment
)
assert "AIRFLOW_CONN_AIRFLOW_DB" in worker_container_env_vars
assert "KEDA_DB_CONN" in worker_container_env_vars

keda_autoscaler_metadata = jmespath.search("spec.triggers[0].metadata", keda_autoscaler)
assert "queryValue" in keda_autoscaler_metadata

secret_data = jmespath.search("data", metadata_connection_secret)
assert "connection" in secret_data.keys()
assert "kedaConnection" in secret_data.keys()

autoscaler_connection_env_var = jmespath.search(
"spec.triggers[0].metadata.connectionStringFromEnv", keda_autoscaler
)
assert autoscaler_connection_env_var == "KEDA_DB_CONN"