[stable/airflow] Add a feature to run extra init scripts in the sched…

…uler
helm · Feb 26, 2020 · 8555df8 · 8555df8
1 parent e6530ae
commit 8555df8
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 1 deletion.
diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 6.1.1
+version: 6.2.0
 appVersion: 1.10.4
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

diff --git a/stable/airflow/README.md b/stable/airflow/README.md
@@ -387,6 +387,7 @@ The following table lists the configurable parameters of the Airflow chart and t
 | `airflow.podAnnotations`                 | annotations for scheduler, worker and web pods          | `{}`                      |
 | `airflow.extraInitContainers`            | additional Init Containers to run in the scheduler pods | `[]`                      |
 | `airflow.extraContainers`                | additional containers to run in the scheduler, worker & web pods | `[]`             |
+| `airflow.extraStartupScripts`            | name of a configmap with extra init scripts to run in the scheduler |               |
 | `airflow.extraVolumeMounts`              | additional volumeMounts to the main container in scheduler, worker & web pods | `[]`|
 | `airflow.extraVolumes`                   | additional volumes for the scheduler, worker & web pods | `[]`                      |
 | `airflow.initdb`                         | run `airflow initdb` when starting the scheduler        | `true`                    |

diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml
@@ -174,6 +174,10 @@ spec:
               subPath: {{ .subPath }}
               {{ end }}
           {{- end }}
+          {{- if .Values.airflow.extraStartupScripts }}
+            - name: extra-startup-scripts
+              mountPath: /usr/local/extra-startup-scripts/
+          {{- end}}
 {{- if .Values.airflow.extraVolumeMounts }}
 {{ toYaml .Values.airflow.extraVolumeMounts | indent 12 }}
 {{- end }}
@@ -203,6 +207,10 @@ spec:
             {{- if .Values.airflow.pools }}
               echo "adding pools" &&
               airflow pool -i /usr/local/variables-pools/pools.json &&
+            {{- end }}
+            {{- if .Values.airflow.extraStartupScripts }}
+              echo "running extra startup scripts" &&
+              /usr/local/extra-startup-scripts/run.sh &&
             {{- end }}
               echo "executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
@@ -227,6 +235,10 @@ spec:
             {{- if .Values.airflow.pools }}
               echo "adding pools" &&
               airflow pool -i /usr/local/variables-pools/pools.json &&
+            {{- end }}
+            {{- if .Values.airflow.extraStartupScripts }}
+              echo "running extra startup scripts" &&
+              /usr/local/extra-startup-scripts/run.sh &&
             {{- end }}
               echo "executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
@@ -280,6 +292,12 @@ spec:
           configMap:
             name: {{ .configMap }}
         {{- end }}
+        {{- if .Values.airflow.extraStartupScripts }}
+        - name: extra-startup-scripts
+          configMap:
+            name: {{ .Values.airflow.extraStartupScripts }}
+            defaultMode: 0740
+        {{- end }}
 {{- if .Values.airflow.extraVolumes }}
 {{ toYaml .Values.airflow.extraVolumes | indent 8 }}
 {{- end }}
diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml
@@ -174,6 +174,18 @@ airflow:
   ## Run initdb when the scheduler starts.
   initdb: true
 
+  # If set, must be the name of a configmap that contains at least one entry, which key must be named 'run.sh',
+  # and corresponds to a shell script that will be executed in the scheduler container after it has finished
+  # initdb and other standard startup scripts (just before it starts).
+  # Other entries provided in the same configmap will also be mounted in the same directory (using the key name
+  # as their filename). This allows you to split your initialization tasks into several scripts that you can call from
+  # the main 'run.sh' script, for example.
+  # Notes:
+  # - the behaviour of those extra scripts should be idempotent, and fast enough, as they will be executed every time
+  #   the scheduler restarts
+  # - if those scripts use sensitive data, you should provide the sensitive data using secrets
+  #   (see .Values.airflow.extraConfigmapMounts and .Values.airflow.extraEnv)
+  extraStartupScripts:
 
 scheduler:
   resources: {}