Create a new documentation package for Helm Chart

chart/values.schema.json

@@ -280,37 +280,37 @@
                     }
                 },
                 "pgbouncer": {
-                    "description": "Configuration of the pgbouncer image.",
+                    "description": "Configuration of the PgBouncer image.",
                     "type": "object",
                     "properties": {
                         "repository": {
-                            "description": "The pgbouncer image repository.",
+                            "description": "The PgBouncer image repository.",
                             "type": "string"
                         },
                         "tag": {
-                            "description": "The pgbouncer image tag.",
+                            "description": "The PgBouncer image tag.",
                             "type": "string"
                         },
                         "pullPolicy": {
-                            "description": "The pgbouncer image pull policy.",
+                            "description": "The PgBouncer image pull policy.",
                             "type": "string"
                         }
                     }
                 },
                 "pgbouncerExporter": {
-                    "description": "Configuration of the pgbouncerExporter image.",
+                    "description": "Configuration of the PgBouncer exporter image.",
                     "type": "object",
                     "properties": {
                         "repository": {
-                            "description": "The pgbouncerExporter image repository.",
+                            "description": "The PgBouncer exporter image repository.",
                             "type": "string"
                         },
                         "tag": {
-                            "description": "The pgbouncerExporter image tag.",
+                            "description": "The PgBouncer exporter image tag.",
                             "type": "string"
                         },
                         "pullPolicy": {
-                            "description": "The pgbouncerExporter image pull policy.",
+                            "description": "The PgBouncer exporter image pull policy.",
                             "type": "string"
                         }
                     }

docs/conf.py

@@ -145,6 +145,9 @@
             'providers_packages_ref',
         ]
     )
+elif PACKAGE_NAME == "helm-chart":
+    # No extra extensions
+    pass
 else:
     extensions.append('autoapi.extension')
 # List of patterns, relative to source directory, that match files and

docs/exts/docs_build/dev_index_template.html.jinja2

@@ -67,6 +67,15 @@
       </ul>
     </div>
   </div>
+  <div class="row">
+    <div class="col-md-12">
+      <h2><a href="/docs/helm-chart/index.html">Helm Chart</a></h2>
+      <p>
+        It will help you set up your own Airflow on a cloud/on-prem k8s environment and leverage its scalable nature to support a large group of users. Thanks to Kubernetes, we are not tied to a specific cloud provider.
+      </p>
+    </div>
+  </div>
+
 </div>
 </body>
 </html>

docs/exts/docs_build/docs_builder.py

@@ -241,4 +241,4 @@ def get_available_providers_packages():
 def get_available_packages():
     """Get list of all available packages to build."""
     provider_package_names = get_available_providers_packages()
-    return ["apache-airflow", *provider_package_names, "apache-airflow-providers"]
+    return ["apache-airflow", *provider_package_names, "apache-airflow-providers", "helm-chart"]

docs/helm-chart/airflow-configuration.rst

@@ -0,0 +1,64 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Configuring Airflow
+-------------------
+
+All Airflow configuration parameters (equivalent of ``airflow.cfg``) are
+stored in
+`values.yaml <https://github.com/apache/airflow/blob/master/chart/values.yaml>`__
+under the ``config`` key . The following code demonstrates how one would
+allow webserver users to view the config from within the webserver
+application. See the bottom line of the example:
+
+.. code-block:: yaml
+
+   # Config settings to go into the mounted airflow.cfg
+   #
+   # Please note that these values are passed through the ``tpl`` function, so are
+   # all subject to being rendered as go templates. If you need to include a
+   # literal ``{{`` in a value, it must be expressed like this:
+   #
+   #    a: '{{ "{{ not a template }}" }}'
+   #
+   # yamllint disable rule:line-length
+   config:
+     core:
+       dags_folder: '{{ include "airflow_dags" . }}'
+       load_examples: 'False'
+       executor: '{{ .Values.executor }}'
+       # For Airflow 1.10, backward compatibility
+       colored_console_log: 'False'
+       remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
+     # Authentication backend used for the experimental API
+     api:
+       auth_backend: airflow.api.auth.backend.deny_all
+     logging:
+       remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
+       colored_console_log: 'False'
+       logging_level: DEBUG
+     metrics:
+       statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
+       statsd_port: 9125
+       statsd_prefix: airflow
+       statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
+     webserver:
+       enable_proxy_fix: 'True'
+       expose_config: 'True'   # <<<<<<<<<< BY DEFAULT THIS IS 'False' BUT WE CHANGE IT TO 'True' PRIOR TO INSTALLING THE CHART
+
+Generally speaking, it is useful to familiarize oneself with the Airflow
+configuration prior to installing and deploying the service.

docs/helm-chart/external-redis.rst

@@ -0,0 +1,33 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+External Redis
+--------------
+
+When using the ``CeleryExecutor`` or the ``CeleryKubernetesExecutor``
+the chart will by default create a redis Deployment/StatefulSet
+alongside airflow. You can also use “your own” redis instance by
+providing the ``data.brokerUrl`` (or ``data.borkerUrlSecretName``) value
+directly:
+
+.. code-block:: bash
+
+   helm install airflow . \
+       --namespace airflow \
+       --set executor=CeleryExecutor \
+       --set redis.enabled=false \
+       --set data.brokerUrl=redis://redis-user:password@redis-host:6379/0

docs/helm-chart/index.rst

@@ -0,0 +1,83 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Helm Chart for Apache Airflow
+=============================
+
+.. toctree::
+    :hidden:
+
+    Home <self>
+    quick-start
+    airflow-configuration
+    manage-dags-files
+    keda
+    external-redis
+
+.. toctree::
+    :hidden:
+    :caption: References
+
+    Parameters <parameters-ref>
+
+
+This chart will bootstrap an `Airflow <https://airflow.apache.org>`__
+deployment on a `Kubernetes <http://kubernetes.io>`__ cluster using the
+`Helm <https://helm.sh>`__ package manager.
+
+Prerequisites
+-------------
+
+-  Kubernetes 1.14+ cluster
+-  Helm 2.11+ or Helm 3.0+
+-  PV provisioner support in the underlying infrastructure
+
+Installing the Chart
+--------------------
+
+To install this repository from source (using helm 3)
+
+.. code-block:: bash
+
+    kubectl create namespace airflow
+    helm dep update
+    helm install airflow . --namespace airflow
+
+The command deploys Airflow on the Kubernetes cluster in the default configuration. The :doc:`parameters-ref`
+section lists the parameters that can be configured during installation.
+
+> **Tip**: List all releases using ``helm list``.
+
+Upgrading the Chart
+-------------------
+
+To upgrade the chart with the release name ``airflow``:
+
+.. code-block:: bash
+
+    helm upgrade airflow . --namespace airflow
+
+Uninstalling the Chart
+----------------------
+
+To uninstall/delete the ``airflow`` deployment:
+
+.. code-block:: bash
+
+    helm delete airflow --namespace airflow
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.

docs/helm-chart/keda.rst

@@ -0,0 +1,72 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Autoscaling with KEDA
+---------------------
+
+*This feature is still experimental.*
+
+KEDA stands for Kubernetes Event Driven Autoscaling.
+`KEDA <https://github.com/kedacore/keda>`__ is a custom controller that
+allows users to create custom bindings to the Kubernetes `Horizontal Pod
+Autoscaler <https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/>`__.
+We have built a scaler that allows users to create scalers based on
+PostgreSQL queries and shared it with the community. This enables us to
+scale the number of airflow workers deployed on Kubernetes by this chart
+depending on the number of task that are ``queued`` or ``running``.
+
+.. code-block:: bash
+
+   helm repo add kedacore https://kedacore.github.io/charts
+
+   helm repo update
+
+   helm install \
+       --set image.keda=docker.io/kedacore/keda:1.2.0 \
+       --set image.metricsAdapter=docker.io/kedacore/keda-metrics-adapter:1.2.0 \
+       --namespace keda --name keda kedacore/keda
+
+Once KEDA is installed (which should be pretty quick since there is only
+one pod). You can try out KEDA autoscaling on this chart by setting
+``workers.keda.enabled=true`` your helm command or in the
+``values.yaml``. (Note: KEDA does not support StatefulSets so you need
+to set ``worker.persistence.enabled`` to ``false``)
+
+.. code-block:: bash
+
+   kubectl create namespace airflow
+
+   helm install airflow . \
+       --namespace airflow \
+       --set executor=CeleryExecutor \
+       --set workers.keda.enabled=true \
+       --set workers.persistence.enabled=false
+
+KEDA will derive the desired number of celery workers by querying
+Airflow metadata database:
+
+.. code-block:: none
+
+   SELECT
+       ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }})
+   FROM task_instance
+   WHERE state='running' OR state='queued'
+
+You should set celery worker concurrency through the helm value
+``config.celery.worker_concurrency`` (i.e. instead of airflow.cfg or
+environment variables) so that the KEDA trigger will be consistent with
+the worker concurrency setting.