Add Local and Sequential Executors to Doc (apache#8084)

(cherry picked from commit ee12529)
cfei18 · Mar 5, 2021 · 8f67d8c · 8f67d8c
1 parent a318ae2
commit 8f67d8c
Show file tree

Hide file tree

Showing 13 changed files with 126 additions and 32 deletions.
diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py
@@ -16,7 +16,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Celery executor."""
+"""CeleryExecutor
+
+.. seealso::
+    For more information on how the CeleryExecutor works, take a look at the guide:
+    :ref:`executor:CeleryExecutor`
+"""
 import logging
 import math
 import os

diff --git a/airflow/executors/dask_executor.py b/airflow/executors/dask_executor.py
@@ -16,6 +16,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+"""
+DaskExecutor
+
+.. seealso::
+    For more information on how the DaskExecutor works, take a look at the guide:
+    :ref:`executor:DaskExecutor`
+"""
 
 import distributed
 import subprocess

diff --git a/airflow/executors/debug_executor.py b/airflow/executors/debug_executor.py
@@ -17,8 +17,11 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-This module contains DebugExecutor that is a single
-process executor meaning it does not use multiprocessing.
+DebugExecutor
+
+.. seealso::
+    For more information on how the DebugExecutor works, take a look at the guide:
+    :ref:`executor:DebugExecutor`
 """
 
 import threading

diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
@@ -14,7 +14,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Kubernetes executor"""
+"""
+KubernetesExecutor
+
+.. seealso::
+    For more information on how the KubernetesExecutor works, take a look at the guide:
+    :ref:`executor:KubernetesExecutor`
+"""
 import base64
 import json
 import multiprocessing

diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py
@@ -17,31 +17,11 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-LocalExecutor runs tasks by spawning processes in a controlled fashion in different
-modes. Given that BaseExecutor has the option to receive a `parallelism` parameter to
-limit the number of process spawned, when this parameter is `0` the number of processes
-that LocalExecutor can spawn is unlimited.
-
-The following strategies are implemented:
-1. Unlimited Parallelism (self.parallelism == 0): In this strategy, LocalExecutor will
-spawn a process every time `execute_async` is called, that is, every task submitted to the
-LocalExecutor will be executed in its own process. Once the task is executed and the
-result stored in the `result_queue`, the process terminates. There is no need for a
-`task_queue` in this approach, since as soon as a task is received a new process will be
-allocated to the task. Processes used in this strategy are of class LocalWorker.
-
-2. Limited Parallelism (self.parallelism > 0): In this strategy, the LocalExecutor spawns
-the number of processes equal to the value of `self.parallelism` at `start` time,
-using a `task_queue` to coordinate the ingestion of tasks and the work distribution among
-the workers, which will take a task as soon as they are ready. During the lifecycle of
-the LocalExecutor, the worker processes are running waiting for tasks, once the
-LocalExecutor receives the call to shutdown the executor a poison token is sent to the
-workers to terminate them. Processes used in this strategy are of class QueuedLocalWorker.
-
-Arguably, `SequentialExecutor` could be thought as a LocalExecutor with limited
-parallelism of just 1 worker, i.e. `self.parallelism = 1`.
-This option could lead to the unification of the executor implementations, running
-locally, into just one `LocalExecutor` with multiple modes.
+LocalExecutor
+
+.. seealso::
+    For more information on how the LocalExecutor works, take a look at the guide:
+    :ref:`executor:LocalExecutor`
 """
 
 import multiprocessing

diff --git a/airflow/executors/sequential_executor.py b/airflow/executors/sequential_executor.py
@@ -17,6 +17,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+SequentialExecutor
+
+.. seealso::
+    For more information on how the SequentialExecutor works, take a look at the guide:
+    :ref:`executor:SequentialExecutor`
+"""
 from builtins import str
 import subprocess
 

diff --git a/docs/executor/celery.rst b/docs/executor/celery.rst
@@ -15,6 +15,8 @@
     specific language governing permissions and limitations
     under the License.
 
+.. _executor:CeleryExecutor:
+
 Celery Executor
 ===============
 

diff --git a/docs/executor/dask.rst b/docs/executor/dask.rst
@@ -15,6 +15,8 @@
     specific language governing permissions and limitations
     under the License.
 
+.. _executor:DaskExecutor:
+
 Dask Executor
 =============
 

diff --git a/docs/executor/debug.rst b/docs/executor/debug.rst
@@ -15,6 +15,9 @@
     specific language governing permissions and limitations
     under the License.
 
+
+.. _executor:DebugExecutor:
+
 Debug Executor
 ==================
 

diff --git a/docs/executor/index.rst b/docs/executor/index.rst
@@ -27,6 +27,10 @@ section of the configuration file.
 
 .. toctree::
     :maxdepth: 1
-    :glob:
 
-    *
+    sequential
+    debug
+    local
+    dask
+    celery
+    kubernetes
diff --git a/docs/executor/kubernetes.rst b/docs/executor/kubernetes.rst
@@ -15,7 +15,7 @@
     specific language governing permissions and limitations
     under the License.
 
-
+.. _executor:KubernetesExecutor:
 
 Kubernetes Executor
 ===================

diff --git a/docs/executor/local.rst b/docs/executor/local.rst
@@ -0,0 +1,49 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+.. _executor:LocalExecutor:
+
+Local Executor
+==============
+
+:class:`~airflow.executors.local_executor.LocalExecutor` runs tasks by spawning processes in a controlled fashion in different modes.
+
+Given that BaseExecutor has the option to receive a ``parallelism`` parameter to limit the number of process spawned,
+when this parameter is ``0`` the number of processes that LocalExecutor can spawn is unlimited.
+
+The following strategies are implemented:
+
+- | **Unlimited Parallelism** (``self.parallelism == 0``): In this strategy, LocalExecutor will
+  | spawn a process every time ``execute_async`` is called, that is, every task submitted to the
+  | :class:`~airflow.executors.local_executor.LocalExecutor` will be executed in its own process. Once the task is executed and the
+  | result stored in the ``result_queue``, the process terminates. There is no need for a
+  | ``task_queue`` in this approach, since as soon as a task is received a new process will be
+  | allocated to the task. Processes used in this strategy are of class :class:`~airflow.executors.local_executor.LocalWorker`.
+
+- | **Limited Parallelism** (``self.parallelism > 0``): In this strategy, the :class:`~airflow.executors.local_executor.LocalExecutor` spawns
+  | the number of processes equal to the value of ``self.parallelism`` at ``start`` time,
+  | using a ``task_queue`` to coordinate the ingestion of tasks and the work distribution among
+  | the workers, which will take a task as soon as they are ready. During the lifecycle of
+  | the LocalExecutor, the worker processes are running waiting for tasks, once the
+  | LocalExecutor receives the call to shutdown the executor a poison token is sent to the
+  | workers to terminate them. Processes used in this strategy are of class :class:`~airflow.executors.local_executor.QueuedLocalWorker`.
+
+Arguably, :class:`~airflow.executors.sequential_executor.SequentialExecutor` could be thought as a ``LocalExecutor`` with limited
+parallelism of just 1 worker, i.e. ``self.parallelism = 1``.
+This option could lead to the unification of the executor implementations, running
+locally, into just one :class:`~airflow.executors.local_executor.LocalExecutor` with multiple modes.
diff --git a/docs/executor/sequential.rst b/docs/executor/sequential.rst
@@ -0,0 +1,26 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+.. _executor:SequentialExecutor:
+
+Sequential Executor
+===================
+
+The :class:`~airflow.executors.sequential_executor.SequentialExecutor` is the default executor when you first install ``airflow``.
+It is the only executor that can be used with ``sqlite`` since ``sqlite`` doesn't support multiple connections.
+This executor will only run one task instance at a time. For production use case, please use other executors.