diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py index 2768030c741f6..128b25b31b7de 100644 --- a/airflow/executors/celery_executor.py +++ b/airflow/executors/celery_executor.py @@ -16,7 +16,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Celery executor.""" +"""CeleryExecutor + +.. seealso:: + For more information on how the CeleryExecutor works, take a look at the guide: + :ref:`executor:CeleryExecutor` +""" import logging import math import os diff --git a/airflow/executors/dask_executor.py b/airflow/executors/dask_executor.py index 7fb8f04c378a4..ea0f397733994 100644 --- a/airflow/executors/dask_executor.py +++ b/airflow/executors/dask_executor.py @@ -16,6 +16,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +DaskExecutor + +.. seealso:: + For more information on how the DaskExecutor works, take a look at the guide: + :ref:`executor:DaskExecutor` +""" import distributed import subprocess diff --git a/airflow/executors/debug_executor.py b/airflow/executors/debug_executor.py index 37bed09487cf2..7f614378ebc30 100644 --- a/airflow/executors/debug_executor.py +++ b/airflow/executors/debug_executor.py @@ -17,8 +17,11 @@ # specific language governing permissions and limitations # under the License. """ -This module contains DebugExecutor that is a single -process executor meaning it does not use multiprocessing. +DebugExecutor + +.. seealso:: + For more information on how the DebugExecutor works, take a look at the guide: + :ref:`executor:DebugExecutor` """ import threading diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py index 8f87c08a672e5..7bbdc985ebcee 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/executors/kubernetes_executor.py @@ -14,7 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Kubernetes executor""" +""" +KubernetesExecutor + +.. seealso:: + For more information on how the KubernetesExecutor works, take a look at the guide: + :ref:`executor:KubernetesExecutor` +""" import base64 import json import multiprocessing diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py index 2e06f4f828680..b4a0fba678f65 100644 --- a/airflow/executors/local_executor.py +++ b/airflow/executors/local_executor.py @@ -17,31 +17,11 @@ # specific language governing permissions and limitations # under the License. """ -LocalExecutor runs tasks by spawning processes in a controlled fashion in different -modes. Given that BaseExecutor has the option to receive a `parallelism` parameter to -limit the number of process spawned, when this parameter is `0` the number of processes -that LocalExecutor can spawn is unlimited. - -The following strategies are implemented: -1. Unlimited Parallelism (self.parallelism == 0): In this strategy, LocalExecutor will -spawn a process every time `execute_async` is called, that is, every task submitted to the -LocalExecutor will be executed in its own process. Once the task is executed and the -result stored in the `result_queue`, the process terminates. There is no need for a -`task_queue` in this approach, since as soon as a task is received a new process will be -allocated to the task. Processes used in this strategy are of class LocalWorker. - -2. Limited Parallelism (self.parallelism > 0): In this strategy, the LocalExecutor spawns -the number of processes equal to the value of `self.parallelism` at `start` time, -using a `task_queue` to coordinate the ingestion of tasks and the work distribution among -the workers, which will take a task as soon as they are ready. During the lifecycle of -the LocalExecutor, the worker processes are running waiting for tasks, once the -LocalExecutor receives the call to shutdown the executor a poison token is sent to the -workers to terminate them. Processes used in this strategy are of class QueuedLocalWorker. - -Arguably, `SequentialExecutor` could be thought as a LocalExecutor with limited -parallelism of just 1 worker, i.e. `self.parallelism = 1`. -This option could lead to the unification of the executor implementations, running -locally, into just one `LocalExecutor` with multiple modes. +LocalExecutor + +.. seealso:: + For more information on how the LocalExecutor works, take a look at the guide: + :ref:`executor:LocalExecutor` """ import multiprocessing diff --git a/airflow/executors/sequential_executor.py b/airflow/executors/sequential_executor.py index a0013e67f61a9..cb5717c9d9001 100644 --- a/airflow/executors/sequential_executor.py +++ b/airflow/executors/sequential_executor.py @@ -17,6 +17,13 @@ # specific language governing permissions and limitations # under the License. +""" +SequentialExecutor + +.. seealso:: + For more information on how the SequentialExecutor works, take a look at the guide: + :ref:`executor:SequentialExecutor` +""" from builtins import str import subprocess diff --git a/docs/executor/celery.rst b/docs/executor/celery.rst index 43f64674fbffe..8348c52a1f14f 100644 --- a/docs/executor/celery.rst +++ b/docs/executor/celery.rst @@ -15,6 +15,8 @@ specific language governing permissions and limitations under the License. +.. _executor:CeleryExecutor: + Celery Executor =============== diff --git a/docs/executor/dask.rst b/docs/executor/dask.rst index 1f3fec38699a6..ba9276ad4491f 100644 --- a/docs/executor/dask.rst +++ b/docs/executor/dask.rst @@ -15,6 +15,8 @@ specific language governing permissions and limitations under the License. +.. _executor:DaskExecutor: + Dask Executor ============= diff --git a/docs/executor/debug.rst b/docs/executor/debug.rst index ca4029be2b68b..98d842401f770 100644 --- a/docs/executor/debug.rst +++ b/docs/executor/debug.rst @@ -15,6 +15,9 @@ specific language governing permissions and limitations under the License. + +.. _executor:DebugExecutor: + Debug Executor ================== diff --git a/docs/executor/index.rst b/docs/executor/index.rst index 28f75663fee77..7994ac74e9454 100644 --- a/docs/executor/index.rst +++ b/docs/executor/index.rst @@ -27,6 +27,10 @@ section of the configuration file. .. toctree:: :maxdepth: 1 - :glob: - * + sequential + debug + local + dask + celery + kubernetes diff --git a/docs/executor/kubernetes.rst b/docs/executor/kubernetes.rst index 53353f9afafce..bd0f8dd49590e 100644 --- a/docs/executor/kubernetes.rst +++ b/docs/executor/kubernetes.rst @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. - +.. _executor:KubernetesExecutor: Kubernetes Executor =================== diff --git a/docs/executor/local.rst b/docs/executor/local.rst new file mode 100644 index 0000000000000..6721cf93f1b4b --- /dev/null +++ b/docs/executor/local.rst @@ -0,0 +1,49 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. _executor:LocalExecutor: + +Local Executor +============== + +:class:`~airflow.executors.local_executor.LocalExecutor` runs tasks by spawning processes in a controlled fashion in different modes. + +Given that BaseExecutor has the option to receive a ``parallelism`` parameter to limit the number of process spawned, +when this parameter is ``0`` the number of processes that LocalExecutor can spawn is unlimited. + +The following strategies are implemented: + +- | **Unlimited Parallelism** (``self.parallelism == 0``): In this strategy, LocalExecutor will + | spawn a process every time ``execute_async`` is called, that is, every task submitted to the + | :class:`~airflow.executors.local_executor.LocalExecutor` will be executed in its own process. Once the task is executed and the + | result stored in the ``result_queue``, the process terminates. There is no need for a + | ``task_queue`` in this approach, since as soon as a task is received a new process will be + | allocated to the task. Processes used in this strategy are of class :class:`~airflow.executors.local_executor.LocalWorker`. + +- | **Limited Parallelism** (``self.parallelism > 0``): In this strategy, the :class:`~airflow.executors.local_executor.LocalExecutor` spawns + | the number of processes equal to the value of ``self.parallelism`` at ``start`` time, + | using a ``task_queue`` to coordinate the ingestion of tasks and the work distribution among + | the workers, which will take a task as soon as they are ready. During the lifecycle of + | the LocalExecutor, the worker processes are running waiting for tasks, once the + | LocalExecutor receives the call to shutdown the executor a poison token is sent to the + | workers to terminate them. Processes used in this strategy are of class :class:`~airflow.executors.local_executor.QueuedLocalWorker`. + +Arguably, :class:`~airflow.executors.sequential_executor.SequentialExecutor` could be thought as a ``LocalExecutor`` with limited +parallelism of just 1 worker, i.e. ``self.parallelism = 1``. +This option could lead to the unification of the executor implementations, running +locally, into just one :class:`~airflow.executors.local_executor.LocalExecutor` with multiple modes. diff --git a/docs/executor/sequential.rst b/docs/executor/sequential.rst new file mode 100644 index 0000000000000..be5b131eef419 --- /dev/null +++ b/docs/executor/sequential.rst @@ -0,0 +1,26 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. _executor:SequentialExecutor: + +Sequential Executor +=================== + +The :class:`~airflow.executors.sequential_executor.SequentialExecutor` is the default executor when you first install ``airflow``. +It is the only executor that can be used with ``sqlite`` since ``sqlite`` doesn't support multiple connections. +This executor will only run one task instance at a time. For production use case, please use other executors.