diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 4d8a48e04fb68..fa007391497d1 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2847,6 +2847,13 @@ components: type: string readOnly: true description: The ID of the DAG. + dag_display_name: + type: string + readOnly: true + description: | + Human centric display text for the DAG. + + *New in version 2.9.0* root_dag_id: type: string readOnly: true @@ -3570,6 +3577,12 @@ components: properties: task_id: type: string + task_display_name: + type: string + description: | + Human centric display text for the task. + + *New in version 2.9.0* dag_id: type: string dag_run_id: @@ -3925,6 +3938,9 @@ components: task_id: type: string readOnly: true + task_display_name: + type: string + readOnly: true owner: type: string readOnly: true diff --git a/airflow/api_connexion/schemas/dag_schema.py b/airflow/api_connexion/schemas/dag_schema.py index 0bffdcf685784..9add4f416cc00 100644 --- a/airflow/api_connexion/schemas/dag_schema.py +++ b/airflow/api_connexion/schemas/dag_schema.py @@ -50,6 +50,7 @@ class Meta: model = DagModel dag_id = auto_field(dump_only=True) + dag_display_name = fields.String(attribute="dag_display_name", dump_only=True) root_dag_id = auto_field(dump_only=True) is_paused = auto_field() is_active = auto_field(dump_only=True) diff --git a/airflow/api_connexion/schemas/task_instance_schema.py b/airflow/api_connexion/schemas/task_instance_schema.py index 8f0fc0533520d..4777b8bd4c577 100644 --- a/airflow/api_connexion/schemas/task_instance_schema.py +++ b/airflow/api_connexion/schemas/task_instance_schema.py @@ -55,6 +55,7 @@ class Meta: state = TaskInstanceStateField() _try_number = auto_field(data_key="try_number") max_tries = auto_field() + task_display_name = fields.String(attribute="task_display_name", dump_only=True) hostname = auto_field() unixname = auto_field() pool = auto_field() diff --git a/airflow/api_connexion/schemas/task_schema.py b/airflow/api_connexion/schemas/task_schema.py index ac1b465bb25b0..8d8f1e33f2985 100644 --- a/airflow/api_connexion/schemas/task_schema.py +++ b/airflow/api_connexion/schemas/task_schema.py @@ -39,6 +39,7 @@ class TaskSchema(Schema): class_ref = fields.Method("_get_class_reference", dump_only=True) operator_name = fields.Method("_get_operator_name", dump_only=True) task_id = fields.String(dump_only=True) + task_display_name = fields.String(attribute="task_display_name", dump_only=True) owner = fields.String(dump_only=True) start_date = fields.DateTime(dump_only=True) end_date = fields.DateTime(dump_only=True) diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index 137cd3827e392..5ea8f3a6a219e 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -326,6 +326,7 @@ def _get_dagbag_dag_details(dag: DAG) -> dict: """Return a dagbag dag details dict.""" return { "dag_id": dag.dag_id, + "dag_display_name": dag.dag_display_name, "root_dag_id": dag.parent_dag.dag_id if dag.parent_dag else None, "is_paused": dag.get_is_paused(), "is_active": dag.get_is_active(), diff --git a/airflow/example_dags/example_display_name.py b/airflow/example_dags/example_display_name.py new file mode 100644 index 0000000000000..b6b447d632a6a --- /dev/null +++ b/airflow/example_dags/example_display_name.py @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pendulum + +from airflow.decorators import dag, task +from airflow.operators.empty import EmptyOperator + + +# [START dag_decorator_usage] +@dag( + schedule=None, + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + catchup=False, + tags=["example"], + dag_display_name="Sample DAG with Display Name", +) +def example_display_name(): + sample_task_1 = EmptyOperator( + task_id="sample_task_1", + task_display_name="Sample Task 1", + ) + + @task(task_display_name="Sample Task 2") + def sample_task_2(): + pass + + sample_task_1 >> sample_task_2() + + +example_dag = example_display_name() +# [END dag_decorator_usage] diff --git a/airflow/example_dags/example_params_trigger_ui.py b/airflow/example_dags/example_params_trigger_ui.py index da28a9cc9bc39..2a1e6c9b34e13 100644 --- a/airflow/example_dags/example_params_trigger_ui.py +++ b/airflow/example_dags/example_params_trigger_ui.py @@ -37,12 +37,13 @@ with DAG( dag_id=Path(__file__).stem, + dag_display_name="Params Trigger UI", description=__doc__.partition(".")[0], doc_md=__doc__, schedule=None, start_date=datetime.datetime(2022, 3, 4), catchup=False, - tags=["example_ui"], + tags=["example", "params"], params={ "names": Param( ["Linda", "Martha", "Thomas"], @@ -57,7 +58,7 @@ }, ) as dag: - @task(task_id="get_names") + @task(task_id="get_names", task_display_name="Get names") def get_names(**kwargs) -> list[str]: ti: TaskInstance = kwargs["ti"] dag_run: DagRun = ti.dag_run @@ -66,7 +67,7 @@ def get_names(**kwargs) -> list[str]: return [] return dag_run.conf["names"] - @task.branch(task_id="select_languages") + @task.branch(task_id="select_languages", task_display_name="Select languages") def select_languages(**kwargs) -> list[str]: ti: TaskInstance = kwargs["ti"] dag_run: DagRun = ti.dag_run @@ -76,19 +77,19 @@ def select_languages(**kwargs) -> list[str]: selected_languages.append(f"generate_{lang}_greeting") return selected_languages - @task(task_id="generate_english_greeting") + @task(task_id="generate_english_greeting", task_display_name="Generate English greeting") def generate_english_greeting(name: str) -> str: return f"Hello {name}!" - @task(task_id="generate_german_greeting") + @task(task_id="generate_german_greeting", task_display_name="Erzeuge Deutsche Begrüßung") def generate_german_greeting(name: str) -> str: return f"Sehr geehrter Herr/Frau {name}." - @task(task_id="generate_french_greeting") + @task(task_id="generate_french_greeting", task_display_name="Produire un message d'accueil en français") def generate_french_greeting(name: str) -> str: return f"Bonjour {name}!" - @task(task_id="print_greetings", trigger_rule=TriggerRule.ALL_DONE) + @task(task_id="print_greetings", task_display_name="Print greetings", trigger_rule=TriggerRule.ALL_DONE) def print_greetings(greetings1, greetings2, greetings3) -> None: for g in greetings1 or []: print(g) diff --git a/airflow/example_dags/example_params_ui_tutorial.py b/airflow/example_dags/example_params_ui_tutorial.py index f4c85c9e430c8..1f293415d15a1 100644 --- a/airflow/example_dags/example_params_ui_tutorial.py +++ b/airflow/example_dags/example_params_ui_tutorial.py @@ -33,12 +33,13 @@ with DAG( dag_id=Path(__file__).stem, + dag_display_name="Params UI tutorial", description=__doc__.partition(".")[0], doc_md=__doc__, schedule=None, start_date=datetime.datetime(2022, 3, 4), catchup=False, - tags=["example_ui"], + tags=["example", "params", "ui"], params={ # Let's start simple: Standard dict values are detected from type and offered as entry form fields. # Detected types are numbers, text, boolean, lists and dicts. @@ -237,7 +238,7 @@ }, ) as dag: - @task + @task(task_display_name="Show used parameters") def show_params(**kwargs) -> None: params: ParamsDict = kwargs["params"] print(f"This DAG was triggered with the following parameters:\n\n{json.dumps(params, indent=4)}\n") diff --git a/airflow/migrations/versions/0139_2_9_0_add_display_name_for_dag_and_task_.py b/airflow/migrations/versions/0139_2_9_0_add_display_name_for_dag_and_task_.py new file mode 100644 index 0000000000000..854eee8e7f82c --- /dev/null +++ b/airflow/migrations/versions/0139_2_9_0_add_display_name_for_dag_and_task_.py @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""add display name for dag and task instance + +Revision ID: ee1467d4aa35 +Revises: b4078ac230a1 +Create Date: 2024-03-24 22:33:36.824827 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = 'ee1467d4aa35' +down_revision = 'b4078ac230a1' +branch_labels = None +depends_on = None +airflow_version = "2.9.0" + + +def upgrade(): + """Apply add display name for dag and task instance""" + op.add_column("dag", sa.Column("dag_display_name", sa.String(2000), nullable=True)) + op.add_column("task_instance", sa.Column("task_display_name", sa.String(2000), nullable=True)) + + +def downgrade(): + """Unapply add display name for dag and task instance""" + op.drop_column("dag", "dag_display_name") + op.drop_column("task_instance", "task_display_name") diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index f2d179f01ba35..380c6fcf00f3d 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -19,6 +19,7 @@ import datetime import inspect +from abc import abstractproperty from functools import cached_property from typing import TYPE_CHECKING, Any, Callable, ClassVar, Collection, Iterable, Iterator, Sequence @@ -159,6 +160,20 @@ def dag_id(self) -> str: def node_id(self) -> str: return self.task_id + @abstractproperty + def task_display_name(self) -> str: ... + + @property + def label(self) -> str | None: + if self.task_display_name and self.task_display_name != self.task_id: + return self.task_display_name + # Prefix handling if no display is given is cloned from taskmixin for compatibility + tg = self.task_group + if tg and tg.node_id and tg.prefix_group_id: + # "task_group_id.task_id" -> "task_id" + return self.task_id[len(tg.node_id) + 1 :] + return self.task_id + @property def is_setup(self) -> bool: raise NotImplementedError() diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 8636dd6c2e92d..5334fc90205aa 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -267,6 +267,7 @@ def partial( doc_json: str | None | ArgNotSet = NOTSET, doc_yaml: str | None | ArgNotSet = NOTSET, doc_rst: str | None | ArgNotSet = NOTSET, + task_display_name: str | None | ArgNotSet = NOTSET, logger_name: str | None | ArgNotSet = NOTSET, allow_nested_operators: bool = True, **kwargs, @@ -334,6 +335,7 @@ def partial( "doc_md": doc_md, "doc_rst": doc_rst, "doc_yaml": doc_yaml, + "task_display_name": task_display_name, "logger_name": logger_name, "allow_nested_operators": allow_nested_operators, } @@ -705,6 +707,7 @@ class derived from this one results in the creation of a task object, that is visible in Task Instance details View in the Webserver :param doc_yaml: Add documentation (in YAML format) or notes to your Task objects that is visible in Task Instance details View in the Webserver + :param task_display_name: The display name of the task which appears on the UI. :param logger_name: Name of the logger used by the Operator to emit logs. If set to `None` (default), the logger name will fall back to `airflow.task.operators.{class.__module__}.{class.__name__}` (e.g. SimpleHttpOperator will have @@ -859,6 +862,7 @@ def __init__( doc_json: str | None = None, doc_yaml: str | None = None, doc_rst: str | None = None, + task_display_name: str | None = None, logger_name: str | None = None, allow_nested_operators: bool = True, **kwargs, @@ -1001,6 +1005,10 @@ def __init__( self.doc_yaml = doc_yaml self.doc_rst = doc_rst self.doc = doc + # Populate the display field only if provided and different from task id + self._task_display_property_value = ( + task_display_name if task_display_name and task_display_name != task_id else None + ) self.upstream_task_ids: set[str] = set() self.downstream_task_ids: set[str] = set() @@ -1188,6 +1196,10 @@ def dag(self, dag: DAG | None): self._dag = dag + @property + def task_display_name(self) -> str: + return self._task_display_property_value or self.task_id + def has_dag(self): """Return True if the Operator has been assigned to a DAG.""" return self._dag is not None diff --git a/airflow/models/dag.py b/airflow/models/dag.py index a230c94fd7529..3e6076e062455 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -156,6 +156,13 @@ from airflow.typing_compat import Literal from airflow.utils.task_group import TaskGroup + # This is a workaround because mypy doesn't work with hybrid_property + # TODO: remove this hack and move hybrid_property back to main import block + # See https://github.com/python/mypy/issues/4430 + hybrid_property = property +else: + from sqlalchemy.ext.hybrid import hybrid_property + log = logging.getLogger(__name__) DEFAULT_VIEW_PRESETS = ["grid", "graph", "duration", "gantt", "landing_times"] @@ -412,6 +419,7 @@ class DAG(LoggingMixin): :param fail_stop: Fails currently running tasks when task in DAG fails. **Warning**: A fail stop dag can only have tasks with the default trigger rule ("all_success"). An exception will be thrown if any task in a fail stop dag has a non default trigger rule. + :param dag_display_name: The display name of the DAG which appears on the UI. """ _comps = { @@ -478,6 +486,7 @@ def __init__( owner_links: dict[str, str] | None = None, auto_register: bool = True, fail_stop: bool = False, + dag_display_name: str | None = None, ): from airflow.utils.task_group import TaskGroup @@ -510,6 +519,8 @@ def __init__( validate_key(dag_id) self._dag_id = dag_id + self._dag_display_property_value = dag_display_name + if concurrency: # TODO: Remove in Airflow 3.0 warnings.warn( @@ -1300,6 +1311,10 @@ def access_control(self): def access_control(self, value): self._access_control = DAG._upgrade_outdated_dag_access_control(value) + @property + def dag_display_name(self) -> str: + return self._dag_display_property_value or self._dag_id + @property def description(self) -> str | None: return self._description @@ -3133,6 +3148,7 @@ def bulk_write_to_db( orm_dag.has_import_errors = False orm_dag.last_parsed_time = timezone.utcnow() orm_dag.default_view = dag.default_view + orm_dag._dag_display_property_value = dag._dag_display_property_value orm_dag.description = dag.description orm_dag.max_active_tasks = dag.max_active_tasks orm_dag.max_active_runs = dag.max_active_runs @@ -3589,6 +3605,8 @@ class DagModel(Base): processor_subdir = Column(String(2000), nullable=True) # String representing the owners owners = Column(String(2000)) + # Display name of the dag + _dag_display_property_value = Column("dag_display_name", String(2000), nullable=True) # Description of the dag description = Column(Text) # Default view of the DAG inside the webserver @@ -3783,6 +3801,10 @@ def set_is_paused(self, is_paused: bool, including_subdags: bool = True, session ) session.commit() + @hybrid_property + def dag_display_name(self) -> str: + return self._dag_display_property_value or self.dag_id + @classmethod @internal_api_call @provide_session @@ -3982,6 +4004,7 @@ def dag( owner_links: dict[str, str] | None = None, auto_register: bool = True, fail_stop: bool = False, + dag_display_name: str | None = None, ) -> Callable[[Callable], Callable[..., DAG]]: """ Python dag decorator which wraps a function into an Airflow DAG. @@ -4038,6 +4061,7 @@ def factory(*args, **kwargs): owner_links=owner_links, auto_register=auto_register, fail_stop=fail_stop, + dag_display_name=dag_display_name, ) as dag_obj: # Set DAG documentation from function documentation if it exists and doc_md is not set. if f.__doc__ and not dag_obj.doc_md: diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py index 862928d7ab2da..994e041d9fa5c 100644 --- a/airflow/models/mappedoperator.py +++ b/airflow/models/mappedoperator.py @@ -386,6 +386,10 @@ def leaves(self) -> Sequence[AbstractOperator]: """Implementing DAGNode.""" return [self] + @property + def task_display_name(self) -> str: + return self.partial_kwargs.get("task_display_name") or self.task_id + @property def owner(self) -> str: # type: ignore[override] return self.partial_kwargs.get("owner", DEFAULT_OWNER) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 9968da5898a66..9125e6912a20b 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1307,6 +1307,7 @@ class TaskInstance(Base, LoggingMixin): next_method = Column(String(1000)) next_kwargs = Column(MutableDict.as_mutable(ExtendedJSON)) + _task_display_property_value = Column("task_display_name", String(2000), nullable=True) # If adding new fields here then remember to add them to # refresh_from_db() or they won't display in the UI correctly @@ -1475,6 +1476,7 @@ def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any "operator": task.task_type, "custom_operator_name": getattr(task, "custom_operator_name", None), "map_index": map_index, + "_task_display_property_value": task.task_display_name, } @reconstructor @@ -1535,6 +1537,10 @@ def operator_name(self) -> str | None: """@property: use a more friendly display name for the operator, if set.""" return self.custom_operator_name or self.operator + @hybrid_property + def task_display_name(self) -> str: + return self._task_display_property_value or self.task_id + @staticmethod def _command_as_list( ti: TaskInstance | TaskInstancePydantic, diff --git a/airflow/serialization/schema.json b/airflow/serialization/schema.json index 31e8d58585514..a2a6732763641 100644 --- a/airflow/serialization/schema.json +++ b/airflow/serialization/schema.json @@ -169,7 +169,8 @@ {"type": "string"} ] }, - "orientation": { "type" : "string"}, + "orientation": { "type" : "string"}, + "_dag_display_property_value": { "type" : "string"}, "_description": { "type" : "string"}, "_concurrency": { "type" : "number"}, "_max_active_tasks": { "type" : "number"}, @@ -239,6 +240,7 @@ "_task_module": { "type": "string" }, "_operator_extra_links": { "$ref": "#/definitions/extra_links" }, "task_id": { "type": "string" }, + "task_display_name": { "type": "string" }, "label": { "type": "string" }, "owner": { "type": "string" }, "start_date": { "$ref": "#/definitions/datetime" }, diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index f0427f8f18190..16a5c9e481d1d 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -1395,6 +1395,7 @@ class SerializedDAG(DAG, BaseSerialization): def __get_constructor_defaults(): param_to_attr = { "max_active_tasks": "_max_active_tasks", + "dag_display_name": "_dag_display_property_value", "description": "_description", "default_view": "_default_view", "access_control": "_access_control", diff --git a/airflow/www/static/js/dag/details/Header.tsx b/airflow/www/static/js/dag/details/Header.tsx index 06c1e2dd11a73..57e0afa1b0103 100644 --- a/airflow/www/static/js/dag/details/Header.tsx +++ b/airflow/www/static/js/dag/details/Header.tsx @@ -34,6 +34,7 @@ import RunTypeIcon from "src/components/RunTypeIcon"; import BreadcrumbText from "./BreadcrumbText"; const dagId = getMetaValue("dag_id"); +const dagDisplayName = getMetaValue("dag_display_name"); const Header = () => { const { @@ -89,7 +90,9 @@ const Header = () => { const lastIndex = taskId ? taskId.lastIndexOf(".") : null; const taskName = - taskId && lastIndex ? taskId.substring(lastIndex + 1) : taskId; + taskInstance?.taskDisplayName && lastIndex + ? taskInstance?.taskDisplayName.substring(lastIndex + 1) + : taskId; const isDagDetails = !runId && !taskId; const isRunDetails = !!(runId && !taskId); @@ -103,7 +106,7 @@ const Header = () => { onClick={clearSelection} _hover={isDagDetails ? { cursor: "default" } : undefined} > - + {runId && ( diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index ea9c0c1df63a1..b8da89e55604b 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -936,6 +936,12 @@ export interface components { DAG: { /** @description The ID of the DAG. */ dag_id?: string; + /** + * @description Human centric display text for the DAG. + * + * *New in version 2.9.0* + */ + dag_display_name?: string; /** @description If the DAG is SubDAG then it is the top level DAG identifier. Otherwise, null. */ root_dag_id?: string | null; /** @description Whether the DAG is paused. */ @@ -1391,6 +1397,12 @@ export interface components { } | null; TaskInstance: { task_id?: string; + /** + * @description Human centric display text for the task. + * + * *New in version 2.9.0* + */ + task_display_name?: string; dag_id?: string; /** * @description The DagRun ID for this task instance @@ -1596,6 +1608,7 @@ export interface components { Task: { class_ref?: components["schemas"]["ClassReference"]; task_id?: string; + task_display_name?: string; owner?: string; /** Format: date-time */ start_date?: string; diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 9d57082dbb86c..4e511a6854490 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -21,7 +21,7 @@ {% from 'airflow/dataset_next_run_modal.html' import dataset_next_run_modal %} {% from 'appbuilder/dag_docs.html' import dag_docs %} -{% block page_title %}{{ dag.dag_id }} - {{ appbuilder.app_name }}{% endblock %} +{% block page_title %}{{ dag.dag_display_name }} - {{ appbuilder.app_name }}{% endblock %} {% block head_css %} {{ super() }} @@ -40,6 +40,7 @@ {% block head_meta %} {{ super() }} + @@ -106,7 +107,7 @@ {% if dag.parent_dag is defined and dag.parent_dag %} - DAG: {{ dag.parent_dag.dag_id }} + DAG: {{ dag.parent_dag.dag_display_name }} {% endif %}
@@ -127,7 +128,7 @@

{{ " disabled" if not can_edit_dag else "" }}> - DAG: {{ dag.dag_id }} + DAG: {{ dag.dag_display_name }} {{ dag.description[0:150] + '…' if dag.description and dag.description|length > 150 else dag.description|default('', true) }} {% if dag_model is defined and dag_model.max_consecutive_failed_dag_runs is defined and dag_model.max_consecutive_failed_dag_runs > 0 %} diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html index 9c787cb1fccdc..1bb5ac25abf4d 100644 --- a/airflow/www/templates/airflow/dags.html +++ b/airflow/www/templates/airflow/dags.html @@ -22,7 +22,7 @@ {% from 'airflow/dataset_next_run_modal.html' import dataset_next_run_modal %} {% from 'airflow/_messages.html' import show_message %} -{%- macro sortable_column(display_name, attribute_name) -%} +{%- macro sortable_column(dag_display_name, attribute_name) -%} {% set curr_ordering_direction = (request.args.get('sorting_direction', 'desc')) %} {% set new_ordering_direction = ('asc' if (request.args.get('sorting_key') != attribute_name or curr_ordering_direction == 'desc') else 'desc') %} - {{ display_name }} + {{ dag_display_name }} - - {{ dag.dag_id }} + {{ dag.dag_display_name }}
{% for tag in dag.tags | sort(attribute='name') %} diff --git a/airflow/www/templates/airflow/grid.html b/airflow/www/templates/airflow/grid.html index 8ca43af0d3822..3c7bb236e178e 100644 --- a/airflow/www/templates/airflow/grid.html +++ b/airflow/www/templates/airflow/grid.html @@ -18,7 +18,7 @@ #} {% extends "airflow/dag.html" %} -{% block page_title %}{{ dag.dag_id }} - Grid - {{ appbuilder.app_name }}{% endblock %} +{% block page_title %}{{ dag.dag_display_name }} - Grid - {{ appbuilder.app_name }}{% endblock %} {% from 'appbuilder/loading_dots.html' import loading_dots %} {% block head_meta %} diff --git a/airflow/www/templates/airflow/task_instance.html b/airflow/www/templates/airflow/task_instance.html index 44a764b63ada3..f7faff3a538cb 100644 --- a/airflow/www/templates/airflow/task_instance.html +++ b/airflow/www/templates/airflow/task_instance.html @@ -25,7 +25,7 @@

- Task Instance: {{ task_id }} + Task Instance: {{ task_display_name }} at {% if map_index is defined and map_index >= 0 %} Map Index: {{ map_index }} diff --git a/airflow/www/templates/airflow/trigger.html b/airflow/www/templates/airflow/trigger.html index 6d60d04383167..d08ac3cef63a2 100644 --- a/airflow/www/templates/airflow/trigger.html +++ b/airflow/www/templates/airflow/trigger.html @@ -155,7 +155,7 @@ {% block content %} {{ super() }}

- Trigger DAG: {{ dag.dag_id }} + Trigger DAG: {{ dag.dag_display_name }} {{ dag.description[0:150] + '…' if dag.description and dag.description|length > 150 else dag.description|default('', true) }}

{{ dag_docs(doc_md, False) }} diff --git a/airflow/www/views.py b/airflow/www/views.py index 17eabf390229e..a9b3d65ca140a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -871,7 +871,16 @@ def index(self): current_dags = all_dags num_of_all_dags = all_dags_count - if arg_sorting_key == "last_dagrun": + if arg_sorting_key == "dag_id": + if arg_sorting_direction == "desc": + current_dags = current_dags.order_by( + func.coalesce(DagModel.dag_display_name, DagModel.dag_id).desc() + ) + else: + current_dags = current_dags.order_by( + func.coalesce(DagModel.dag_display_name, DagModel.dag_id) + ) + elif arg_sorting_key == "last_dagrun": dag_run_subquery = ( select( DagRun.dag_id, @@ -888,10 +897,8 @@ def index(self): current_dags = current_dags.order_by( null_case, dag_run_subquery.c.max_execution_date.desc() ) - else: current_dags = current_dags.order_by(null_case, dag_run_subquery.c.max_execution_date) - else: sort_column = DagModel.__table__.c.get(arg_sorting_key) if sort_column is not None: @@ -1456,6 +1463,7 @@ def rendered_templates(self, session): html_dict=html_dict, dag=dag, task_id=task_id, + task_display_name=task.task_display_name, execution_date=execution_date, map_index=map_index, form=form, @@ -1520,6 +1528,7 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION): html_dict={"k8s": content}, dag=dag, task_id=task_id, + task_display_name=task.task_display_name, execution_date=execution_date, map_index=map_index, form=form, @@ -1626,7 +1635,7 @@ def log(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) dag_model = DagModel.get_dagmodel(dag_id) - ti = session.scalar( + ti: TaskInstance = session.scalar( select(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) .limit(1) @@ -1645,6 +1654,7 @@ def log(self, session: Session = NEW_SESSION): title="Log by attempts", dag_id=dag_id, task_id=task_id, + task_display_name=ti.task_display_name, execution_date=execution_date, map_index=map_index, form=form, @@ -1806,6 +1816,7 @@ def include_task_attrs(attr_name): root=root, dag=dag, title=title, + task_display_name=task.task_display_name, ) @expose("/xcom") @@ -1824,7 +1835,9 @@ def xcom(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) root = request.args.get("root", "") dag = DagModel.get_dagmodel(dag_id) - ti = session.scalar(select(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).limit(1)) + ti: TaskInstance = session.scalar( + select(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).limit(1) + ) if not ti: flash(f"Task [{dag_id}.{task_id}] doesn't seem to exist at the moment", "error") @@ -1846,6 +1859,7 @@ def xcom(self, session: Session = NEW_SESSION): show_trigger_form_if_no_params=conf.getboolean("webserver", "show_trigger_form_if_no_params"), attributes=attributes, task_id=task_id, + task_display_name=ti.task_display_name, execution_date=execution_date, map_index=map_index, form=form, diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 3aea124fd8958..7a1c0a9fba53e 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444 \ No newline at end of file +3ec33f4a14388277f9aba431c06c3bfa9d044ab2eae466aa394aa9618d2f3eb5 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 8ae1b83685443..144668b90032a 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - - + + %3 - + job @@ -243,76 +243,76 @@ dag_run_note - -dag_run_note - -dag_run_id - [INTEGER] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] + +dag_run_note + +dag_run_id + [INTEGER] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] ab_user--dag_run_note - -0..N + +0..N {0,1} task_instance_note - -task_instance_note - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] + +task_instance_note + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] ab_user--task_instance_note - -0..N -{0,1} + +0..N +{0,1} @@ -535,129 +535,132 @@ dataset - -dataset - -id - [INTEGER] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -extra - [JSON] - NOT NULL - -is_orphaned - [BOOLEAN] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -uri - [VARCHAR(3000)] - NOT NULL + +dataset + +id + [INTEGER] + NOT NULL + +created_at + [TIMESTAMP] + NOT NULL + +extra + [JSON] + NOT NULL + +is_orphaned + [BOOLEAN] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +uri + [VARCHAR(3000)] + NOT NULL dag_schedule_dataset_reference - -dag_schedule_dataset_reference - -dag_id - [VARCHAR(250)] - NOT NULL - -dataset_id - [INTEGER] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL + +dag_schedule_dataset_reference + +dag_id + [VARCHAR(250)] + NOT NULL + +dataset_id + [INTEGER] + NOT NULL + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL dataset--dag_schedule_dataset_reference - -1 -1 + +1 +1 task_outlet_dataset_reference - -task_outlet_dataset_reference - -dag_id - [VARCHAR(250)] - NOT NULL - -dataset_id - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL + +task_outlet_dataset_reference + +dag_id + [VARCHAR(250)] + NOT NULL + +dataset_id + [INTEGER] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL dataset--task_outlet_dataset_reference - -1 -1 + +1 +1 dataset_dag_run_queue - -dataset_dag_run_queue - -dataset_id - [INTEGER] - NOT NULL - -target_dag_id - [VARCHAR(250)] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL + +dataset_dag_run_queue + +dataset_id + [INTEGER] + NOT NULL + +target_dag_id + [VARCHAR(250)] + NOT NULL + +created_at + [TIMESTAMP] + NOT NULL dataset--dataset_dag_run_queue - -1 -1 + +1 +1 dag - -dag + +dag + +dag_id + [VARCHAR(250)] + NOT NULL -dag_id - [VARCHAR(250)] - NOT NULL +dag_display_name + [VARCHAR(2000)] dataset_expression [JSON] @@ -743,98 +746,98 @@ dag--dag_schedule_dataset_reference - -1 -1 + +1 +1 dag--task_outlet_dataset_reference - -1 -1 + +1 +1 dag--dataset_dag_run_queue - -1 -1 + +1 +1 dag_tag - -dag_tag - -dag_id - [VARCHAR(250)] - NOT NULL - -name - [VARCHAR(100)] - NOT NULL + +dag_tag + +dag_id + [VARCHAR(250)] + NOT NULL + +name + [VARCHAR(100)] + NOT NULL dag--dag_tag - -1 -1 + +1 +1 dag_owner_attributes - -dag_owner_attributes - -dag_id - [VARCHAR(250)] - NOT NULL - -owner - [VARCHAR(500)] - NOT NULL - -link - [VARCHAR(500)] - NOT NULL + +dag_owner_attributes + +dag_id + [VARCHAR(250)] + NOT NULL + +owner + [VARCHAR(500)] + NOT NULL + +link + [VARCHAR(500)] + NOT NULL dag--dag_owner_attributes - -1 -1 + +1 +1 dag_warning - -dag_warning - -dag_id - [VARCHAR(250)] - NOT NULL - -warning_type - [VARCHAR(50)] - NOT NULL - -message - [TEXT] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL + +dag_warning + +dag_id + [VARCHAR(250)] + NOT NULL + +warning_type + [VARCHAR(50)] + NOT NULL + +message + [TEXT] + NOT NULL + +timestamp + [TIMESTAMP] + NOT NULL dag--dag_warning - -1 -1 + +1 +1 @@ -937,9 +940,9 @@ dag_run--dag_run_note - -1 -1 + +1 +1 @@ -958,483 +961,486 @@ dag_run--dagrun_dataset_event - -1 -1 + +1 +1 task_instance - -task_instance - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -custom_operator_name - [VARCHAR(1000)] - -duration - [DOUBLE_PRECISION] - -end_date - [TIMESTAMP] - -executor_config - [BYTEA] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -rendered_map_index - [VARCHAR(250)] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -trigger_id - [INTEGER] - -trigger_timeout - [TIMESTAMP] - -try_number - [INTEGER] - -unixname - [VARCHAR(1000)] - -updated_at - [TIMESTAMP] + +task_instance + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +custom_operator_name + [VARCHAR(1000)] + +duration + [DOUBLE_PRECISION] + +end_date + [TIMESTAMP] + +executor_config + [BYTEA] + +external_executor_id + [VARCHAR(250)] + +hostname + [VARCHAR(1000)] + +job_id + [INTEGER] + +max_tries + [INTEGER] + +next_kwargs + [JSON] + +next_method + [VARCHAR(1000)] + +operator + [VARCHAR(1000)] + +pid + [INTEGER] + +pool + [VARCHAR(256)] + NOT NULL + +pool_slots + [INTEGER] + NOT NULL + +priority_weight + [INTEGER] + +queue + [VARCHAR(256)] + +queued_by_job_id + [INTEGER] + +queued_dttm + [TIMESTAMP] + +rendered_map_index + [VARCHAR(250)] + +start_date + [TIMESTAMP] + +state + [VARCHAR(20)] + +task_display_name + [VARCHAR(2000)] + +trigger_id + [INTEGER] + +trigger_timeout + [TIMESTAMP] + +try_number + [INTEGER] + +unixname + [VARCHAR(1000)] + +updated_at + [TIMESTAMP] dag_run--task_instance - -1 -1 + +1 +1 dag_run--task_instance - -1 -1 + +1 +1 task_reschedule - -task_reschedule - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - NOT NULL - -end_date - [TIMESTAMP] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -reschedule_date - [TIMESTAMP] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -try_number - [INTEGER] - NOT NULL + +task_reschedule + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + NOT NULL + +end_date + [TIMESTAMP] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +reschedule_date + [TIMESTAMP] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +try_number + [INTEGER] + NOT NULL dag_run--task_reschedule - -0..N -1 + +0..N +1 dag_run--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_instance_note - -1 -1 + +1 +1 task_instance--task_instance_note - -1 -1 + +1 +1 task_instance--task_instance_note - -1 -1 + +1 +1 task_instance--task_instance_note - -1 -1 + +1 +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_instance--task_reschedule - -0..N -1 + +0..N +1 task_fail - -task_fail - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - -end_date - [TIMESTAMP] - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - -task_id - [VARCHAR(250)] - NOT NULL + +task_fail + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + +end_date + [TIMESTAMP] + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + +task_id + [VARCHAR(250)] + NOT NULL task_instance--task_fail - -0..N -1 + +0..N +1 task_instance--task_fail - -0..N -1 + +0..N +1 task_instance--task_fail - -0..N -1 + +0..N +1 task_instance--task_fail - -0..N -1 + +0..N +1 task_map - -task_map - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -keys - [JSON] - -length - [INTEGER] - NOT NULL + +task_map + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +keys + [JSON] + +length + [INTEGER] + NOT NULL task_instance--task_map - -1 -1 + +1 +1 task_instance--task_map - -1 -1 + +1 +1 task_instance--task_map - -1 -1 + +1 +1 task_instance--task_map - -1 -1 + +1 +1 xcom - -xcom - -dag_run_id - [INTEGER] - NOT NULL - -key - [VARCHAR(512)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - -value - [BYTEA] + +xcom + +dag_run_id + [INTEGER] + NOT NULL + +key + [VARCHAR(512)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +timestamp + [TIMESTAMP] + NOT NULL + +value + [BYTEA] task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -1 -1 + +1 +1 task_instance--xcom - -0..N -1 + +0..N +1 task_instance--xcom - -1 -1 + +1 +1 rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - [JSON] - -rendered_fields - [JSON] - NOT NULL + +rendered_task_instance_fields + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + [JSON] + +rendered_fields + [JSON] + NOT NULL task_instance--rendered_task_instance_fields - -1 -1 + +1 +1 task_instance--rendered_task_instance_fields - -1 -1 + +1 +1 task_instance--rendered_task_instance_fields - -1 -1 + +1 +1 task_instance--rendered_task_instance_fields - -1 -1 + +1 +1 @@ -1614,38 +1620,38 @@ trigger--task_instance - -0..N -{0,1} + +0..N +{0,1} session - -session - -id - [INTEGER] - NOT NULL - -data - [BYTEA] - -expiry - [TIMESTAMP] - -session_id - [VARCHAR(255)] + +session + +id + [INTEGER] + NOT NULL + +data + [BYTEA] + +expiry + [TIMESTAMP] + +session_id + [VARCHAR(255)] alembic_version - -alembic_version - -version_num - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + [VARCHAR(32)] + NOT NULL diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 63096ce8b7470..a28f2ee766907 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +| ``ee1467d4aa35`` (head) | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | | | | | table | diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 30f87e1f68097..8578f633cf6bb 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -181,6 +181,7 @@ def test_should_respond_200(self): assert response.status_code == 200 assert { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": "Ii90bXAvZGFnXzEucHki.EnmIdPaUPo26lHQClbWMbDFD1Pk", @@ -223,6 +224,7 @@ def test_should_respond_200_with_schedule_interval_none(self, session): assert response.status_code == 200 assert { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": "Ii90bXAvZGFnXzEucHki.EnmIdPaUPo26lHQClbWMbDFD1Pk", @@ -329,6 +331,7 @@ def test_should_respond_200(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "dataset_expression": None, "default_view": None, @@ -389,6 +392,7 @@ def test_should_respond_200_with_dataset_expression(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "dataset_expression": { "any": [ @@ -454,6 +458,7 @@ def test_should_response_200_with_doc_md_none(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag2", + "dag_display_name": "test_dag2", "dag_run_timeout": None, "dataset_expression": None, "default_view": None, @@ -507,6 +512,7 @@ def test_should_response_200_for_null_start_date(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag3", + "dag_display_name": "test_dag3", "dag_run_timeout": None, "dataset_expression": None, "default_view": None, @@ -563,6 +569,7 @@ def test_should_respond_200_serialized(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "dataset_expression": None, "default_view": None, @@ -626,6 +633,7 @@ def test_should_respond_200_serialized(self, url_safe_serializer): "catchup": True, "concurrency": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "dataset_expression": None, "default_view": None, @@ -739,6 +747,7 @@ def test_should_respond_200(self, session, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -771,6 +780,7 @@ def test_should_respond_200(self, session, url_safe_serializer): }, { "dag_id": "TEST_DAG_2", + "dag_display_name": "TEST_DAG_2", "description": None, "fileloc": "/tmp/dag_2.py", "file_token": file_token2, @@ -815,6 +825,7 @@ def test_only_active_true_returns_active_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -860,6 +871,7 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -892,6 +904,7 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer): }, { "dag_id": "TEST_DAG_DELETED_1", + "dag_display_name": "TEST_DAG_DELETED_1", "description": None, "fileloc": "/tmp/dag_del_1.py", "file_token": file_token_2, @@ -1062,6 +1075,7 @@ def test_paused_true_returns_paused_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_PAUSED_1", + "dag_display_name": "TEST_DAG_PAUSED_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1106,6 +1120,7 @@ def test_paused_false_returns_unpaused_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_UNPAUSED_1", + "dag_display_name": "TEST_DAG_UNPAUSED_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1150,6 +1165,7 @@ def test_paused_none_returns_all_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_PAUSED_1", + "dag_display_name": "TEST_DAG_PAUSED_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1182,6 +1198,7 @@ def test_paused_none_returns_all_dags(self, url_safe_serializer): }, { "dag_id": "TEST_DAG_UNPAUSED_1", + "dag_display_name": "TEST_DAG_UNPAUSED_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1256,6 +1273,7 @@ def test_should_respond_200_on_patch_is_paused(self, url_safe_serializer, sessio assert response.status_code == 200 expected_response = { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1392,6 +1410,7 @@ def test_should_respond_200_with_update_mask(self, url_safe_serializer): assert response.status_code == 200 expected_response = { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1491,6 +1510,7 @@ def test_should_respond_200_on_patch_is_paused(self, session, url_safe_serialize "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1523,6 +1543,7 @@ def test_should_respond_200_on_patch_is_paused(self, session, url_safe_serialize }, { "dag_id": "TEST_DAG_2", + "dag_display_name": "TEST_DAG_2", "description": None, "fileloc": "/tmp/dag_2.py", "file_token": file_token2, @@ -1580,6 +1601,7 @@ def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1612,6 +1634,7 @@ def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, }, { "dag_id": "TEST_DAG_2", + "dag_display_name": "TEST_DAG_2", "description": None, "fileloc": "/tmp/dag_2.py", "file_token": file_token2, @@ -1709,6 +1732,7 @@ def test_only_active_true_returns_active_dags(self, url_safe_serializer, session "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1762,6 +1786,7 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -1794,6 +1819,7 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): }, { "dag_id": "TEST_DAG_DELETED_1", + "dag_display_name": "TEST_DAG_DELETED_1", "description": None, "fileloc": "/tmp/dag_del_1.py", "file_token": file_token_2, @@ -2010,6 +2036,7 @@ def test_should_respond_200_and_pause_dags(self, url_safe_serializer): "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -2042,6 +2069,7 @@ def test_should_respond_200_and_pause_dags(self, url_safe_serializer): }, { "dag_id": "TEST_DAG_2", + "dag_display_name": "TEST_DAG_2", "description": None, "fileloc": "/tmp/dag_2.py", "file_token": file_token2, @@ -2095,6 +2123,7 @@ def test_should_respond_200_and_pause_dag_pattern(self, session, url_safe_serial "dags": [ { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, @@ -2127,6 +2156,7 @@ def test_should_respond_200_and_pause_dag_pattern(self, session, url_safe_serial }, { "dag_id": "TEST_DAG_10", + "dag_display_name": "TEST_DAG_10", "description": None, "fileloc": "/tmp/dag_10.py", "file_token": file_token10, @@ -2182,6 +2212,7 @@ def test_should_respond_200_and_reverse_ordering(self, session, url_safe_seriali "dags": [ { "dag_id": "TEST_DAG_2", + "dag_display_name": "TEST_DAG_2", "description": None, "fileloc": "/tmp/dag_2.py", "file_token": file_token10, @@ -2214,6 +2245,7 @@ def test_should_respond_200_and_reverse_ordering(self, session, url_safe_seriali }, { "dag_id": "TEST_DAG_1", + "dag_display_name": "TEST_DAG_1", "description": None, "fileloc": "/tmp/dag_1.py", "file_token": file_token, diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index 30c227ea9a15d..8d5c854eb4d83 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -239,6 +239,7 @@ def test_mapped_task_instances(self, one_task_with_mapped_tis, session): "start_date": "2020-01-01T00:00:00+00:00", "state": "success", "task_id": "task_2", + "task_display_name": "task_2", "try_number": 0, "unixname": getuser(), "trigger": None, diff --git a/tests/api_connexion/endpoints/test_task_endpoint.py b/tests/api_connexion/endpoints/test_task_endpoint.py index b8ef8dc0cf650..454b0db7525d1 100644 --- a/tests/api_connexion/endpoints/test_task_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_endpoint.py @@ -129,6 +129,7 @@ def test_should_respond_200(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": "op1", + "task_display_name": "op1", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -164,6 +165,7 @@ def test_mapped_task(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": "mapped_task", + "task_display_name": "mapped_task", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -215,6 +217,7 @@ def test_should_respond_200_serialized(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": "op1", + "task_display_name": "op1", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -290,6 +293,7 @@ def test_should_respond_200(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": "op1", + "task_display_name": "op1", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -320,6 +324,7 @@ def test_should_respond_200(self): "retry_exponential_backoff": False, "start_date": "2020-06-16T00:00:00+00:00", "task_id": self.task_id2, + "task_display_name": self.task_id2, "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -360,6 +365,7 @@ def test_get_tasks_mapped(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": "mapped_task", + "task_display_name": "mapped_task", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -389,6 +395,7 @@ def test_get_tasks_mapped(self): "retry_exponential_backoff": False, "start_date": "2020-06-15T00:00:00+00:00", "task_id": self.task_id3, + "task_display_name": self.task_id3, "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index e6a3e0dfbb547..26f573be1e3e2 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -243,6 +243,7 @@ def test_should_respond_200(self, username, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -300,6 +301,7 @@ def test_should_respond_200_with_task_state_in_deferred(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "deferred", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -346,6 +348,7 @@ def test_should_respond_200_with_task_state_in_removed(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "removed", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -403,6 +406,7 @@ def test_should_respond_200_task_instance_with_sla_and_rendered(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -454,6 +458,7 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -2378,6 +2383,7 @@ def test_should_respond_200(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", @@ -2436,6 +2442,7 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "print_the_context", + "task_display_name": "print_the_context", "try_number": 0, "unixname": getuser(), "dag_run_id": "TEST_DAG_RUN_ID", diff --git a/tests/api_connexion/schemas/test_dag_schema.py b/tests/api_connexion/schemas/test_dag_schema.py index b5be848165e1d..d6dfeb759a718 100644 --- a/tests/api_connexion/schemas/test_dag_schema.py +++ b/tests/api_connexion/schemas/test_dag_schema.py @@ -51,6 +51,7 @@ def test_serialize_test_dag_schema(url_safe_serializer): assert { "dag_id": "test_dag_id", + "dag_display_name": "test_dag_id", "description": "The description", "fileloc": "/root/airflow/dags/my_dag.py", "file_token": url_safe_serializer.dumps("/root/airflow/dags/my_dag.py"), @@ -89,6 +90,7 @@ def test_serialize_test_dag_collection_schema(url_safe_serializer): "dags": [ { "dag_id": "test_dag_id_a", + "dag_display_name": "test_dag_id_a", "description": None, "fileloc": "/tmp/a.py", "file_token": url_safe_serializer.dumps("/tmp/a.py"), @@ -118,6 +120,7 @@ def test_serialize_test_dag_collection_schema(url_safe_serializer): }, { "dag_id": "test_dag_id_b", + "dag_display_name": "test_dag_id_b", "description": None, "fileloc": "/tmp/a.py", "file_token": url_safe_serializer.dumps("/tmp/a.py"), @@ -168,6 +171,7 @@ def test_serialize_test_dag_detail_schema(url_safe_serializer): "concurrency": 16, "max_active_tasks": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "default_view": "duration", "description": None, @@ -225,6 +229,7 @@ def test_serialize_test_dag_with_dataset_schedule_detail_schema(url_safe_seriali "concurrency": 16, "max_active_tasks": 16, "dag_id": "test_dag", + "dag_display_name": "test_dag", "dag_run_timeout": None, "default_view": "duration", "description": None, diff --git a/tests/api_connexion/schemas/test_task_instance_schema.py b/tests/api_connexion/schemas/test_task_instance_schema.py index ad2b49a8606ac..2e774bec6d487 100644 --- a/tests/api_connexion/schemas/test_task_instance_schema.py +++ b/tests/api_connexion/schemas/test_task_instance_schema.py @@ -89,6 +89,7 @@ def test_task_instance_schema_without_sla_and_rendered(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "TEST_TASK_ID", + "task_display_name": "TEST_TASK_ID", "try_number": 0, "unixname": getuser(), "dag_run_id": None, @@ -143,6 +144,7 @@ def test_task_instance_schema_with_sla_and_rendered(self, session): "start_date": "2020-01-02T00:00:00+00:00", "state": "running", "task_id": "TEST_TASK_ID", + "task_display_name": "TEST_TASK_ID", "try_number": 0, "unixname": getuser(), "dag_run_id": None, diff --git a/tests/api_connexion/schemas/test_task_schema.py b/tests/api_connexion/schemas/test_task_schema.py index 54403ebbf0bf2..e7abd814b8605 100644 --- a/tests/api_connexion/schemas/test_task_schema.py +++ b/tests/api_connexion/schemas/test_task_schema.py @@ -52,6 +52,7 @@ def test_serialize(self): "retry_exponential_backoff": False, "start_date": "2020-06-16T00:00:00+00:00", "task_id": "task_id", + "task_display_name": "task_id", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", @@ -99,6 +100,7 @@ def test_serialize(self): "retry_exponential_backoff": False, "start_date": None, "task_id": "task_id1", + "task_display_name": "task_id1", "template_fields": [], "trigger_rule": "all_success", "ui_color": "#e8f7e4", diff --git a/tests/conftest.py b/tests/conftest.py index adba65c7746ae..e31bd09abad2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -911,6 +911,7 @@ def create_dummy_dag(dag_maker): def create_dag( dag_id="dag", task_id="op1", + task_display_name=None, max_active_tis_per_dag=16, max_active_tis_per_dagrun=None, pool="default_pool", @@ -927,6 +928,7 @@ def create_dag( with dag_maker(dag_id, **kwargs) as dag: op = EmptyOperator( task_id=task_id, + task_display_name=task_display_name, max_active_tis_per_dag=max_active_tis_per_dag, max_active_tis_per_dagrun=max_active_tis_per_dagrun, executor_config=executor_config or {}, diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 02069d382c747..120856dbbfd24 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -3227,7 +3227,7 @@ def test_set_state_up_for_retry(self, create_task_instance): def test_refresh_from_db(self, create_task_instance): run_date = timezone.utcnow() - + hybrid_props = ["task_display_name"] expected_values = { "task_id": "test_refresh_from_db_task", "dag_id": "test_refresh_from_db_dag", @@ -3259,6 +3259,7 @@ def test_refresh_from_db(self, create_task_instance): "next_kwargs": None, "next_method": None, "updated_at": None, + "task_display_name": "Test Refresh from DB Task", } # Make sure we aren't missing any new value in our expected_values list. expected_keys = {f"task_instance.{key.lstrip('_')}" for key in expected_values} @@ -3267,9 +3268,14 @@ def test_refresh_from_db(self, create_task_instance): "This prevents refresh_from_db() from missing a field." ) - ti = create_task_instance(task_id=expected_values["task_id"], dag_id=expected_values["dag_id"]) + ti = create_task_instance( + task_id=expected_values["task_id"], + task_display_name=expected_values["task_display_name"], + dag_id=expected_values["dag_id"], + ) for key, expected_value in expected_values.items(): - setattr(ti, key, expected_value) + if key not in hybrid_props: + setattr(ti, key, expected_value) with create_session() as session: session.merge(ti) session.commit() @@ -3282,9 +3288,10 @@ def test_refresh_from_db(self, create_task_instance): ti.refresh_from_db() for key, expected_value in expected_values.items(): assert hasattr(ti, key), f"Key {key} is missing in the TaskInstance." - assert ( - getattr(ti, key) == expected_value - ), f"Key: {key} had different values. Make sure it loads it in the refresh refresh_from_db()" + if key not in hybrid_props: + assert ( + getattr(ti, key) == expected_value + ), f"Key: {key} had different values. Make sure it loads it in the refresh refresh_from_db()" def test_operator_field_with_serialization(self, create_task_instance): ti = create_task_instance() diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index aee0cad6cf465..4b5632cd20f8a 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -1252,6 +1252,7 @@ def test_no_new_fields_added_to_base_operator(self): "_log_config_logger_name": "airflow.task.operators", "_post_execute_hook": None, "_pre_execute_hook": None, + "_task_display_property_value": None, "allow_nested_operators": True, "depends_on_past": False, "do_xcom_push": True, diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 4497b3955afe3..4c57f8026e6b1 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -115,14 +115,16 @@ def test_should_render_dag_with_task_instances(self, session, dag_maker): def test_should_render_dag_with_mapped_operator(self, session, dag_maker): with dag_maker(dag_id="DAG_ID", session=session) as dag: - BashOperator.partial(task_id="first").expand(bash_command=["echo hello", "echo world"]) + BashOperator.partial(task_id="first", task_display_name="First Task").expand( + bash_command=["echo hello", "echo world"] + ) dot = dot_renderer.render_dag(dag) source = dot.source # Should render DAG title assert "label=DAG_ID" in source assert ( - 'first [color="#000000" fillcolor="#f0ede4" label=first shape=rectangle style="filled,rounded"]' + 'first [color="#000000" fillcolor="#f0ede4" label="First Task" shape=rectangle style="filled,rounded"]' in source ) diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 61661e89419aa..90cc5bbc1c67a 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -1063,6 +1063,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "also_run_this", "task_id": "also_run_this", "trigger_id": None, "trigger_timeout": None, @@ -1096,6 +1097,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "run_after_loop", "task_id": "run_after_loop", "trigger_id": None, "trigger_timeout": None, @@ -1129,6 +1131,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "run_this_last", "task_id": "run_this_last", "trigger_id": None, "trigger_timeout": None, @@ -1162,6 +1165,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "runme_0", "task_id": "runme_0", "trigger_id": None, "trigger_timeout": None, @@ -1195,6 +1199,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "runme_1", "task_id": "runme_1", "trigger_id": None, "trigger_timeout": None, @@ -1228,6 +1233,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "runme_2", "task_id": "runme_2", "trigger_id": None, "trigger_timeout": None, @@ -1261,6 +1267,7 @@ def test_task_instances(admin_client): "run_id": "TEST_DAGRUN", "start_date": None, "state": None, + "task_display_name": "this_will_skip", "task_id": "this_will_skip", "trigger_id": None, "trigger_timeout": None,