From c349a229308b9cad3af7bde4dd7d913e5de7a69c Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Fri, 5 Jun 2020 15:39:42 +0100 Subject: [PATCH] Add 'main' param to template_fields in DataprocSubmitPySparkJobOperator (#9154) (cherry picked from commit 9bcdadaf7e6e73d3d2246fbbd32a9f30a1b43ca9) --- airflow/contrib/operators/dataproc_operator.py | 4 ++-- tests/contrib/operators/test_dataproc_operator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index a5e126b2288cc0..ff07be59b79e83 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -1057,7 +1057,7 @@ class DataProcPySparkOperator(DataProcJobBaseOperator): Start a PySpark Job on a Cloud DataProc cluster. :param main: [Required] The Hadoop Compatible Filesystem (HCFS) URI of the main - Python file to use as the driver. Must be a .py file. + Python file to use as the driver. Must be a .py file. (templated) :type main: str :param arguments: Arguments for the job. (templated) :type arguments: list @@ -1077,7 +1077,7 @@ class DataProcPySparkOperator(DataProcJobBaseOperator): :type dataproc_pyspark_jars: list """ - template_fields = ['arguments', 'job_name', 'cluster_name', + template_fields = ['main', 'arguments', 'job_name', 'cluster_name', 'region', 'dataproc_jars', 'dataproc_properties'] ui_color = '#0273d4' job_type = 'pysparkJob' diff --git a/tests/contrib/operators/test_dataproc_operator.py b/tests/contrib/operators/test_dataproc_operator.py index d2beeb3d5c8009..7e21ffd967d378 100644 --- a/tests/contrib/operators/test_dataproc_operator.py +++ b/tests/contrib/operators/test_dataproc_operator.py @@ -1112,7 +1112,7 @@ def test_render_template(self): ) self.assertEqual( - task.template_fields, ['arguments', 'job_name', 'cluster_name', + task.template_fields, ['main', 'arguments', 'job_name', 'cluster_name', 'region', 'dataproc_jars', 'dataproc_properties']) ti = TaskInstance(task, DEFAULT_DATE)