diff --git a/airflow/contrib/hooks/spark_submit_hook.py b/airflow/contrib/hooks/spark_submit_hook.py index 22900d8e6d865..15fe9677380ab 100644 --- a/airflow/contrib/hooks/spark_submit_hook.py +++ b/airflow/contrib/hooks/spark_submit_hook.py @@ -126,7 +126,7 @@ def __init__(self, env_vars=None, verbose=False, spark_binary=None): - self._conf = conf + self._conf = conf or {} self._conn_id = conn_id self._files = files self._py_files = py_files @@ -248,9 +248,11 @@ def _build_spark_submit_command(self, application): # The url of the spark master connection_cmd += ["--master", self._connection['master']] - if self._conf: - for key in self._conf: - connection_cmd += ["--conf", "{}={}".format(key, str(self._conf[key]))] + if 'spark.kubernetes.namespace' in self._conf: + conn_data['namespace'] = self._conf['spark.kubernetes.namespace'] + + for key in self._conf: + connection_cmd += ["--conf", "{}={}".format(key, str(self._conf[key]))] if self._env_vars and (self._is_kubernetes or self._is_yarn): if self._is_yarn: tmpl = "spark.yarn.appMasterEnv.{}={}"