[AIRFLOW-6949] Respect explicit spark.kubernetes.namespace conf to …

…SparkSubmitOperator (#7575)

This means the value from the Operator/dag file takes precedence over
the connection

The previous behaviour was to emit one line from the conf arg, but then
a later one from the connection:

```
--conf spark.kubernetes.namespace=airflow \
--conf spark.kubernetes.namespace=default \
```

(cherry picked from commit b59042b)

airflow/contrib/hooks/spark_submit_hook.py

@@ -126,7 +126,7 @@ def __init__(self,
                  env_vars=None,
                  verbose=False,
                  spark_binary=None):
-        self._conf = conf
+        self._conf = conf or {}
         self._conn_id = conn_id
         self._files = files
         self._py_files = py_files
@@ -248,9 +248,11 @@ def _build_spark_submit_command(self, application):
         # The url of the spark master
         connection_cmd += ["--master", self._connection['master']]
 
-        if self._conf:
-            for key in self._conf:
-                connection_cmd += ["--conf", "{}={}".format(key, str(self._conf[key]))]
+        if 'spark.kubernetes.namespace' in self._conf:
+            conn_data['namespace'] = self._conf['spark.kubernetes.namespace']
+
+        for key in self._conf:
+            connection_cmd += ["--conf", "{}={}".format(key, str(self._conf[key]))]
         if self._env_vars and (self._is_kubernetes or self._is_yarn):
             if self._is_yarn:
                 tmpl = "spark.yarn.appMasterEnv.{}={}"