do not add hive_options to CopyTo

apache · Jun 25, 2024 · 1578273 · 1578273
1 parent 43b931c
commit 1578273
Show file tree

Hide file tree

Showing 10 changed files with 8 additions and 24 deletions.
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
@@ -1285,6 +1285,11 @@ impl TableOptions {
             return ConfigField::set(self, key, value);
         }
 
+        // Only used for hive.keep_partition_by_columns
+        if prefix == "hive" {
+            return Ok(());
+        }
+
         let Some(e) = self.extensions.0.get_mut(prefix) else {
             return _config_err!("Could not find config namespace \"{prefix}\"");
         };

diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
@@ -1338,7 +1338,6 @@ impl DataFrame {
             FormatOptions::CSV(props),
             HashMap::new(),
             options.partition_by,
-            Default::default(),
         )?
         .build()?;
 
@@ -1394,7 +1393,6 @@ impl DataFrame {
             FormatOptions::JSON(props),
             Default::default(),
             options.partition_by,
-            Default::default(),
         )?
         .build()?;
 

diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rs
@@ -66,7 +66,6 @@ impl DataFrame {
             FormatOptions::PARQUET(props),
             Default::default(),
             options.partition_by,
-            Default::default(),
         )?
         .build()?;
         DataFrame {

diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
@@ -762,7 +762,6 @@ impl DefaultPhysicalPlanner {
                 format_options,
                 partition_by,
                 options: source_option_tuples,
-                hive_options
             }) => {
                 let input_exec = children.one()?;
                 let parsed_url = ListingTableUrl::parse(output_url)?;
@@ -778,7 +777,7 @@ impl DefaultPhysicalPlanner {
                     .map(|s| (s.to_string(), arrow_schema::DataType::Null))
                     .collect::<Vec<_>>();
 
-                let keep_partition_by_columns = hive_options
+                let keep_partition_by_columns = source_option_tuples
                     .get("hive.keep_partition_by_columns")
                     .map(|v| v.trim() == "true")
                     .unwrap_or(false) || session_state.config().options().execution.keep_partition_by_columns;

diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
@@ -274,15 +274,13 @@ impl LogicalPlanBuilder {
         format_options: FormatOptions,
         options: HashMap<String, String>,
         partition_by: Vec<String>,
-        hive_options: HashMap<String, String>,
     ) -> Result<Self> {
         Ok(Self::from(LogicalPlan::Copy(CopyTo {
             input: Arc::new(input),
             output_url,
             partition_by,
             format_options,
             options,
-            hive_options,
         })))
     }
 

diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs
@@ -428,24 +428,17 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {
                 format_options,
                 partition_by: _,
                 options,
-                hive_options,
             }) => {
                 let op_str = options
                     .iter()
                     .map(|(k, v)| format!("{}={}", k, v))
                     .collect::<Vec<_>>()
                     .join(", ");
-                let hive_op_str = hive_options
-                    .iter()
-                    .map(|(k, v)| format!("{}={}", k, v))
-                    .collect::<Vec<_>>()
-                    .join(", ");
                 json!({
                     "Node Type": "CopyTo",
                     "Output URL": output_url,
                     "Format Options": format!("{}", format_options),
-                    "Options": op_str,
-                    "Hive Options": hive_op_str,
+                    "Options": op_str
                 })
             }
             LogicalPlan::Ddl(ddl) => {

diff --git a/datafusion/expr/src/logical_plan/dml.rs b/datafusion/expr/src/logical_plan/dml.rs
@@ -39,8 +39,6 @@ pub struct CopyTo {
     pub format_options: FormatOptions,
     /// SQL Options that can affect the formats
     pub options: HashMap<String, String>,
-    /// Hive Options that can affect hive-style partitioning
-    pub hive_options: HashMap<String, String>,
 }
 
 // Implement PartialEq manually

diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
@@ -860,14 +860,12 @@ impl LogicalPlan {
                 format_options,
                 options,
                 partition_by,
-                hive_options,
             }) => Ok(LogicalPlan::Copy(CopyTo {
                 input: Arc::new(inputs.swap_remove(0)),
                 output_url: output_url.clone(),
                 format_options: format_options.clone(),
                 options: options.clone(),
                 partition_by: partition_by.clone(),
-                hive_options: hive_options.clone(),
             })),
             LogicalPlan::Values(Values { schema, .. }) => {
                 Ok(LogicalPlan::Values(Values {

diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs
@@ -258,15 +258,13 @@ impl TreeNode for LogicalPlan {
                 partition_by,
                 format_options,
                 options,
-                hive_options,
             }) => rewrite_arc(input, f)?.update_data(|input| {
                 LogicalPlan::Copy(CopyTo {
                     input,
                     output_url,
                     partition_by,
                     format_options,
                     options,
-                    hive_options,
                 })
             }),
             LogicalPlan::Ddl(ddl) => {

diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
@@ -881,7 +881,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
         };
 
         let mut options = HashMap::new();
-        let mut hive_options = HashMap::new();
         for (key, value) in statement.options {
             let value_string = match value_to_string(&value) {
                 None => {
@@ -896,7 +895,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 } else {
                     key
                 };
-                hive_options.insert(renamed_key.to_lowercase(), value_string.to_lowercase());
+                options.insert(renamed_key.to_lowercase(), value_string.to_lowercase());
             } else if !(&key.contains('.')) {
                 // If config does not belong to any namespace, assume it is
                 // a format option and apply the format prefix for backwards
@@ -950,7 +949,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
             format_options: file_type.into(),
             partition_by,
             options,
-            hive_options,
         }))
     }