improve handling of flag, added test for config error

apache · Jun 27, 2024 · e352203 · e352203
1 parent e91397a
commit e352203
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 11 deletions.
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
@@ -777,16 +777,15 @@ impl DefaultPhysicalPlanner {
                     .map(|s| (s.to_string(), arrow_schema::DataType::Null))
                     .collect::<Vec<_>>();
 
-                let keep_partition_by_columns = source_option_tuples
+                let keep_partition_by_columns = match source_option_tuples
                     .get("execution.keep_partition_by_columns")
-                    .map(|v| v.trim() == "true")
-                    .unwrap_or(
-                        session_state
-                            .config()
-                            .options()
-                            .execution
-                            .keep_partition_by_columns,
-                    );
+                    .map(|v| v.trim()) {
+                    None => session_state.config().options().execution.keep_partition_by_columns,
+                    Some("true") => true,
+                    Some("false") => false,
+                    Some(value) =>
+                        return Err(DataFusionError::Configuration(format!("provided value for 'execution.keep_partition_by_columns' was not recognized: \"{}\"", value))),
+                };
 
                 // Set file sink related options
                 let config = FileSinkConfig {

diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs
@@ -1475,7 +1475,7 @@ mod tests {
     fn copy_to_multi_options() -> Result<(), ParserError> {
         // order of options is preserved
         let sql =
-            "COPY foo TO bar STORED AS parquet OPTIONS ('format.row_group_size' 55, 'format.compression' snappy, 'keep_partition_by_columns' true)";
+            "COPY foo TO bar STORED AS parquet OPTIONS ('format.row_group_size' 55, 'format.compression' snappy, 'execution.keep_partition_by_columns' true)";
 
         let expected_options = vec![
             (
@@ -1487,7 +1487,7 @@ mod tests {
                 Value::SingleQuotedString("snappy".to_string()),
             ),
             (
-                "keep_partition_by_columns".to_string(),
+                "execution.keep_partition_by_columns".to_string(),
                 Value::SingleQuotedString("true".to_string()),
             ),
         ];

diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt
@@ -606,3 +606,7 @@ COPY (select col2, sum(col1) from source_table
 # Copy from table with non literal
 query error DataFusion error: SQL error: ParserError\("Unexpected token \("\)
 COPY source_table  to '/tmp/table.parquet' (row_group_size 55 + 102);
+
+# Copy using execution.keep_partition_by_columns with an invalid value
+query error DataFusion error: Invalid or Unsupported Configuration: provided value for 'execution.keep_partition_by_columns' was not recognized: "invalid_value"
+COPY source_table  to '/tmp/table.parquet' OPTIONS (execution.keep_partition_by_columns invalid_value);