change prefix to execution. , update override order for condition.

apache · Jun 26, 2024 · e91397a · e91397a
1 parent 89adc88
commit e91397a
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 25 deletions.
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
@@ -1285,8 +1285,7 @@ impl TableOptions {
             return ConfigField::set(self, key, value);
         }
 
-        // Only used for hive.keep_partition_by_columns
-        if prefix == "hive" {
+        if prefix == "execution" {
             return Ok(());
         }
 

diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
@@ -778,14 +778,15 @@ impl DefaultPhysicalPlanner {
                     .collect::<Vec<_>>();
 
                 let keep_partition_by_columns = source_option_tuples
-                    .get("hive.keep_partition_by_columns")
+                    .get("execution.keep_partition_by_columns")
                     .map(|v| v.trim() == "true")
-                    .unwrap_or(false)
-                    || session_state
-                        .config()
-                        .options()
-                        .execution
-                        .keep_partition_by_columns;
+                    .unwrap_or(
+                        session_state
+                            .config()
+                            .options()
+                            .execution
+                            .keep_partition_by_columns,
+                    );
 
                 // Set file sink related options
                 let config = FileSinkConfig {

diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
@@ -889,14 +889,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 Some(v) => v,
             };
 
-            if key.to_lowercase().contains("keep_partition_by_columns") {
-                let renamed_key = if !&key.starts_with("hive.") {
-                    format!("hive.{}", key)
-                } else {
-                    key
-                };
-                options.insert(renamed_key.to_lowercase(), value_string.to_lowercase());
-            } else if !(&key.contains('.')) {
+            if !(&key.contains('.')) {
                 // If config does not belong to any namespace, assume it is
                 // a format option and apply the format prefix for backwards
                 // compatibility.

diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt
@@ -169,7 +169,7 @@ physical_plan
 # Copy to directory as partitioned files with keep_partition_by_columns enabled
 query TT
 COPY (values ('1', 'a'), ('2', 'b'), ('3', 'c')) TO 'test_files/scratch/copy/partitioned_table4/' STORED AS parquet PARTITIONED BY (column1)
-OPTIONS (KEEP_PARTITION_BY_COLUMNS true);
+OPTIONS (execution.keep_partition_by_columns true);
 ----
 3
 

diff --git a/docs/source/user-guide/sql/dml.md b/docs/source/user-guide/sql/dml.md
@@ -41,8 +41,8 @@ clause is not specified, it will be inferred from the file extension if possible
 `PARTITIONED BY` specifies the columns to use for partitioning the output files into
 separate hive-style directories. By default, columns used in `PARTITIONED BY` will be removed
 from the output format. If you want to keep the columns, you should provide the option
-`KEEP_PARTITION_BY_COLUMNS true`. `KEEP_PARTITION_BY_COLUMNS` flag can also be enabled
-through `ExecutionOptions` within `SessionConfig`.
+`execution.keep_partition_by_columns true`. `execution.keep_partition_by_columns` flag can also 
+be enabled through `ExecutionOptions` within `SessionConfig`.
 
 The output format is determined by the first match of the following rules:
 

diff --git a/docs/source/user-guide/sql/write_options.md b/docs/source/user-guide/sql/write_options.md
@@ -70,15 +70,15 @@ In this example, we write the entirety of `source_table` out to a folder of parq
 
 ## Available Options
 
-### Hive Specific Options
+### Execution Specific Options
 
 The following options are available when writing hive-style partitioned data.
 
-| Option                    | Description                                                                        | Default Value |
-| ------------------------- | ---------------------------------------------------------------------------------- | ------------- |
-| KEEP_PARTITION_BY_COLUMNS | Flag to retain the columns in the output data when using `PARTITIONED BY` queries. | false         |
+| Option                              | Description                                                                        | Default Value |
+|-------------------------------------| ---------------------------------------------------------------------------------- | ------------- |
+| execution.keep_partition_by_columns | Flag to retain the columns in the output data when using `PARTITIONED BY` queries. | false         |
 
-Note: `KEEP_PARTITION_BY_COLUMNS` flag can also be enabled through `ExecutionOptions` within `SessionConfig`.
+Note: `execution.keep_partition_by_columns` flag can also be enabled through `ExecutionOptions` within `SessionConfig`.
 
 ### JSON Format Specific Options