Handle review comments

apache · Aug 26, 2020 · 84f7e95 · 84f7e95
1 parent 6cf355a
commit 84f7e95
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -270,7 +270,8 @@ private[spark] object Utils extends Logging {
   }
 
   /**
-   * Move data to trash if 'spark.sql.truncate.trash.enabled' is true
+   * Move data to trash if 'spark.sql.truncate.trash.enabled' is true, else
+   * delete the data permanently. If move data to trash failed fallback to hard deletion. 
    */
   def moveToTrashIfEnabled(
       fs: FileSystem,
@@ -281,7 +282,8 @@ private[spark] object Utils extends Logging {
       logDebug(s"will move data ${partitionPath.toString} to trash")
       val isSuccess = Trash.moveToAppropriateTrash(fs, partitionPath, hadoopConf)
       if (!isSuccess) {
-        logWarning(s"Failed to move data ${partitionPath.toString} to trash")
+        logWarning(s"Failed to move data ${partitionPath.toString} to trash. " +
+          "Fallback to hard deletion")
         return fs.delete(partitionPath, true)
       }
       isSuccess

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2728,7 +2728,8 @@ object SQLConf {
         "to trash directory or deleted permanently. The trash retention time is controlled by " +
         "fs.trash.interval, and in default, the server side configuration value takes " +
         "precedence over the client-side one. Note that if fs.trash.interval is non-positive, " +
-        "this will be a no-op and log a warning message.")
+        "this will be a no-op and log a warning message. If the data fails to be moved to "  +
+        "trash, Spark will turn to delete it permanently.")
       .version("3.1.0")
       .booleanConf
       .createWithDefault(false)