Fixing compilation (for Spark 2.4)

apache · Jul 26, 2022 · 6194091 · 6194091
1 parent 41e45cb
commit 6194091
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 19 deletions.
diff --git a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
@@ -40,7 +40,6 @@
 import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.hadoop.CachingPath;
 import org.apache.hudi.metadata.HoodieTableMetadata;
-import org.apache.hudi.hadoop.CachingPath;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 
@@ -403,7 +402,7 @@ Path fullPartitionPath(Path basePath) {
         return new CachingPath(basePath, createPathUnsafe(path));
       }
 
-      return new CachingPath(basePath);
+      return basePath;
     }
 
     @Override

diff --git a/...tasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala b/...tasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
@@ -20,7 +20,8 @@ package org.apache.hudi
 import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hudi.BaseHoodieTableFileIndex.PartitionPath
 import org.apache.hudi.DataSourceReadOptions.{QUERY_TYPE, QUERY_TYPE_INCREMENTAL_OPT_VAL, QUERY_TYPE_READ_OPTIMIZED_OPT_VAL, QUERY_TYPE_SNAPSHOT_OPT_VAL}
-import org.apache.hudi.SparkHoodieTableFileIndex.{deduceQueryType, generateFieldMap, toJavaOption}
+import org.apache.hudi.HoodieConversionUtils.toJavaOption
+import org.apache.hudi.SparkHoodieTableFileIndex.{deduceQueryType, generateFieldMap, shouldValidatePartitionColumns}
 import org.apache.hudi.client.common.HoodieSparkEngineContext
 import org.apache.hudi.common.bootstrap.index.BootstrapIndex
 import org.apache.hudi.common.config.TypedProperties
@@ -41,6 +42,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
 import org.apache.spark.unsafe.types.UTF8String
 
 import scala.collection.JavaConverters._
+import scala.language.implicitConversions
 
 /**
  * Implementation of the [[BaseHoodieTableFileIndex]] for Spark
@@ -247,14 +249,14 @@ class SparkHoodieTableFileIndex(spark: SparkSession,
         // HIVE_STYLE_PARTITIONING is disable.
         // e.g. convert "/xx/xx/2021/02" to "/xx/xx/year=2021/month=02"
         val partitionWithName =
-          partitionFragments.zip(partitionColumns).map {
-            case (partition, columnName) =>
-              if (partition.indexOf("=") == -1) {
-                s"${columnName}=$partition"
-              } else {
-                partition
-              }
-          }.mkString("/")
+        partitionFragments.zip(partitionColumns).map {
+          case (partition, columnName) =>
+            if (partition.indexOf("=") == -1) {
+              s"${columnName}=$partition"
+            } else {
+              partition
+            }
+        }.mkString("/")
 
         val pathWithPartitionName = new CachingPath(basePath, createPathUnsafe(partitionWithName))
         val partitionValues = parsePartitionPath(pathWithPartitionName, partitionSchema)
@@ -274,20 +276,14 @@ class SparkHoodieTableFileIndex(spark: SparkSession,
       Set(basePath),
       partitionDataTypes,
       DateTimeUtils.getTimeZone(timeZoneId),
-      validatePartitionValues = spark.sessionState.conf.validatePartitionColumns
+      validatePartitionValues = shouldValidatePartitionColumns(spark)
     )
       .toSeq(partitionSchema)
   }
 }
 
-object SparkHoodieTableFileIndex {
 
-  implicit def toJavaOption[T](opt: Option[T]): org.apache.hudi.common.util.Option[T] =
-    if (opt.isDefined) {
-      org.apache.hudi.common.util.Option.of(opt.get)
-    } else {
-      org.apache.hudi.common.util.Option.empty()
-    }
+object SparkHoodieTableFileIndex {
 
   /**
    * This method unravels [[StructType]] into a [[Map]] of pairs of dot-path notation with corresponding
@@ -344,4 +340,9 @@ object SparkHoodieTableFileIndex {
       override def invalidate(): Unit = cache.invalidateAll()
     }
   }
+
+  private def shouldValidatePartitionColumns(spark: SparkSession): Boolean = {
+    // NOTE: We can't use helper, method nor the config-entry to stay compatible w/ Spark 2.4
+    spark.sessionState.conf.getConfString("spark.sql.sources.validatePartitionColumns", "true").toBoolean
+  }
 }