Support Spark 3.4.2, lift pyspark dependency range (#397)

oap-project · Jun 26, 2024 · cbce5df · cbce5df
1 parent 5f169e3
commit cbce5df
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
@@ -22,7 +22,8 @@ import com.intel.raydp.shims.{SparkShims, SparkShimDescriptor}
 object SparkShimProvider {
   val SPARK340_DESCRIPTOR = SparkShimDescriptor(3, 4, 0)
   val SPARK341_DESCRIPTOR = SparkShimDescriptor(3, 4, 1)
-  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR")
+  val SPARK342_DESCRIPTOR = SparkShimDescriptor(3, 4, 2)
+  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR", s"$SPARK342_DESCRIPTOR")
   val DESCRIPTOR = SPARK341_DESCRIPTOR
 }
 

diff --git a/python/setup.py b/python/setup.py
@@ -93,7 +93,7 @@ def run(self):
         "psutil",
         "pyarrow >= 4.0.1",
         "ray >= 2.1.0",
-        "pyspark >= 3.1.1, <= 3.4.1",
+        "pyspark >= 3.1.1, <= 3.5.0",
         "netifaces",
         "protobuf > 3.19.5, <= 3.20.3"
     ]