Bump default Spark-version to 3.5.0

.github/workflows/add-to-project.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

.github/workflows/mvn-verify-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         java-version: [8, 11]
-        spark-version: ['311', '320', '333', '341']
+        spark-version: ['314', '325', '334', '350', '400']
     steps:
     - uses: actions/checkout@v4
 

.github/workflows/python-unit-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

.github/workflows/release.yml

@@ -1,3 +1,17 @@
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Release
 
 on:

.github/workflows/signoff-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

core/README.md

@@ -20,14 +20,14 @@ mvn clean package
 ```
 
 After a successful build, the jar of 'rapids-4-spark-tools_2.12-*-SNAPSHOT.jar' will be in 'target/' directory.  
-This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.3.3.
+This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.5.0.
 
 For development purpose, you may need to run the tests against different spark versions.
 To run the tests against a specific Spark version, you can use the `-Dbuildver=XXX` command line option.  
-For instance to build Spark 3.4.1 you would use:
+For instance to build Spark 3.5.1 you would use:
 
 ```shell script
-mvn -Dbuildver=341 clean package
+mvn -Dbuildver=351 clean package
 ```
 
 Run `mvn help:all-profiles` to list supported Spark versions.

core/pom.xml

@@ -268,7 +268,6 @@
         <profile>
             <id>release333</id>
             <activation>
-                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>333</value>
@@ -296,6 +295,21 @@
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
+        <profile>
+            <id>release335</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>335</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>335</buildver>
+                <spark.version>${spark335.version}</spark.version>
+                <delta.core.version>${delta23x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
         <profile>
             <id>release340</id>
             <activation>
@@ -341,9 +355,25 @@
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
+        <profile>
+            <id>release343</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>343</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>343</buildver>
+                <spark.version>${spark343.version}</spark.version>
+                <delta.core.version>${delta24x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
         <profile>
             <id>release350</id>
             <activation>
+                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>350</value>
@@ -352,7 +382,7 @@
             <properties>
                 <buildver>350</buildver>
                 <spark.version>${spark350.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -367,7 +397,22 @@
             <properties>
                 <buildver>351</buildver>
                 <spark.version>${spark351.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release352</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>352</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>352</buildver>
+                <spark.version>${spark352.version}</spark.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -382,7 +427,7 @@
             <properties>
                 <buildver>400</buildver>
                 <spark.version>${spark400.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -402,12 +447,15 @@
         <spark331.version>3.3.1</spark331.version>
         <spark332.version>3.3.2</spark332.version>
         <spark333.version>3.3.3</spark333.version>
-        <spark334.version>3.3.4-SNAPSHOT</spark334.version>
+        <spark334.version>3.3.4</spark334.version>
+        <spark335.version>3.3.5-SNAPSHOT</spark335.version>
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
-        <spark342.version>3.4.2-SNAPSHOT</spark342.version>
-        <spark350.version>3.5.0-SNAPSHOT</spark350.version>
-        <spark351.version>3.5.1-SNAPSHOT</spark351.version>
+        <spark342.version>3.4.2</spark342.version>
+        <spark343.version>3.4.3-SNAPSHOT</spark343.version>
+        <spark350.version>3.5.0</spark350.version>
+        <spark351.version>3.5.1</spark351.version>
+        <spark352.version>3.5.2-SNAPSHOT</spark352.version>
         <spark400.version>4.0.0-SNAPSHOT</spark400.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
@@ -439,7 +487,10 @@
         <delta22x.version>2.2.0</delta22x.version>
         <delta23x.version>2.3.0</delta23x.version>
         <delta24x.version>2.4.0</delta24x.version>
-        <delta.core.version>${delta10x.version}</delta.core.version>
+        <!-- TODO: Fix delta version should be 3.1.0 but this requires scala-binary 2.13 which we
+                   don't have yet -->
+        <delta31x.version>2.4.0</delta31x.version>
+        <delta.core.version>${delta24x.version}</delta.core.version>
         <!-- environment properties -->
         <java.version>1.8</java.version>
         <platform-encoding>UTF-8</platform-encoding>

user_tools/src/spark_rapids_pytools/resources/databricks_aws-configs.json

@@ -2,6 +2,32 @@
   "dependencies": {
       "deployMode": {
         "LOCAL": [
+          {
+            "name": "Apache Spark",
+            "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+            "type": "archive",
+            "relativePath": "jars/*",
+            "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+            "size": 400395283
+          },
+          {
+            "name": "Hadoop AWS",
+            "uri": "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar",
+            "type": "jar",
+            "md5": "59907e790ce713441955015d79f670bc",
+            "sha1": "a65839fbf1869f81a1632e09f415e586922e4f80",
+            "size": 962685
+          },
+          {
+            "name": "AWS Java SDK Bundled",
+            "uri": "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar",
+            "type": "jar",
+            "md5": "8a22f2d30b7e8eee9ea44f04fb13b35a",
+            "sha1": "02deec3a0ad83d13d032b1812421b23d7a961eea",
+            "size": 280645251
+          }
+        ],
+        "SPARK333-LOCAL": [
           {
             "name": "Apache Spark",
             "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",

user_tools/src/spark_rapids_pytools/resources/databricks_azure-configs.json

@@ -2,6 +2,24 @@
   "dependencies": {
       "deployMode": {
         "LOCAL": [
+          {
+            "name": "Apache Spark",
+            "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+            "type": "archive",
+            "relativePath": "jars/*",
+            "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+            "size": 400395283
+          },
+          {
+            "name": "Hadoop Azure",
+            "uri": "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.6/hadoop-azure-3.3.6.jar",
+            "type": "jar",
+            "md5": "0fb8b8e565fd920fb809220cb2cc5ee7",
+            "sha1": "24425e7fad3a302715cefd570a0f4bdf3f50bc8e",
+            "size": 609646
+          }
+        ],
+        "SPARK333-LOCAL": [
           {
             "name": "Apache Spark",
             "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",

user_tools/src/spark_rapids_pytools/resources/dataproc-configs.json

@@ -2,6 +2,24 @@
   "dependencies": {
     "deployMode": {
       "LOCAL": [
+        {
+          "name": "Apache Spark",
+          "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+          "type": "archive",
+          "relativePath": "jars/*",
+          "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+          "size": 400395283
+        },
+        {
+          "name": "GCS Connector Hadoop3",
+          "uri": "https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/hadoop3-2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar",
+          "type": "jar",
+          "md5": "2ee6ad7215304cf5da8e731afb36ad72",
+          "sha1": "3bea6d5e62663a2a5c03d8ca44dff4921aeb3170",
+          "size": 39359477
+        }
+      ],
+      "SPARK333-LOCAL": [
         {
           "name": "Apache Spark",
           "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",

user_tools/src/spark_rapids_pytools/resources/dataproc_gke-configs.json

@@ -2,6 +2,24 @@
   "dependencies": {
     "deployMode": {
       "LOCAL": [
+        {
+          "name": "Apache Spark",
+          "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+          "type": "archive",
+          "relativePath": "jars/*",
+          "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+          "size": 400395283
+        },
+        {
+          "name": "GCS Connector Hadoop3",
+          "uri": "https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/hadoop3-2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar",
+          "type": "jar",
+          "md5": "2ee6ad7215304cf5da8e731afb36ad72",
+          "sha1": "3bea6d5e62663a2a5c03d8ca44dff4921aeb3170",
+          "size": 39359477
+        }
+      ],
+      "SPARK333-LOCAL": [
         {
           "name": "Apache Spark",
           "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",

user_tools/src/spark_rapids_pytools/resources/emr-configs.json

@@ -2,6 +2,32 @@
   "dependencies": {
     "deployMode": {
       "LOCAL": [
+        {
+          "name": "Apache Spark",
+          "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+          "type": "archive",
+          "relativePath": "jars/*",
+          "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+          "size": 400395283
+        },
+        {
+          "name": "Hadoop AWS",
+          "uri": "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar",
+          "type": "jar",
+          "md5": "59907e790ce713441955015d79f670bc",
+          "sha1": "a65839fbf1869f81a1632e09f415e586922e4f80",
+          "size": 962685
+        },
+        {
+          "name": "AWS Java SDK Bundled",
+          "uri": "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar",
+          "type": "jar",
+          "md5": "8a22f2d30b7e8eee9ea44f04fb13b35a",
+          "sha1": "02deec3a0ad83d13d032b1812421b23d7a961eea",
+          "size": 280645251
+        }
+      ],
+      "SPARK333-LOCAL": [
         {
           "name": "Apache Spark",
           "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",

user_tools/src/spark_rapids_pytools/resources/onprem-configs.json

@@ -2,6 +2,16 @@
   "dependencies": {
     "deployMode": {
       "LOCAL": [
+        {
+          "name": "Apache Spark",
+          "uri": "https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
+          "type": "archive",
+          "relativePath": "jars/*",
+          "sha512": "8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319",
+          "size": 400395283
+        }
+      ],
+      "SPARK333-LOCAL": [
         {
           "name": "Apache Spark",
           "uri": "https://archive.apache.org/dist/spark/spark-3.3.3/spark-3.3.3-bin-hadoop3.tgz",