Upgrade to Spark 3.5.3 and rebuild images on new debian release

apache · Sep 24, 2024 · 2615f49 · 2615f49
1 parent 381e3a0
commit 2615f49
Show file tree

Hide file tree

Showing 11 changed files with 25 additions and 26 deletions.
diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark351_amd64.yml b/docker/compose/docker-compose_hadoop284_hive233_spark351_amd64.yml
@@ -21,7 +21,7 @@ services:
     hostname: namenode
     container_name: namenode
     environment:
-      - CLUSTER_NAME=hudi_hadoop284_hive232_spark351
+      - CLUSTER_NAME=hudi_hadoop284_hive232_spark353
     ports:
       - "50070:50070"
       - "8020:8020"
@@ -36,12 +36,12 @@ services:
       retries: 3
 
   datanode1:
-    image: apachehudi/hudi-hadoop_2.8.4-datanode:latest
+    image: apachehudi/hudi-hadoop_2.8.4-datanode:bullseye
     platform: linux/amd64
     container_name: datanode1
     hostname: datanode1
     environment:
-      - CLUSTER_NAME=hudi_hadoop284_hive232_spark351
+      - CLUSTER_NAME=hudi_hadoop284_hive232_spark353
     env_file:
       - ./hadoop.env
     ports:
@@ -61,12 +61,12 @@ services:
       - namenode
 
   historyserver:
-    image: apachehudi/hudi-hadoop_2.8.4-history:latest
+    image: apachehudi/hudi-hadoop_2.8.4-history:bullseye
     platform: linux/amd64
     hostname: historyserver
     container_name: historyserver
     environment:
-      - CLUSTER_NAME=hudi_hadoop284_hive232_spark351
+      - CLUSTER_NAME=hudi_hadoop284_hive232_spark353
     depends_on:
       - "namenode"
     links:
@@ -91,7 +91,7 @@ services:
     container_name: hive-metastore-postgresql
 
   hivemetastore:
-    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:bullseye
     platform: linux/amd64
     hostname: hivemetastore
     container_name: hivemetastore
@@ -117,7 +117,7 @@ services:
       - "namenode"
 
   hiveserver:
-    image: apachehudi/hudi-hadoop_3.3.5-hive_2.3.3:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:bullseye
     platform: linux/amd64
     hostname: hiveserver
     container_name: hiveserver
@@ -139,7 +139,7 @@ services:
       - ${HUDI_WS}:/var/hoodie/ws
 
   sparkmaster:
-    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_3.5.1:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_3.5.3:bullseye
     platform: linux/amd64
     hostname: sparkmaster
     container_name: sparkmaster
@@ -159,7 +159,7 @@ services:
       - "namenode"
 
   spark-worker-1:
-    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_3.5.1:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_3.5.3:bullseye
     platform: linux/amd64
     hostname: spark-worker-1
     container_name: spark-worker-1
@@ -286,7 +286,7 @@ services:
       - 8126:8126
 
   adhoc-1:
-    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_3.5.1:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_3.5.3:bullseye
     platform: linux/amd64
     hostname: adhoc-1
     container_name: adhoc-1
@@ -311,7 +311,7 @@ services:
       - ${HUDI_WS}:/var/hoodie/ws
 
   adhoc-2:
-    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_3.5.1:latest
+    image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_3.5.3:bullseye
     platform: linux/amd64
     hostname: adhoc-2
     container_name: adhoc-2

diff --git a/docker/hoodie/hadoop/base/Dockerfile b/docker/hoodie/hadoop/base/Dockerfile
@@ -15,7 +15,7 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM openjdk:8u212-jdk-slim-stretch
+FROM openjdk:8u342-jdk-slim-bullseye
 MAINTAINER Hoodie
 USER root
 

diff --git a/docker/hoodie/hadoop/datanode/Dockerfile b/docker/hoodie/hadoop/datanode/Dockerfile
@@ -17,7 +17,7 @@
 
 ARG HADOOP_VERSION=2.8.4 
 ARG HADOOP_DN_PORT=50075
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:bullseye
 
 ENV HADOOP_DN_PORT ${HADOOP_DN_PORT}
 

diff --git a/docker/hoodie/hadoop/historyserver/Dockerfile b/docker/hoodie/hadoop/historyserver/Dockerfile
@@ -17,7 +17,7 @@
 
 ARG HADOOP_VERSION=2.8.4 
 ARG HADOOP_HISTORY_PORT=8188
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:bullseye
 
 ENV HADOOP_HISTORY_PORT ${HADOOP_HISTORY_PORT}
 

diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile
@@ -16,7 +16,7 @@
 # limitations under the License.
 
 ARG HADOOP_VERSION=2.8.4 
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:bullseye
 
 ENV HIVE_HOME /opt/hive
 ENV PATH $HIVE_HOME/bin:$PATH

diff --git a/docker/hoodie/hadoop/namenode/Dockerfile b/docker/hoodie/hadoop/namenode/Dockerfile
@@ -17,7 +17,7 @@
 
 ARG HADOOP_VERSION=2.8.4 
 ARG HADOOP_WEBHDFS_PORT=50070
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:bullseye
 
 ENV HADOOP_WEBHDFS_PORT ${HADOOP_WEBHDFS_PORT}
 

diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile
@@ -17,13 +17,13 @@
 
 ARG HADOOP_VERSION=2.8.4
 ARG HIVE_VERSION=2.3.3
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}:bullseye
 
 ENV ENABLE_INIT_DAEMON true
 ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
 ENV INIT_DAEMON_STEP spark_master_init
 
-ARG SPARK_VERSION=3.5.1
+ARG SPARK_VERSION=3.5.3
 ARG SPARK_HADOOP_VERSION=3
 
 ENV SPARK_VERSION ${SPARK_VERSION}
@@ -35,7 +35,7 @@ COPY finish-step.sh /
 
 # Need to do this all in one step because running separate commands doubles the image size
 RUN echo "Installing Spark-version (${SPARK_VERSION})" \
-      &&  wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
+      &&  wget https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
       && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
       && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} /opt/spark \
       && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \

diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile b/docker/hoodie/hadoop/sparkadhoc/Dockerfile
@@ -17,8 +17,8 @@
 
 ARG HADOOP_VERSION=2.8.4
 ARG HIVE_VERSION=2.3.3
-ARG SPARK_VERSION=3.5.1
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
+ARG SPARK_VERSION=3.5.3
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}:bullseye
 
 ARG PRESTO_VERSION=0.268
 ARG TRINO_VERSION=368

diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile b/docker/hoodie/hadoop/sparkmaster/Dockerfile
@@ -17,8 +17,8 @@
 
 ARG HADOOP_VERSION=2.8.4
 ARG HIVE_VERSION=2.3.3
-ARG SPARK_VERSION=3.5.1
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
+ARG SPARK_VERSION=3.5.3
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}:bullseye
 
 COPY master.sh /opt/spark
 

diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile b/docker/hoodie/hadoop/sparkworker/Dockerfile
@@ -17,8 +17,8 @@
 
 ARG HADOOP_VERSION=2.8.4
 ARG HIVE_VERSION=2.3.3
-ARG SPARK_VERSION=3.5.1
-FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
+ARG SPARK_VERSION=3.5.3
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}:bullseye
 
 COPY worker.sh /opt/spark
 

diff --git a/pom.xml b/pom.xml
@@ -445,7 +445,6 @@
         <version>${maven-docker-plugin.version}</version>
         <configuration>
           <skip>${skipDocker}</skip>
-          <platform>linux/arm64</platform>
         </configuration>
       </plugin>
       <plugin>