Make Java 21 minimum version to build Arrow

Change miminum java build version to 21. This doesn't change the minimum version to use Arrow which is still Java 8 Update docker images for java jni/conda integration tests to use recent python image in order to install java 21
laurentgo · Jun 25, 2024 · e31e655 · e31e655
1 parent 8d650d8
commit e31e655
Show file tree

Hide file tree

Showing 18 changed files with 240 additions and 67 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,12 +32,13 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Debian 12 Complete Documentation
+    name: AMD64 Ubuntu 24.04 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
       JDK: 21
+      UBUNTU: 24.04
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -50,8 +51,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: debian-docs-${{ hashFiles('cpp/**') }}
-          restore-keys: debian-docs-
+          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: ubuntu-docs-
       - name: Setup Python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
@@ -63,7 +64,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
           JDK: 21
-        run: archery docker run debian-docs
+        run: archery docker run ubuntu-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -74,4 +75,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push debian-docs
+        run: archery docker push ubuntu-docs
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -62,6 +62,7 @@ jobs:
         maven: [3.9.6]
         image: [java]
     env:
+      JDK: 22
       TEST_JDK: ${{ matrix.jdk }}
       MAVEN: ${{ matrix.maven }}
     steps:

diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -51,7 +51,7 @@ env:
 
 jobs:
   docker:
-    name: AMD64 manylinux2014 Java JNI
+    name: AMD64 manylinux Java JNI
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 240
@@ -68,8 +68,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }}
-          restore-keys: java-jni-manylinux-2014-
+          key: java-jni-manylinux-2-28-${{ hashFiles('cpp/**', 'java/**') }}
+          restore-keys: java-jni-manylinux-2-28-
       - name: Setup Python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
@@ -80,7 +80,7 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run java-jni-manylinux-2014
+        run: archery docker run java-jni-manylinux-2-28
       - name: Docker Push
         if: >-
           success() &&
@@ -91,7 +91,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push java-jni-manylinux-2014
+        run: archery docker push java-jni-manylinux-2-28
 
   docker_integration_python:
     name: AMD64 Conda Java C Data Interface Integration

diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile
@@ -20,7 +20,7 @@ ARG arch=amd64
 ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
-ARG jdk=11
+ARG jdk=21
 ARG maven=3.8.7
 RUN mamba install -q -y \
         maven=${maven} \

diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile → ci/docker/java-jni-manylinux.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile → ci/docker/java-jni-manylinux.dockerfile
@@ -33,7 +33,7 @@ RUN vcpkg install \
         --x-feature=s3
 
 # Install Java
-ARG java=1.8.0
+ARG java=21
 ARG maven=3.9.3
 RUN yum install -y java-$java-openjdk-devel && \
       yum clean all && \

diff --git a/ci/docker/java.dockerfile b/ci/docker/java.dockerfile
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM maven:3.9.6-eclipse-temurin-22
+
+COPY --from=maven:3.9.6-eclipse-temurin-8 /opt/java/openjdk /opt/java/openjdk8
+COPY --from=maven:3.9.6-eclipse-temurin-11 /opt/java/openjdk /opt/java/openjdk11
+COPY --from=maven:3.9.6-eclipse-temurin-17 /opt/java/openjdk /opt/java/openjdk17
+COPY --from=maven:3.9.6-eclipse-temurin-21 /opt/java/openjdk /opt/java/openjdk21
+
+
+env JAVA8_HOME /opt/java/openjdk8
+env JAVA11_HOME /opt/java/openjdk11
+env JAVA17_HOME /opt/java/openjdk17
+env JAVA21_HOME /opt/java/openjdk21
+env JAVA22_HOME /opt/java/openjdk
+
+RUN find "$JAVA8_HOME/lib" "$JAVA11_HOME/lib" "$JAVA17_HOME/lib" "$JAVA21_HOME/lib" "$JAVA22_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \
+    ldconfig;
+
+COPY ci/maven-toolchains.xml /usr/share/maven/conf/toolchains.xml
+
+CMD ["mvn"]
diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
@@ -23,21 +23,18 @@ ARG jdk=8
 
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-# See R install instructions at https://cloud.r-project.org/bin/linux/
+# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu
 RUN apt-get update -y && \
     apt-get install -y \
         apt-transport-https \
+        software-properties-common \
         dirmngr \
         gpg \
         lsb-release && \
-    gpg --keyserver keyserver.ubuntu.com \
-        --recv-key 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 && \
-    gpg --export 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 | \
-        gpg --no-default-keyring \
-            --keyring /usr/share/keyrings/cran.gpg \
-            --import - && \
-    echo "deb [signed-by=/usr/share/keyrings/cran.gpg] https://cloud.r-project.org/bin/linux/$(lsb_release -is | tr 'A-Z' 'a-z') $(lsb_release -cs)-cran40/" | \
-        tee /etc/apt/sources.list.d/cran.list && \
+    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \
+        tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \
+    # NOTE: Only R >= 4.0 is available in this repo
+    add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \
     if [ -f /etc/apt/sources.list.d/debian.sources ]; then \
         sed -i \
             -e 's/main$/main contrib non-free non-free-firmware/g' \
@@ -47,8 +44,7 @@ RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
         autoconf-archive \
         automake \
-        chromium \
-        chromium-sandbox \
+        chromium-browser \
         curl \
         doxygen \
         gi-docgen \

diff --git a/ci/maven-toolchains.xml b/ci/maven-toolchains.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<toolchains xmlns="http://maven.apache.org/TOOLCHAINS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/TOOLCHAINS/1.1.0 http://maven.apache.org/xsd/toolchains-1.1.0.xsd">
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>1.8</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA8_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>11</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA11_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>17</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA17_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>21</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA21_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+  <toolchain>
+    <type>jdk</type>
+    <provides>
+      <version>22</version>
+      <vendor>openjdk</vendor>
+    </provides>
+    <configuration>
+      <jdkHome>${env.JAVA22_HOME}</jdkHome>
+    </configuration>
+  </toolchain>
+</toolchains>
diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh
@@ -35,9 +35,14 @@ echo "=== Clear output directories and leftovers ==="
 rm -rf ${build_dir}
 
 echo "=== Building Arrow C++ libraries ==="
-devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \
+devtoolset_version=$(rpm -qa "devtoolset-*-gcc" "gcc-toolset-*-gcc" --queryformat %{VERSION} | \
                        grep -o "^[0-9]*")
-devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+if [ -d "/opt/rh/devtoolset-${devtoolset_version}" ]; then
+  devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+else
+  devtoolset_include_cpp="/opt/rh/gcc-toolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
+fi
+
 : ${ARROW_ACERO:=ON}
 export ARROW_ACERO
 : ${ARROW_BUILD_TESTS:=ON}

diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -57,7 +57,7 @@ jobs:
           archery docker run \
             -e ARROW_JAVA_BUILD=OFF \
             -e ARROW_JAVA_TEST=OFF \
-            java-jni-manylinux-2014
+            java-jni-manylinux-2-28
       - name: Compress into single artifact to keep directory structure
         run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
@@ -69,7 +69,7 @@ jobs:
       {{ macros.github_login_dockerhub()|indent }}
       - name: Push Docker image
         shell: bash
-        run: archery docker push java-jni-manylinux-2014
+        run: archery docker push java-jni-manylinux-2-28
     {% endif %}
 
   build-cpp-macos:

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -65,7 +65,7 @@ groups:
     - r-binary-packages
     - ubuntu-*
     - wheel-*
-    - test-debian-*-docs
+    - test-ubuntu-*-docs
 
 {############################# Testing tasks #################################}
 
@@ -1446,15 +1446,16 @@ tasks:
   {% endfor %}
 
   # be sure to update binary-task.rb when upgrading ubuntu
-  test-debian-12-docs:
+  test-ubuntu-24.04-docs:
     ci: github
     template: docs/github.linux.yml
     params:
       env:
-        JDK: 17
+        JDK: 21
+        UBUNTU: 24.04
       pr_number: Unset
       flags: "-v $PWD/build/:/build/"
-      image: debian-docs
+      image: ubuntu-docs
       publish: false
     artifacts:
       - docs.tar.gz
@@ -1583,8 +1584,9 @@ tasks:
     template: docs/github.linux.yml
     params:
       env:
-        JDK: 17
+        JDK: 21
+        UBUNTU: 24.04
       pr_number: Unset
       flags: "-v $PWD/build/:/build/"
-      image: debian-docs
+      image: ubuntu-docs
       publish: true