Crossbow #269472
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # NOTE: must set "Crossbow" as name to have the badge links working in the | |
| # github comment reports! | |
| name: Crossbow | |
| on: | |
| push: | |
| branches: | |
| - "*-github-*" | |
| env: | |
| ARCHERY_DEBUG: 1 | |
| jobs: | |
| build-cpp-ubuntu: | |
| name: Build C++ libraries Ubuntu ${{ matrix.platform.arch }} | |
| runs-on: ${{ matrix.platform.runs_on }} | |
| env: | |
| # architecture name used for archery build | |
| ARCH: ${{ matrix.platform.archery_arch }} | |
| ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }} | |
| ARCH_SHORT: ${{ matrix.platform.archery_arch_short }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - runs_on: ["ubuntu-latest"] | |
| arch: "x86_64" | |
| archery_arch: "amd64" | |
| archery_arch_alias: "x86_64" | |
| archery_arch_short: "amd64" | |
| - runs_on: ["self-hosted", "Linux", "arm64"] | |
| arch: "aarch_64" | |
| archery_arch: "arm64v8" | |
| archery_arch_alias: "aarch64" | |
| archery_arch_short: "arm64" | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| path: arrow | |
| repository: pitrou/arrow | |
| ref: 8a67335ec0db9e60b3697e81959dd0ed2190009b | |
| submodules: recursive | |
| - name: Free up disk space | |
| if: runner.os == 'Linux' && runner.arch == 'X64' | |
| shell: bash | |
| run: | | |
| arrow/ci/scripts/util_free_space.sh | |
| - name: Set up Python by actions/setup-python | |
| if: | | |
| !(runner.os == 'Linux' && runner.arch != 'X64') | |
| uses: actions/setup-python@v4 | |
| with: | |
| cache: 'pip' | |
| python-version: 3.12 | |
| - name: Set up Python by apt | |
| if: runner.os == 'Linux' && runner.arch != 'X64' | |
| run: | | |
| sudo apt update | |
| sudo apt-get install -y python3-pip | |
| pip install -U pip | |
| echo "$HOME/.local/bin" >>"$GITHUB_PATH" | |
| - name: Install Archery | |
| shell: bash | |
| run: pip install -e arrow/dev/archery[all] | |
| - name: Build C++ libraries | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" | |
| SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}" | |
| SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}" | |
| SCCACHE_S3_KEY_PREFIX: "sccache" | |
| run: | | |
| archery docker run \ | |
| -e ARROW_JAVA_BUILD=OFF \ | |
| -e ARROW_JAVA_TEST=OFF \ | |
| java-jni-manylinux-2014 | |
| - name: Compress into single artifact to keep directory structure | |
| run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/ | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ubuntu-shared-lib-${{ matrix.platform.arch }} | |
| path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz | |
| build-cpp-macos: | |
| name: Build C++ libraries macOS ${{ matrix.platform.arch }} | |
| runs-on: ${{ matrix.platform.runs_on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - { runs_on: ["macos-13"], arch: "x86_64"} | |
| - { runs_on: ["macos-14"], arch: "aarch_64" } | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: "14.0" | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| path: arrow | |
| repository: pitrou/arrow | |
| ref: 8a67335ec0db9e60b3697e81959dd0ed2190009b | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| cache: 'pip' | |
| python-version: 3.12 | |
| - name: Install Archery | |
| shell: bash | |
| run: pip install -e arrow/dev/archery[all] | |
| - name: Install dependencies | |
| run: | | |
| # We want to use llvm@14 to avoid shared z3 | |
| # dependency. llvm@14 doesn't depend on z3 and llvm depends | |
| # on z3. And Homebrew's z3 provides only shared library. It | |
| # doesn't provides static z3 because z3's CMake doesn't accept | |
| # building both shared and static libraries at once. | |
| # See also: Z3_BUILD_LIBZ3_SHARED in | |
| # https://github.com/Z3Prover/z3/blob/master/README-CMake.md | |
| # | |
| # If llvm is installed, Apache Arrow C++ uses llvm rather than | |
| # llvm@14 because llvm is newer than llvm@14. | |
| brew uninstall llvm || : | |
| # Ensure updating python@XXX with the "--overwrite" option. | |
| # If python@XXX is updated without "--overwrite", it causes | |
| # a conflict error. Because Python 3 installed not by | |
| # Homebrew exists in /usr/local on GitHub Actions. If | |
| # Homebrew's python@XXX is updated without "--overwrite", it | |
| # tries to replace /usr/local/bin/2to3 and so on and causes | |
| # a conflict error. | |
| brew update | |
| for python_package in $(brew list | grep python@); do | |
| brew install --overwrite ${python_package} | |
| done | |
| brew bundle --file=arrow/cpp/Brewfile | |
| # We want to link aws-sdk-cpp statically but Homebrew's | |
| # aws-sdk-cpp provides only shared library. If we have | |
| # Homebrew's aws-sdk-cpp, our build mix Homebrew's | |
| # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's | |
| # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. | |
| brew uninstall aws-sdk-cpp | |
| # We want to use bundled RE2 for static linking. If | |
| # Homebrew's RE2 is installed, its header file may be used. | |
| # We uninstall Homebrew's RE2 to ensure using bundled RE2. | |
| brew uninstall grpc || : # gRPC depends on RE2 | |
| brew uninstall [email protected] || : # gRPC 1.54 may be installed too | |
| brew uninstall re2 | |
| # We want to use bundled Protobuf for static linking. If | |
| # Homebrew's Protobuf is installed, its library file may be | |
| # used on test We uninstall Homebrew's Protobuf to ensure using | |
| # bundled Protobuf. | |
| brew uninstall protobuf | |
| brew bundle --file=arrow/java/Brewfile | |
| - name: Build C++ libraries | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" | |
| SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}" | |
| SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}" | |
| SCCACHE_S3_KEY_PREFIX: "sccache" | |
| run: | | |
| set -e | |
| # make brew Java available to CMake | |
| export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home | |
| arrow/ci/scripts/java_jni_macos_build.sh \ | |
| $GITHUB_WORKSPACE/arrow \ | |
| $GITHUB_WORKSPACE/arrow/cpp-build \ | |
| $GITHUB_WORKSPACE/arrow/java-dist | |
| - name: Compress into single artifact to keep directory structure | |
| run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/ | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: macos-shared-lib-${{ matrix.platform.arch }} | |
| path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz | |
| build-cpp-windows: | |
| name: Build C++ libraries Windows | |
| runs-on: windows-2019 | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| path: arrow | |
| repository: pitrou/arrow | |
| ref: 8a67335ec0db9e60b3697e81959dd0ed2190009b | |
| submodules: recursive | |
| - name: Set up Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| - name: Download Timezone Database | |
| shell: bash | |
| run: arrow/ci/scripts/download_tz_database.sh | |
| - name: Install sccache | |
| shell: bash | |
| run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache | |
| - name: Build C++ libraries | |
| shell: cmd | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" | |
| SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}" | |
| SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}" | |
| SCCACHE_S3_KEY_PREFIX: "sccache" | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 | |
| REM For ORC | |
| set TZDIR=/c/msys64/usr/share/zoneinfo | |
| bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist" | |
| - name: Compress into single artifact to keep directory structure | |
| shell: bash | |
| run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/ | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: windows-shared-lib | |
| path: arrow-shared-libs-windows.tar.gz | |
| package-jars: | |
| name: Build jar files | |
| runs-on: macos-latest | |
| needs: | |
| - build-cpp-ubuntu | |
| - build-cpp-macos | |
| - build-cpp-windows | |
| steps: | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| path: arrow | |
| repository: pitrou/arrow | |
| ref: 8a67335ec0db9e60b3697e81959dd0ed2190009b | |
| submodules: recursive | |
| - name: Download Libraries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Decompress artifacts | |
| run: | | |
| mv artifacts/*/*.tar.gz . | |
| tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz | |
| tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz | |
| tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz | |
| tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz | |
| tar -xvzf arrow-shared-libs-windows.tar.gz | |
| - name: Test that shared libraries exist | |
| run: | | |
| set -x | |
| test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so | |
| test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so | |
| test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so | |
| test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so | |
| test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so | |
| test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so | |
| test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so | |
| test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so | |
| test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib | |
| test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib | |
| test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib | |
| test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib | |
| test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib | |
| test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib | |
| test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib | |
| test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib | |
| test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll | |
| test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll | |
| test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll | |
| - name: Build bundled jar | |
| env: | |
| MAVEN_ARGS: >- | |
| --no-transfer-progress | |
| run: | | |
| set -e | |
| pushd arrow/java | |
| mvn versions:set -DnewVersion=18.0.0-SNAPSHOT | |
| mvn versions:set -DnewVersion=18.0.0-SNAPSHOT -f bom | |
| popd | |
| arrow/ci/scripts/java_full_build.sh \ | |
| $GITHUB_WORKSPACE/arrow \ | |
| $GITHUB_WORKSPACE/arrow/java-dist | |
| - name: Set up Python by actions/setup-python | |
| if: | | |
| !(runner.os == 'Linux' && runner.arch != 'X64') | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.12 | |
| - name: Set up Python by apt | |
| if: runner.os == 'Linux' && runner.arch != 'X64' | |
| run: | | |
| sudo apt update | |
| sudo apt install -y \ | |
| libgit2-dev \ | |
| libpython3-dev \ | |
| python3-pip | |
| sudo python3 -m pip install --upgrade pip | |
| - name: Checkout Crossbow | |
| uses: actions/checkout@v4 | |
| with: | |
| path: crossbow | |
| ref: actions-9df394f913 | |
| - name: Setup Crossbow | |
| shell: bash | |
| run: | | |
| python3 -m pip install -e arrow/dev/archery[crossbow] | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Upload artifacts | |
| shell: bash | |
| run: | | |
| archery crossbow \ | |
| --queue-path $(pwd)/crossbow \ | |
| --queue-remote https://github.com/ursacomputing/crossbow \ | |
| upload-artifacts \ | |
| --sha actions-9df394f913-github-java-jars \ | |
| --tag actions-9df394f913-github-java-jars \ | |
| "arrow/java-dist/*.jar" \ | |
| "arrow/java-dist/*.json" \ | |
| "arrow/java-dist/*.pom" \ | |
| "arrow/java-dist/*.xml" \ | |
| "arrow/java-dist/*.zip" | |
| env: | |
| CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} | |
| - name: Verify uploaded artifacts | |
| shell: bash | |
| run: | | |
| archery crossbow \ | |
| --queue-path $(pwd)/crossbow \ | |
| --queue-remote https://github.com/ursacomputing/crossbow \ | |
| status \ | |
| --task-filter 'java-jars' \ | |
| --no-fetch \ | |
| --validate \ | |
| actions-9df394f913 | |
| env: | |
| CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} |