From 2e491a8996df99b0d973c8e186ef8ac16f9a8b09 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 6 Sep 2023 12:29:09 -0700 Subject: [PATCH] GitHub cache and ccache Use ccache in CIs. Set up GitHub Cache for storing the ccache data. --- .github/workflows/ci.yml | 15 +++ .github/workflows/cleanup-cache-postpr.yml | 40 ++++++++ .github/workflows/cleanup-cache.yml | 63 ++++++++++++ .github/workflows/codeql.yml | 28 +++++- .../dependencies/dependencies_ccache.sh | 13 +++ .../dependencies/dependencies_mac.sh | 1 + .github/workflows/hip.yml | 19 +++- .github/workflows/intel.yml | 49 ++++++++++ .github/workflows/macos.yml | 16 +++ .github/workflows/post-pr.yml | 20 ++++ .github/workflows/stubs.yml | 20 +++- .github/workflows/ubuntu.yml | 98 ++++++++++++++++++- 12 files changed, 374 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/cleanup-cache-postpr.yml create mode 100644 .github/workflows/cleanup-cache.yml create mode 100755 .github/workflows/dependencies/dependencies_ccache.sh create mode 100644 .github/workflows/post-pr.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ecbcdd6..57e091d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,3 +33,18 @@ jobs: name: 🔄 Update Stub Files needs: [ubuntu, intel, hip, macos, windows] uses: ./.github/workflows/stubs.yml + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml new file mode 100644 index 00000000..27eaa225 --- /dev/null +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -0,0 +1,40 @@ +name: CleanUpCachePostPR + +on: + workflow_run: + workflows: [PostPR] + types: + - completed + +jobs: + CleanUpCcacheCachePostPR: + name: Clean Up Ccahe Cache Post PR + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccahe + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for k in $keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 00000000..d1fd20ed --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,63 @@ +name: CleanUpCache + +on: + workflow_run: + workflows: [👑 CI] + types: + - completed + +jobs: + CleanUpCcacheCache: + name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccahe + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + # push or pull_request or schedule or ... + EVENT=${{ github.event.workflow_run.event }} + + # Triggering workflow run name (e.g., LinuxClang) + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + + if [[ $EVENT == "pull_request" ]]; then + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + else + BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} + fi + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + # In our cache keys, substring after `-git-` is git hash, substring + # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). + # The goal is to keep the last used key of each job and delete all others. + + # something like ccache-LinuxClang- + keyprefix="ccache-${WORKFLOW_NAME}-" + + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) + + # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for j in $cached_jobs + do + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) + for k in $old_keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + done + unset IFS diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f747a3fe..7d8f042f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,10 +44,23 @@ jobs: python -m pip install --upgrade mpi4py python -m pip install --upgrade pytest + .github/workflows/dependencies/dependencies_ccache.sh + sudo ln -s /usr/local/bin/ccache /usr/local/bin/g++ + + - name: Set Up Cache + if: ${{ matrix.language == 'cpp' }} + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Configure (C++) if: ${{ matrix.language == 'cpp' }} run: | - $CMAKE -S . -B build -DAMReX_SPACEDIM="1;2;3" + $CMAKE -S . -B build -DAMReX_SPACEDIM="1;2;3" \ + -DCMAKE_CXX_COMPILER="/usr/local/bin/g++" - name: Initialize CodeQL uses: github/codeql-action/init@v2 @@ -62,6 +75,19 @@ jobs: - name: Build (C++) if: ${{ matrix.language == 'cpp' }} run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=400M + ccache -z + + $CMAKE --build build -j 2 + + ccache -s + du -hs ~/.cache/ccache + + # Make sure CodeQL has something to do + touch build/_deps/fetchedamrex-src/Src/Base/AMReX.cpp + export CCACHE_DISABLE=1 $CMAKE --build build -j 2 - name: Perform CodeQL Analysis diff --git a/.github/workflows/dependencies/dependencies_ccache.sh b/.github/workflows/dependencies/dependencies_ccache.sh new file mode 100755 index 00000000..cbedaee6 --- /dev/null +++ b/.github/workflows/dependencies/dependencies_ccache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +if [[ $# -eq 2 ]]; then + CVER=$1 +else + CVER=4.8 +fi + +wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz +tar xvf ccache-${CVER}-linux-x86_64.tar.xz +sudo mv -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/ +sudo rm -rf ccache-${CVER}-linux-x86_64 +sudo rm -f ccache-${CVER}-linux-x86_64.tar.xz diff --git a/.github/workflows/dependencies/dependencies_mac.sh b/.github/workflows/dependencies/dependencies_mac.sh index d265391c..95f5786b 100755 --- a/.github/workflows/dependencies/dependencies_mac.sh +++ b/.github/workflows/dependencies/dependencies_mac.sh @@ -11,3 +11,4 @@ brew update brew install gfortran || true brew install libomp || true brew install open-mpi || true +brew install ccache || true diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 260a7691..98fde5cd 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -18,10 +18,24 @@ jobs: - uses: actions/checkout@v3 - name: install dependencies shell: bash - run: .github/workflows/dependencies/hip.sh + run: | + .github/workflows/dependencies/hip.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build ImpactX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=300M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -43,3 +57,6 @@ jobs: -DAMReX_AMD_ARCH=gfx900 \ -DAMReX_SPACEDIM="1;2;3" cmake --build build --target pip_install -j 2 + + ccache -s + du -hs ~/.cache/ccache diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index f86ef79f..28c854a7 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -15,6 +15,14 @@ jobs: - name: Dependencies run: | .github/workflows/dependencies/dpcpp.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization @@ -24,6 +32,12 @@ jobs: source /opt/intel/oneapi/setvars.sh set -e + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + export CCACHE_DEPEND=1 + ccache -z + export CC=$(which icx) export CXX=$(which icpx) python3 -m pip install -U pip setuptools wheel @@ -39,6 +53,9 @@ jobs: -DAMReX_SPACEDIM="3" cmake --build build --target pip_install -j 2 + ccache -s + du -hs ~/.cache/ccache + tests-icpx: name: ICPX runs-on: ubuntu-20.04 @@ -47,6 +64,14 @@ jobs: - name: Dependencies run: | .github/workflows/dependencies/dpcpp.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization @@ -56,6 +81,11 @@ jobs: source /opt/intel/oneapi/setvars.sh set -e + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + ccache -z + export CC=$(which icx) export CXX=$(which icpx) python3 -m pip install -U pip setuptools wheel @@ -69,6 +99,9 @@ jobs: -DAMReX_SPACEDIM="1;2;3" cmake --build build --target pip_install -j 2 + ccache -s + du -hs ~/.cache/ccache + - name: Run tests run: | set +e @@ -95,6 +128,14 @@ jobs: echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mpi-devel + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build env: {CXXFLAGS: "-Werror"} run: | @@ -102,6 +143,11 @@ jobs: source /opt/intel/oneapi/setvars.sh set -e + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CXX=$(which icpc) export CC=$(which icc) python3 -m pip install -U pip setuptools wheel @@ -116,6 +162,9 @@ jobs: -DAMReX_SPACEDIM="1;2;3" cmake --build build --target pip_install -j 2 + ccache -s + du -hs ~/.cache/ccache + - name: Run tests run: | set +e diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 9398f04b..d61b4f95 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -18,8 +18,20 @@ jobs: - uses: actions/checkout@v3 - name: Dependencies run: .github/workflows/dependencies/dependencies_mac.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: /Users/runner/Library/Caches/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CMAKE_BUILD_PARALLEL_LEVEL=3 python3 -m pip install -U pip setuptools wheel pytest @@ -28,6 +40,10 @@ jobs: python3 -c "import amrex.space1d as amr; print(amr.__version__)" python3 -c "import amrex.space2d as amr; print(amr.__version__)" python3 -c "import amrex.space3d as amr; print(amr.__version__)" + + ccache -s + du -hs /Users/runner/Library/Caches/ccache + - name: Unit tests run: | python3 -m pytest tests/ diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml new file mode 100644 index 00000000..f5b91403 --- /dev/null +++ b/.github/workflows/post-pr.yml @@ -0,0 +1,20 @@ +name: PostPR +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/stubs.yml b/.github/workflows/stubs.yml index 0b3e412a..63b3f4b0 100644 --- a/.github/workflows/stubs.yml +++ b/.github/workflows/stubs.yml @@ -40,15 +40,33 @@ jobs: python-version: '3.9' - name: Dependencies - run: .github/workflows/dependencies/dependencies_gcc10.sh + run: | + .github/workflows/dependencies/dependencies_gcc10.sh + .github/workflows/dependencies/dependencies_ccache.sh + + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + ccache -z + python3 -m pip install -U pip setuptools wheel python3 -m pip install -U pip mpi4py pytest pybind11-stubgen pre-commit cmake -S . -B build -DAMReX_SPACEDIM="1;2;3" -DpyAMReX_IPO=OFF cmake --build build -j 2 --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: Update Stubs run: | .github/update_stub.sh diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index f0304093..9b9b9145 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -23,14 +23,32 @@ jobs: with: python-version: '3.9' - name: Dependencies - run: .github/workflows/dependencies/gcc7.sh + run: | + .github/workflows/dependencies/gcc7.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=800M + ccache -z + python3 -m pip install -U pip setuptools wheel pytest AMREX_MPI=ON python3 -m pip install -v . python3 -c "import amrex.space1d as amr; print(amr.__version__)" python3 -c "import amrex.space2d as amr; print(amr.__version__)" python3 -c "import amrex.space3d as amr; print(amr.__version__)" + + ccache -s + du -hs ~/.cache/ccache + - name: Unit tests run: | mpiexec -np 1 python3 -m pytest tests/ @@ -43,9 +61,23 @@ jobs: steps: - uses: actions/checkout@v3 - name: Dependencies - run: .github/workflows/dependencies/dependencies_gcc10.sh + run: | + .github/workflows/dependencies/dependencies_gcc10.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CC=$(which gcc-10) export CXX=$(which g++-10) python3 -m pip install -U pip setuptools wheel @@ -60,6 +92,9 @@ jobs: -DAMReX_SPACEDIM="1;2;3" cmake --build build --target pip_install -j 2 + ccache -s + du -hs ~/.cache/ccache + - name: Unit tests run: | ctest --test-dir build --output-on-failure @@ -78,9 +113,23 @@ jobs: steps: - uses: actions/checkout@v3 - name: Dependencies - run: .github/workflows/dependencies/dependencies_clang6.sh + run: | + .github/workflows/dependencies/dependencies_clang6.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=500M + ccache -z + export CC=$(which clang-6.0) export CXX=$(which clang++-6.0) python3 -m pip install -U pip pytest @@ -88,6 +137,10 @@ jobs: python3 -c "import amrex.space1d as amr; print(amr.__version__)" python3 -c "import amrex.space2d as amr; print(amr.__version__)" python3 -c "import amrex.space3d as amr; print(amr.__version__)" + + ccache -s + du -hs ~/.cache/ccache + - name: Unit tests run: | python3 -m pytest tests/ @@ -104,14 +157,32 @@ jobs: steps: - uses: actions/checkout@v3 - name: Dependencies - run: .github/workflows/dependencies/dependencies_clang14_libcpp.sh + run: | + .github/workflows/dependencies/dependencies_clang14_libcpp.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=300M + ccache -z + python3 -m pip install -U pip pytest python3 -m pip install -v . python3 -c "import amrex.space1d as amr; print(amr.__version__)" python3 -c "import amrex.space2d as amr; print(amr.__version__)" python3 -c "import amrex.space3d as amr; print(amr.__version__)" + + ccache -s + du -hs ~/.cache/ccache + - name: Unit tests run: | python3 -m pytest tests/ @@ -124,9 +195,23 @@ jobs: steps: - uses: actions/checkout@v3 - name: Dependencies - run: .github/workflows/dependencies/dependencies_nvcc11.sh + run: | + .github/workflows/dependencies/dependencies_nvcc11.sh + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=800M + ccache -z + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" @@ -148,3 +233,6 @@ jobs: -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build --target pip_install -j 2 + + ccache -s + du -hs ~/.cache/ccache