From 69c1eb841f3370b99084f0afc55311b5ba9f769f Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Mon, 11 Jul 2022 20:47:59 -0500 Subject: [PATCH 1/3] Combine ubuntu-focal-external.yml and ubuntu-focal.yml --- .github/workflows/ubuntu-focal-external.yml | 295 -------------------- .github/workflows/ubuntu-focal.yml | 287 ++++++++++++++++++- README.md | 3 +- 3 files changed, 286 insertions(+), 299 deletions(-) delete mode 100644 .github/workflows/ubuntu-focal-external.yml diff --git a/.github/workflows/ubuntu-focal-external.yml b/.github/workflows/ubuntu-focal-external.yml deleted file mode 100644 index f2ef67706..000000000 --- a/.github/workflows/ubuntu-focal-external.yml +++ /dev/null @@ -1,295 +0,0 @@ -name: Ubuntu 20.04 (GCC, Python, ROCm) - -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main, develop ] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - OMNITRACE_VERBOSE: 1 - OMNITRACE_CI: ON - -jobs: - ubuntu-focal-external: - runs-on: ubuntu-20.04 - container: - image: jrmadsen/omnitrace-ci:ubuntu-20.04 - strategy: - matrix: - compiler: ['g++-7', 'g++-8'] - lto: ['OFF'] - strip: ['OFF'] - python: ['ON'] - build-type: ['Release'] - mpi-headers: ['OFF'] - static-libgcc: ['OFF'] - static-libstdcxx: ['OFF'] - include: - - compiler: 'g++-9' - lto: 'OFF' - strip: 'ON' - python: 'OFF' - build-type: 'Release' - mpi-headers: 'ON' - static-libgcc: 'ON' - static-libstdcxx: 'ON' - - compiler: 'g++-10' - lto: 'OFF' - strip: 'ON' - python: 'ON' - build-type: 'RelWithDebInfo' - mpi-headers: 'ON' - static-libgcc: 'ON' - static-libstdcxx: 'OFF' - - compiler: 'g++-11' - lto: 'ON' - strip: 'ON' - python: 'OFF' - build-type: 'Release' - mpi-headers: 'ON' - static-libgcc: 'ON' - static-libstdcxx: 'OFF' - - steps: - - uses: actions/checkout@v2 - - - name: Install Packages - timeout-minutes: 5 - run: - apt-get update && - apt-get install -y software-properties-common && - add-apt-repository -y ppa:ubuntu-toolchain-r/test && - apt-get update && - apt-get upgrade -y && - apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev ${{ matrix.compiler }} && - python3 -m pip install --upgrade pip && - python3 -m pip install numpy && - python3 -m pip install perfetto && - python3 -m pip install 'cmake==3.16.3' && - for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done - - - name: Configure Env - run: - echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && - echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && - echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH && - echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - - - name: Configure CMake - timeout-minutes: 10 - run: - git config --global --add safe.directory ${PWD} && - cmake --version && - cmake -B build - -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} - -DCMAKE_INSTALL_PREFIX=/opt/omnitrace - -DOMNITRACE_BUILD_TESTING=ON - -DOMNITRACE_USE_MPI=OFF - -DOMNITRACE_USE_HIP=OFF - -DOMNITRACE_USE_OMPT=OFF - -DOMNITRACE_USE_PAPI=OFF - -DOMNITRACE_USE_PYTHON=${{ matrix.python }} - -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }} - -DOMNITRACE_STRIP_LIBRARIES=${{ matrix.strip }} - -DOMNITRACE_BUILD_LTO=${{ matrix.lto }} - -DOMNITRACE_BUILD_STATIC_LIBGCC=${{ matrix.static-libgcc }} - -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${{ matrix.static-libstdcxx }} - -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs - -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9" - - - name: Build - timeout-minutes: 60 - run: - cmake --build build --target all --parallel 2 -- VERBOSE=1 - - - name: Install - timeout-minutes: 5 - run: - cmake --build build --target install --parallel 2 - - - name: Test - timeout-minutes: 45 - run: - cd build && - ldd ./omnitrace && - ./omnitrace --help && - ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log && - ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure - - - name: Test Install - timeout-minutes: 10 - run: | - set -v - export OMNITRACE_DEBUG=ON - which omnitrace-avail - ldd $(which omnitrace-avail) - omnitrace-avail --help - omnitrace-avail -a - which omnitrace-critical-trace - ldd $(which omnitrace-critical-trace) - which omnitrace - ldd $(which omnitrace) - omnitrace --help - omnitrace -e -v 1 -o ls.inst --simulate -- ls - for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -o ls.inst -- ls - ./ls.inst - omnitrace -e -v 1 --simulate -- ls - for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -- ls - - - name: Test User API - timeout-minutes: 10 - run: | - set -v - ./scripts/test-find-package.sh --install-dir /opt/omnitrace - - - name: CTest Artifacts - if: success() || failure() - uses: actions/upload-artifact@v2 - with: - name: ctest-${{ github.job }}-log - path: | - build/*.log - - - name: Data Artifacts - if: success() || failure() - uses: actions/upload-artifact@v2 - with: - name: data-${{ github.job }}-files - path: | - build/omnitrace-tests-output/**/*.txt - build/omnitrace-tests-output/**/*-instr*.json - - ubuntu-focal-external-rocm: - runs-on: ubuntu-20.04 - container: - image: jrmadsen/omnitrace-ci:ubuntu-20.04 - strategy: - matrix: - compiler: ['g++'] - rocm_version: ['4.3', '4.5', 'debian'] - - env: - BUILD_TYPE: MinSizeRel - OMNITRACE_OUTPUT_PATH: omnitrace-tests-output - OMNITRACE_OUTPUT_PREFIX: "%argt%/" - - steps: - - uses: actions/checkout@v2 - - - name: Install Packages - timeout-minutes: 5 - run: - apt-get update && - apt-get install -y software-properties-common wget gnupg2 && - wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && - echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list && - apt-get update && - apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev libopenmpi-dev curl && - python3 -m pip install --upgrade pip && - python3 -m pip install 'cmake==3.16.3' && - for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done - - - name: Configure Env - run: - echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && - echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && - echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV && - echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH && - echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - - - name: Configure CMake - timeout-minutes: 10 - run: - git config --global --add safe.directory ${PWD} && - cmake --version && - cmake -B build - -DCMAKE_C_COMPILER=${CC} - -DCMAKE_CXX_COMPILER=${CXX} - -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} - -DCMAKE_INSTALL_PREFIX=/opt/omnitrace - -DOMNITRACE_BUILD_TESTING=OFF - -DOMNITRACE_BUILD_DEVELOPER=ON - -DOMNITRACE_BUILD_EXTRA_OPTIMIZATIONS=OFF - -DOMNITRACE_BUILD_LTO=OFF - -DOMNITRACE_USE_MPI=OFF - -DOMNITRACE_USE_MPI_HEADERS=ON - -DOMNITRACE_USE_HIP=ON - -DOMNITRACE_MAX_THREADS=64 - -DOMNITRACE_USE_SANITIZER=OFF - -DOMNITRACE_USE_PAPI=OFF - -DOMNITRACE_INSTALL_PERFETTO_TOOLS=ON - - - name: Build - timeout-minutes: 45 - run: - cmake --build build --target all --parallel 2 -- VERBOSE=1 - - - name: Install - run: - cmake --build build --target install --parallel 2 - - - name: Test - timeout-minutes: 45 - run: - cd build && - ldd ./omnitrace && - ./omnitrace --help && - ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log && - ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure - - - name: Test Install - timeout-minutes: 10 - run: | - set -v - traced --background - export OMNITRACE_DEBUG=ON - export OMNITRACE_PERFETTO_BACKEND=system - which omnitrace-avail - ldd $(which omnitrace-avail) - omnitrace-avail --help - omnitrace-avail -a - which omnitrace-critical-trace - ldd $(which omnitrace-critical-trace) - which omnitrace - ldd $(which omnitrace) - omnitrace --help - omnitrace -e -v 1 -o ls.inst --simulate -- ls - omnitrace -e -v 1 --simulate -- ls - omnitrace -e -v 1 -o ls.inst -- ls - perfetto --out ls-perfetto-trace.proto --txt -c omnitrace.cfg - ./ls.inst - omnitrace -e -v 1 -- ls - du -m ls-perfetto-trace.proto - - - name: Test User API - timeout-minutes: 10 - run: | - set -v - ./scripts/test-find-package.sh --install-dir /opt/omnitrace - - - name: CTest Artifacts - if: success() || failure() - uses: actions/upload-artifact@v2 - with: - name: ctest-${{ github.job }}-log - path: | - build/*.log - - - name: Data Artifacts - if: success() || failure() - uses: actions/upload-artifact@v2 - with: - name: data-${{ github.job }}-files - path: | - omnitrace-tests-output/**/*.txt - build/omnitrace-tests-output/**/*.txt - build/omnitrace-tests-output/**/*-instr*.json diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index e238b2a07..c1500380e 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -1,4 +1,4 @@ -name: Ubuntu 20.04 (GCC, MPICH, OpenMPI) +name: Ubuntu 20.04 (GCC, Python, ROCm, MPICH, OpenMPI) on: push: @@ -11,11 +11,289 @@ concurrency: cancel-in-progress: true env: - ELFUTILS_DOWNLOAD_VERSION: 0.186 OMNITRACE_VERBOSE: 1 OMNITRACE_CI: ON jobs: + ubuntu-focal-external: + runs-on: ubuntu-20.04 + container: + image: jrmadsen/omnitrace-ci:ubuntu-20.04 + strategy: + matrix: + compiler: ['g++-7', 'g++-8'] + lto: ['OFF'] + strip: ['OFF'] + python: ['ON'] + build-type: ['Release'] + mpi-headers: ['OFF'] + static-libgcc: ['OFF'] + static-libstdcxx: ['OFF'] + include: + - compiler: 'g++-9' + lto: 'OFF' + strip: 'ON' + python: 'OFF' + build-type: 'Release' + mpi-headers: 'ON' + static-libgcc: 'ON' + static-libstdcxx: 'ON' + - compiler: 'g++-10' + lto: 'OFF' + strip: 'ON' + python: 'ON' + build-type: 'RelWithDebInfo' + mpi-headers: 'ON' + static-libgcc: 'ON' + static-libstdcxx: 'OFF' + - compiler: 'g++-11' + lto: 'ON' + strip: 'ON' + python: 'OFF' + build-type: 'Release' + mpi-headers: 'ON' + static-libgcc: 'ON' + static-libstdcxx: 'OFF' + + steps: + - uses: actions/checkout@v2 + + - name: Install Packages + timeout-minutes: 5 + run: + apt-get update && + apt-get install -y software-properties-common && + add-apt-repository -y ppa:ubuntu-toolchain-r/test && + apt-get update && + apt-get upgrade -y && + apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev ${{ matrix.compiler }} && + python3 -m pip install --upgrade pip && + python3 -m pip install numpy && + python3 -m pip install perfetto && + python3 -m pip install 'cmake==3.16.3' && + for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done + + - name: Configure Env + run: + echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && + echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && + echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH && + echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Configure CMake + timeout-minutes: 10 + run: + git config --global --add safe.directory ${PWD} && + cmake --version && + cmake -B build + -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} + -DCMAKE_INSTALL_PREFIX=/opt/omnitrace + -DOMNITRACE_BUILD_TESTING=ON + -DOMNITRACE_USE_MPI=OFF + -DOMNITRACE_USE_HIP=OFF + -DOMNITRACE_USE_OMPT=OFF + -DOMNITRACE_USE_PAPI=OFF + -DOMNITRACE_USE_PYTHON=${{ matrix.python }} + -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }} + -DOMNITRACE_STRIP_LIBRARIES=${{ matrix.strip }} + -DOMNITRACE_BUILD_LTO=${{ matrix.lto }} + -DOMNITRACE_BUILD_STATIC_LIBGCC=${{ matrix.static-libgcc }} + -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${{ matrix.static-libstdcxx }} + -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs + -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9" + + - name: Build + timeout-minutes: 60 + run: + cmake --build build --target all --parallel 2 -- VERBOSE=1 + + - name: Install + timeout-minutes: 5 + run: + cmake --build build --target install --parallel 2 + + - name: Test + timeout-minutes: 45 + run: + cd build && + ldd ./omnitrace && + ./omnitrace --help && + ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log && + ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure + + - name: Test Install + timeout-minutes: 10 + run: | + set -v + export OMNITRACE_DEBUG=ON + which omnitrace-avail + ldd $(which omnitrace-avail) + omnitrace-avail --help + omnitrace-avail -a + which omnitrace-critical-trace + ldd $(which omnitrace-critical-trace) + which omnitrace + ldd $(which omnitrace) + omnitrace --help + omnitrace -e -v 1 -o ls.inst --simulate -- ls + for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done + omnitrace -e -v 1 -o ls.inst -- ls + ./ls.inst + omnitrace -e -v 1 --simulate -- ls + for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done + omnitrace -e -v 1 -- ls + + - name: Test User API + timeout-minutes: 10 + run: | + set -v + ./scripts/test-find-package.sh --install-dir /opt/omnitrace + + - name: CTest Artifacts + if: success() || failure() + uses: actions/upload-artifact@v2 + with: + name: ctest-${{ github.job }}-log + path: | + build/*.log + + - name: Data Artifacts + if: success() || failure() + uses: actions/upload-artifact@v2 + with: + name: data-${{ github.job }}-files + path: | + build/omnitrace-tests-output/**/*.txt + build/omnitrace-tests-output/**/*-instr*.json + + ubuntu-focal-external-rocm: + runs-on: ubuntu-20.04 + container: + image: jrmadsen/omnitrace-ci:ubuntu-20.04 + strategy: + matrix: + compiler: ['g++'] + rocm_version: ['4.3', '4.5', 'debian'] + + env: + BUILD_TYPE: MinSizeRel + OMNITRACE_OUTPUT_PATH: omnitrace-tests-output + OMNITRACE_OUTPUT_PREFIX: "%argt%/" + + steps: + - uses: actions/checkout@v2 + + - name: Install Packages + timeout-minutes: 5 + run: + apt-get update && + apt-get install -y software-properties-common wget gnupg2 && + wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && + echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list && + apt-get update && + apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev libopenmpi-dev curl && + python3 -m pip install --upgrade pip && + python3 -m pip install 'cmake==3.16.3' && + for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done + + - name: Configure Env + run: + echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && + echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && + echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV && + echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH && + echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Configure CMake + timeout-minutes: 10 + run: + git config --global --add safe.directory ${PWD} && + cmake --version && + cmake -B build + -DCMAKE_C_COMPILER=${CC} + -DCMAKE_CXX_COMPILER=${CXX} + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} + -DCMAKE_INSTALL_PREFIX=/opt/omnitrace + -DOMNITRACE_BUILD_TESTING=OFF + -DOMNITRACE_BUILD_DEVELOPER=ON + -DOMNITRACE_BUILD_EXTRA_OPTIMIZATIONS=OFF + -DOMNITRACE_BUILD_LTO=OFF + -DOMNITRACE_USE_MPI=OFF + -DOMNITRACE_USE_MPI_HEADERS=ON + -DOMNITRACE_USE_HIP=ON + -DOMNITRACE_MAX_THREADS=64 + -DOMNITRACE_USE_SANITIZER=OFF + -DOMNITRACE_USE_PAPI=OFF + -DOMNITRACE_INSTALL_PERFETTO_TOOLS=ON + + - name: Build + timeout-minutes: 45 + run: + cmake --build build --target all --parallel 2 -- VERBOSE=1 + + - name: Install + run: + cmake --build build --target install --parallel 2 + + - name: Test + timeout-minutes: 45 + run: + cd build && + ldd ./omnitrace && + ./omnitrace --help && + ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log && + ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure + + - name: Test Install + timeout-minutes: 10 + run: | + set -v + traced --background + export OMNITRACE_DEBUG=ON + export OMNITRACE_PERFETTO_BACKEND=system + which omnitrace-avail + ldd $(which omnitrace-avail) + omnitrace-avail --help + omnitrace-avail -a + which omnitrace-critical-trace + ldd $(which omnitrace-critical-trace) + which omnitrace + ldd $(which omnitrace) + omnitrace --help + omnitrace -e -v 1 -o ls.inst --simulate -- ls + omnitrace -e -v 1 --simulate -- ls + omnitrace -e -v 1 -o ls.inst -- ls + perfetto --out ls-perfetto-trace.proto --txt -c omnitrace.cfg + ./ls.inst + omnitrace -e -v 1 -- ls + du -m ls-perfetto-trace.proto + + - name: Test User API + timeout-minutes: 10 + run: | + set -v + ./scripts/test-find-package.sh --install-dir /opt/omnitrace + + - name: CTest Artifacts + if: success() || failure() + uses: actions/upload-artifact@v2 + with: + name: ctest-${{ github.job }}-log + path: | + build/*.log + + - name: Data Artifacts + if: success() || failure() + uses: actions/upload-artifact@v2 + with: + name: data-${{ github.job }}-files + path: | + omnitrace-tests-output/**/*.txt + build/omnitrace-tests-output/**/*.txt + build/omnitrace-tests-output/**/*-instr*.json + ubuntu-focal: runs-on: ubuntu-20.04 strategy: @@ -40,6 +318,11 @@ jobs: papi: 'OFF' deps: '' + env: + ELFUTILS_DOWNLOAD_VERSION: 0.186 + OMNITRACE_VERBOSE: 1 + OMNITRACE_CI: ON + steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 150f36bf1..44bfafda2 100755 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # Omnitrace: Application Profiling, Tracing, and Analysis [![Ubuntu 18.04 with GCC and MPICH](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-bionic.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-bionic.yml) -[![Ubuntu 20.04 with GCC and ROCm](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external.yml) -[![Ubuntu 20.04 with GCC and MPI](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml) +[![Ubuntu 20.04 with GCC, ROCm, and MPI](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml) [![OpenSUSE 15.x with GCC](https://github.com/AMDResearch/omnitrace/actions/workflows/opensuse.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/opensuse.yml) > ***[Omnitrace](https://github.com/AMDResearch/omnitrace) is an AMD open source research project and is not supported as part of the ROCm software stack.*** From 3f1aec16158cacb62cb3308b02ae72f199539020 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 12 Jul 2022 15:47:45 -0500 Subject: [PATCH 2/3] Fix cpack-opensuse.yml - Remove Build step for RPM was moving wrong folder --- .github/workflows/cpack-opensuse.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpack-opensuse.yml b/.github/workflows/cpack-opensuse.yml index f97f70127..398060b31 100644 --- a/.github/workflows/cpack-opensuse.yml +++ b/.github/workflows/cpack-opensuse.yml @@ -71,7 +71,7 @@ jobs: continue-on-error: true run: | mv build-release/stgz/*.sh ./ - mv build-release/deb/*.rpm ./ + mv build-release/rpm/*.rpm ./ sudo rm -rf build-release sudo rm -rf /opt/omnitrace From e9193fa58bc15454210a68d32ead75fb88ba07e6 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 12 Jul 2022 15:54:12 -0500 Subject: [PATCH 3/3] Update docker scripts --- docker/build-docker-ci.sh | 67 +++++++++++++++++++++++++++++---- docker/build-docker-release.sh | 68 ++++++++++++++++++++++++++++++--- docker/build-docker.sh | 69 +++++++++++++++++++++++++++------- docker/test-docker-release.sh | 61 +++++++++++++++++++++++++++--- 4 files changed, 234 insertions(+), 31 deletions(-) diff --git a/docker/build-docker-ci.sh b/docker/build-docker-ci.sh index 19b40e449..d56d1eef7 100755 --- a/docker/build-docker-ci.sh +++ b/docker/build-docker-ci.sh @@ -2,49 +2,102 @@ set -e +: ${USER:=$(whoami)} : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} : ${NJOBS=$(nproc)} : ${ELFUTILS_VERSION:=0.186} : ${PUSH:=0} +verbose-run() +{ + echo -e "\n### Executing \"${@}\"... ###\n" + eval $@ +} + +tolower() +{ + echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; +} + +toupper() +{ + echo "$@" | awk -F '\|~\|' '{print toupper($1)}'; +} + +usage() +{ + print_option() { printf " --%-20s %-24s %s\n" "${1}" "${2}" "${3}"; } + echo "Options:" + print_option "help -h" "" "This message" + print_option "push" "" "Push the container to DockerHub when completed" + + echo "" + print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } + print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option "jobs -j" "[N]" "parallel build jobs" "${NJOBS}" + print_default_option elfutils-version "[0.183..0.186]" "ElfUtils version" "${ELFUTILS_VERSION}" + print_default_option user "[USERNAME]" "DockerHub username" "${USER}" +} + send-error() { + usage echo -e "\nError: ${@}" exit 1 } -verbose-run() +reset-last() { - echo -e "\n\n### Executing \"${@}\"... ###\n" - eval $@ + last() { send-error "Unsupported argument :: ${1}"; } } +reset-last + n=0 while [[ $# -gt 0 ]] do case "${1}" in + -h|--help) + usage + exit 0 + ;; "--distro") shift DISTRO=${1} + last() { DISTRO="${DISTRO} ${1}"; } ;; "--versions") shift VERSIONS=${1} + last() { VERSIONS="${VERSIONS} ${1}"; } ;; - "-j") + --jobs|-j) shift NJOBS=${1} + reset-last ;; "--elfutils-version") shift ELFUTILS_VERSION=${1} + reset-last + ;; + --user|-u) + shift + USER=${1} + reset-last ;; "--push") PUSH=1 + reset-last + ;; + --*) + reset-last + last ${1} ;; *) - send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" + last ${1} ;; esac n=$((${n} + 1)) @@ -74,7 +127,7 @@ for VERSION in ${VERSIONS} do verbose-run docker build . \ -f ${DOCKER_FILE} \ - --tag jrmadsen/omnitrace-ci:${DISTRO}-${VERSION} \ + --tag ${USER}/omnitrace-ci:${DISTRO}-${VERSION} \ --build-arg DISTRO=${DISTRO_IMAGE} \ --build-arg VERSION=${VERSION} \ --build-arg NJOBS=${NJOBS} \ @@ -84,7 +137,7 @@ done if [ "${PUSH}" -gt 0 ]; then for VERSION in ${VERSIONS} do - verbose-run docker push jrmadsen/omnitrace-ci:${DISTRO}-${VERSION} + verbose-run docker push ${USER}/omnitrace-ci:${DISTRO}-${VERSION} done fi diff --git a/docker/build-docker-release.sh b/docker/build-docker-release.sh index 28d348252..46319370e 100755 --- a/docker/build-docker-release.sh +++ b/docker/build-docker-release.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash -e if [ ! -f CMakeLists.txt ]; then if [ ! -f ../CMakeLists.txt ]; then @@ -11,16 +11,49 @@ fi set -e +tolower() +{ + echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; +} + +toupper() +{ + echo "$@" | awk -F '\|~\|' '{print toupper($1)}'; +} + +usage() +{ + print_option() { printf " --%-20s %-24s %s\n" "${1}" "${2}" "${3}"; } + echo "Options:" + print_option "help -h" "" "This message" + + echo "" + print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } + print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}" + print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}" + print_default_option user "[USERNAME]" "DockerHub username" "${USER}" + #print_default_option lto "[on|off]" "Enable LTO" "${LTO}" + + echo "" + echo "Usage: ${BASH_SOURCE[0]} -- " + echo " e.g:" + echo " ${BASH_SOURCE[0]} --distro ubuntu --versions 20.04 --rocm-versions 5.0 5.1 -- --core +nopython --rocm-mpi +nopython" + echo " ${BASH_SOURCE[0]} --distro ubuntu --versions 20.04 --python-version 6 7 8 9 10 -- --rocm +python --rocm-mpi +nopython" +} + send-error() { + usage echo -e "\nError: ${@}" exit 1 } verbose-run() { - echo -e "\n\n### Executing \"${@}\"... ###\n" - eval $@ + echo -e "\n### Executing \"${@}\"... ###\n" + exec "${@}" } build-release() @@ -33,9 +66,19 @@ build-release() shift shift shift - verbose-run docker run --rm -v ${PWD}:/home/omnitrace --env DISTRO=${OS} --env ROCM_VERSION=${ROCM_VERSION} --env VERSION=${CODE_VERSION} --env PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" ${CONTAINER} /home/omnitrace/scripts/build-release.sh ${@} + local DOCKER_ARGS="" + tty -s && DOCKER_ARGS="-it" || DOCKER_ARGS="" + verbose-run docker run ${DOCKER_ARGS} --rm -v ${PWD}:/home/omnitrace --stop-signal "SIGINT" --env DISTRO=${OS} --env ROCM_VERSION=${ROCM_VERSION} --env VERSION=${CODE_VERSION} --env PYTHON_VERSIONS="${PYTHON_VERSIONS}" ${CONTAINER} /home/omnitrace/scripts/build-release.sh ${@} +} + +reset-last() +{ + last() { send-error "Unsupported argument :: ${1}"; } } +reset-last + +: ${USER:=$(whoami)} : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} : ${ROCM_VERSIONS:=5.0 4.5 4.3} @@ -46,21 +89,34 @@ n=0 while [[ $# -gt 0 ]] do case "${1}" in + -h|--help) + usage + exit 0 + ;; "--distro") shift DISTRO=${1} + last() { DISTRO="${DISTRO} ${1}"; } ;; "--versions") shift VERSIONS=${1} + last() { VERSIONS="${VERSIONS} ${1}"; } ;; "--rocm-versions") shift ROCM_VERSIONS=${1} + last() { ROCM_VERSIONS="${ROCM_VERSIONS} ${1}"; } ;; "--python-versions") shift PYTHON_VERSIONS=${1} + last() { PYTHON_VERSIONS="${PYTHON_VERSIONS} ${1}"; } + ;; + --user|-u) + shift + USER=${1} + reset-last ;; "--") shift @@ -68,7 +124,7 @@ do break ;; *) - send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" + last ${1} ;; esac n=$((${n} + 1)) @@ -82,6 +138,6 @@ do TAG=${DISTRO}-${VERSION} for ROCM_VERSION in ${ROCM_VERSIONS} do - build-release jrmadsen/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${DISTRO}-${VERSION} ${ROCM_VERSION} ${CODE_VERSION} ${SCRIPT_ARGS} + build-release ${USER}/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${DISTRO}-${VERSION} ${ROCM_VERSION} ${CODE_VERSION} ${SCRIPT_ARGS} done done diff --git a/docker/build-docker.sh b/docker/build-docker.sh index 532ea1fd0..e6946056b 100755 --- a/docker/build-docker.sh +++ b/docker/build-docker.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +: ${USER:=$(whoami)} : ${ROCM_VERSIONS:="5.0 4.5 4.3"} : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} @@ -8,48 +9,90 @@ set -e +tolower() +{ + echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; +} + +toupper() +{ + echo "$@" | awk -F '\|~\|' '{print toupper($1)}'; +} + +usage() +{ + print_option() { printf " --%-20s %-24s %s\n" "${1}" "${2}" "${3}"; } + echo "Options:" + print_option "help -h" "" "This message" + + echo "" + print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } + print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}" + print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}" + print_default_option user "[USERNAME]" "DockerHub username" "${USER}" + #print_default_option lto "[on|off]" "Enable LTO" "${LTO}" +} + send-error() { + usage echo -e "\nError: ${@}" exit 1 } verbose-run() { - echo -e "\n\n### Executing \"${@}\"... ###\n" + echo -e "\n### Executing \"${@}\"... ###\n" eval $@ } +reset-last() +{ + last() { send-error "Unsupported argument :: ${1}"; } +} + +reset-last + n=0 while [[ $# -gt 0 ]] do case "${1}" in + -h|--help) + usage + exit 0 + ;; "--distro") shift DISTRO=${1} + last() { DISTRO="${DISTRO} ${1}"; } ;; "--versions") shift VERSIONS=${1} + last() { VERSIONS="${VERSIONS} ${1}"; } ;; "--rocm-versions") shift ROCM_VERSIONS=${1} + last() { ROCM_VERSIONS="${ROCM_VERSIONS} ${1}"; } ;; "--python-versions") shift PYTHON_VERSIONS=${1} + last() { PYTHON_VERSIONS="${PYTHON_VERSIONS} ${1}"; } + ;; + --user|-u) + shift + USER=${1} + reset-last + ;; + "--*") + send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" ;; *) - if [ "${n}" -eq 0 ]; then - DISTRO=${1} - elif [ "${n}" -eq 1 ]; then - VERSIONS=${1} - elif [ "${n}" -eq 2 ]; then - ROCM_VERSIONS=${1} - else - send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" - fi + last ${1} ;; esac n=$((${n} + 1)) @@ -79,7 +122,7 @@ do *) ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" + verbose-run docker build . -f ${DOCKER_FILE} --tag ${USER}/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" elif [ "${DISTRO}" = "centos" ]; then case "${VERSION}" in 7) @@ -110,7 +153,7 @@ do send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}" ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" + verbose-run docker build . -f ${DOCKER_FILE} --tag ${USER}/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" elif [ "${DISTRO}" = "opensuse" ]; then case "${VERSION}" in 15.*) @@ -153,7 +196,7 @@ do send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}" ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" + verbose-run docker build . -f ${DOCKER_FILE} --tag ${USER}/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" fi done done diff --git a/docker/test-docker-release.sh b/docker/test-docker-release.sh index ae794fd09..a0ad8d10d 100755 --- a/docker/test-docker-release.sh +++ b/docker/test-docker-release.sh @@ -11,6 +11,35 @@ fi set -e +tolower() +{ + echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; +} + +toupper() +{ + echo "$@" | awk -F '\|~\|' '{print toupper($1)}'; +} + +usage() +{ + print_option() { printf " --%-20s %-24s %s\n" "${1}" "${2}" "${3}"; } + echo "Options:" + print_option "help -h" "" "This message" + + echo "" + print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } + print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}" + print_default_option user "[USERNAME]" "DockerHub username" "${USER}" + + echo "" + echo "Usage: ${BASH_SOURCE[0]} -- " + echo " e.g:" + echo " ${BASH_SOURCE[0]} --distro ubuntu --versions 20.04 --rocm-versions 5.0 5.1 -- --stgz /path/to/stgz/installer" +} + send-error() { echo -e "\nError: ${@}" @@ -19,17 +48,27 @@ send-error() verbose-run() { - echo -e "\n\n### Executing \"${@}\"... ###\n" - eval $@ + echo -e "\n### Executing \"${@}\"... ###\n" + exec "${@}" } test-release() { CONTAINER=${1} shift - verbose-run docker run --rm -v ${PWD}:/home/omnitrace ${CONTAINER} /home/omnitrace/scripts/test-release.sh ${@} + local DOCKER_ARGS="" + tty -s && DOCKER_ARGS="-it" || DOCKER_ARGS="" + verbose-run docker run ${DOCKER_ARGS} --rm -v ${PWD}:/home/omnitrace ${CONTAINER} /home/omnitrace/scripts/test-release.sh ${@} +} + +reset-last() +{ + last() { send-error "Unsupported argument :: ${1}"; } } +reset-last + +: ${USER:=$(whoami)} : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} : ${ROCM_VERSIONS:=5.0 4.5 4.3} @@ -38,17 +77,29 @@ n=0 while [[ $# -gt 0 ]] do case "${1}" in + -h|--help) + usage + exit 0 + ;; "--distro") shift DISTRO=${1} + last() { DISTRO="${DISTRO} ${1}"; } ;; "--versions") shift VERSIONS=${1} + last() { VERSIONS="${VERSIONS} ${1}"; } ;; "--rocm-versions") shift ROCM_VERSIONS=${1} + last() { ROCM_VERSIONS="${ROCM_VERSIONS} ${1}"; } + ;; + --user|-u) + shift + USER=${1} + reset-last ;; "--") shift @@ -56,7 +107,7 @@ do break ;; *) - send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" + last ${1} ;; esac n=$((${n} + 1)) @@ -70,6 +121,6 @@ do TAG=${DISTRO}-${VERSION} for ROCM_VERSION in ${ROCM_VERSIONS} do - test-release jrmadsen/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${SCRIPT_ARGS} + test-release ${USER}/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${SCRIPT_ARGS} done done