From 8304ca6e384422f721ea0f62bbf824172ec774a5 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Thu, 15 Aug 2024 16:25:45 -0600 Subject: [PATCH 01/13] pre-commit: disable no-commit-to-branch --- .pre-commit-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 817cce1b..9137ec50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,9 @@ repos: - id: fix-byte-order-marker - id: mixed-line-ending args: [--fix, "lf"] - - id: no-commit-to-branch - args: [--branch, main, --branch, master, --branch, devel] + # TODO disable until we can skip on merge commits to main/master/devel + # - id: no-commit-to-branch + # args: [--branch, main, --branch, master, --branch, devel] - id: trailing-whitespace exclude: | (?x)( From 2fb69cd6c889a48fb567d4d3301888b82739b51d Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Fri, 9 Aug 2024 18:40:20 +0000 Subject: [PATCH 02/13] CI: add dependabot --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..23c4cb3b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 381533e67a405a084e20c801c35ab603758f46f1 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Fri, 16 Aug 2024 09:34:20 -0600 Subject: [PATCH 03/13] bamboo: exit identically on macOS and Linux for unavailable modules --- buildsys/bamboo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index e3632781..9358c2ea 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -873,7 +873,7 @@ darwinSetMPI() { else echo "ERROR: unable to locate /etc/profile.modules - cannot load modules" - exit + exit 1 fi echo "bamboo.sh: MacOS build." From aacd73aa79b83ee66173e7e6434a71e0fa2f239e Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Wed, 28 Aug 2024 12:21:48 -0600 Subject: [PATCH 04/13] bamboo: factor out module initialization for both Linux and macOS --- buildsys/bamboo.sh | 61 +++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 9358c2ea..3dc73420 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -697,19 +697,7 @@ getconfig() { fi } -#------------------------------------------------------------------------- -# Function: linuxSetMPI -# Description: -# Purpose: Performs selection and loading of Bost and MPI modules -# for Linux -# Input: -# $1 - Bamboo Project -# $2 - mpi request -# $3 compiler (optional) -# Output: -# Return value: -linuxSetMPI() { - +set_up_environment_modules() { # For some reason, .bashrc is not being run prior to # this script. Kludge initialization of modules. @@ -725,13 +713,29 @@ linuxSetMPI() { echo "bamboo.sh: loaded /etc/profile.d/modules" fi - echo "Testing modules utility via ModuleEx..." - echo "ModuleEx avail" - ModuleEx avail - if [ $? -ne 0 ] ; then - echo " ModuleEx Failed" - exit 1 - fi + echo "Testing modules utility via ModuleEx..." + echo "ModuleEx avail" + ModuleEx avail + if [ $? -ne 0 ] ; then + echo " ModuleEx Failed" + exit 1 + fi +} + +#------------------------------------------------------------------------- +# Function: linuxSetMPI +# Description: +# Purpose: Performs selection and loading of Bost and MPI modules +# for Linux +# Input: +# $1 - Bamboo Project +# $2 - mpi request +# $3 compiler (optional) +# Output: +# Return value: +linuxSetMPI() { + + set_up_environment_modules # build MPI selector if [[ "$2" =~ openmpi.* ]] @@ -853,7 +857,6 @@ darwinSetMPI() { macosVersion=`echo ${macosVersionFull} | awk -F. '{print $1 "." $2 }'` echo " ******************* macosVersion= $macosVersion " - # macports or hybrid clang/macports PATH="/opt/local/bin:/usr/local/bin:$PATH" export PATH @@ -862,19 +865,11 @@ darwinSetMPI() { export ACLOCAL_FLAGS="-I/opt/local/share/aclocal $ACLOCAL_FLAGS" echo $ACLOCAL_FLAGS - # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac) - if [ -f /etc/profile.modules ] - then - source /etc/profile.modules - echo "bamboo.sh: loaded /etc/profile.modules." - # put any module loads here - echo "bamboo.sh: Loading Modules for MacOSX" - ldModules_MacOS_Clang $compiler $2 # any Xcode + set_up_environment_modules - else - echo "ERROR: unable to locate /etc/profile.modules - cannot load modules" - exit 1 - fi + # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac) + echo "bamboo.sh: Loading Modules for MacOSX" + ldModules_MacOS_Clang $compiler $2 # any Xcode echo "bamboo.sh: MacOS build." echo "bamboo.sh: MPI = $2" From eaf7d269af51d611f16db91fe18a13efab77cd12 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Wed, 28 Aug 2024 12:27:10 -0600 Subject: [PATCH 05/13] bamboo: add lmod init locations --- buildsys/bamboo.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 3dc73420..09feba3c 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -701,17 +701,17 @@ set_up_environment_modules() { # For some reason, .bashrc is not being run prior to # this script. Kludge initialization of modules. - echo "Attempt to initialize the modules utility. Look for modules init file in 1 of 2 places" - - echo "Location 1: ls -l /etc/profile.modules" - echo "Location 2: ls -l /etc/profile.d/modules.sh" - if [ -r /etc/profile.modules ] ; then - source /etc/profile.modules - echo "bamboo.sh: loaded /etc/profile.modules" - elif [ -r /etc/profile.d/modules.sh ] ; then - source /etc/profile.d/modules.sh - echo "bamboo.sh: loaded /etc/profile.d/modules" - fi + echo "Attempt to initialize the modules utility" + + locations=(/etc/profile.modules /etc/profile.d/modules.sh /opt/homebrew/opt/lmod/init/profile /usr/share/lmod/lmod/init/bash) + + for location in "${locations[@]}"; do + if [ -r "${location}" ]; then + # shellcheck disable=SC1090 + source "${location}" + echo "bamboo.sh: loaded ${location}" + fi + done echo "Testing modules utility via ModuleEx..." echo "ModuleEx avail" From 1674c45f46a4b11a52a56bb1cc1b4fe8157bd0c1 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Fri, 16 Aug 2024 10:12:54 -0600 Subject: [PATCH 06/13] bamboo: fix lmod location on Ubuntu --- buildsys/bamboo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 09feba3c..4b62c197 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -703,7 +703,7 @@ set_up_environment_modules() { echo "Attempt to initialize the modules utility" - locations=(/etc/profile.modules /etc/profile.d/modules.sh /opt/homebrew/opt/lmod/init/profile /usr/share/lmod/lmod/init/bash) + locations=(/etc/profile.modules /etc/profile.d/modules.sh /opt/homebrew/opt/lmod/init/profile /usr/share/lmod/lmod/init/profile) for location in "${locations[@]}"; do if [ -r "${location}" ]; then From b2b4218226142818d7128718570abfae249eb0d9 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Wed, 28 Aug 2024 12:23:09 -0600 Subject: [PATCH 07/13] bamboo: tidying --- buildsys/bamboo.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 4b62c197..dceaa3a4 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -675,9 +675,6 @@ getconfig() { ;; *) - #----------------------------------------------------------------- - # Unrecognized Scenario, This is an error in the bamboo code - #----------------------------------------------------------------- echo ' ' ; echo "Unrecognized Scenario, This is an error in the bamboo code" echo " UNRECOGNIZED: ${1}" exit 1 @@ -863,7 +860,7 @@ darwinSetMPI() { # Point to aclocal per instructions from sourceforge on MacOSX installation export ACLOCAL_FLAGS="-I/opt/local/share/aclocal $ACLOCAL_FLAGS" - echo $ACLOCAL_FLAGS + echo "ACLOCAL_FLAGS=${ACLOCAL_FLAGS}" set_up_environment_modules From a4daaa74c710a3e2b1421a243c2df40fee7818a3 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Fri, 16 Aug 2024 14:04:51 -0600 Subject: [PATCH 08/13] bamboo: set defaults for number of threads and ranks --- buildsys/bamboo.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index dceaa3a4..0ec85068 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -1764,6 +1764,10 @@ export SST_BUILD_TYPE="" # - If false, use each given branch as-is. export SST_TEST_MERGE=${SST_TEST_MERGE:-true} +# Number of threads and (MPI) ranks to run tests with (not compile with) +export SST_MULTI_THREAD_COUNT=${SST_MULTI_THREAD_COUNT:-1} +export SST_MULTI_RANK_COUNT=${SST_MULTI_RANK_COUNT:-1} + cloneOtherRepos # Load test definitions From ea232bfe2809d266fad91a9c4013507bc39b3243 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Tue, 20 Aug 2024 08:15:57 -0600 Subject: [PATCH 09/13] bamboo: remove dead module load --- buildsys/bamboo.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 0ec85068..d038a578 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -755,10 +755,7 @@ linuxSetMPI() { elif [[ "$3" =~ intel.* ]] then ModuleEx load intel/${4} - if [[ "$3" == *intel-15* ]] ; then - ModuleEx load gcc/gcc-4.8.1 - fi - + echo "LOADED intel/${4} compiler" fi fi From 18ea0185fdca277e41c7625fa55b4508d3a025ce Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Wed, 28 Aug 2024 12:15:05 -0600 Subject: [PATCH 10/13] bamboo: create sstmainline_coreonly_config_no_mpi --- buildsys/bamboo.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index d038a578..5273cdba 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -445,6 +445,28 @@ getconfig() { externalelementConfigStr="${NOBUILD}" junoConfigStr="${NOBUILD}" ;; + + sstmainline_coreonly_config_no_mpi) + #----------------------------------------------------------------- + # sstmainline_coreonly_config + # This option used for configuring SST with supported stabledevel deps + #----------------------------------------------------------------- + if [[ ${MPIHOME:+isSet} == isSet ]] ; then + echo ' ' ; echo " Test is flawed! MPI module is loaded!" ; echo ' ' + exit 1 + fi + export | egrep SST_DEPS_ + coreMiscEnv="${cc_environment}" + elementsMiscEnv="${cc_environment}" + depsStr="-r none" # Dependencies only needed for elements + setConvenienceVars "$depsStr" + coreConfigStr="$corebaseoptions $coreMiscEnv --without-mpi" + elementsConfigStr="${NOBUILD}" + macroConfigStr="${NOBUILD}" + externalelementConfigStr="${NOBUILD}" + junoConfigStr="${NOBUILD}" + ;; + sstmainline_config_linux_with_cuda) #----------------------------------------------------------------- # sstmainline_config_linux_with_cuda @@ -1837,7 +1859,7 @@ else echo "bamboo.sh: KERNEL = $kernel" case ${build_type} in - sstmainline_config|sstmainline_coreonly_config|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist) + sstmainline_config|sstmainline_coreonly_config|sstmainline_coreonly_config_no_mpi|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist) # Save Parameters in case they are needed later SST_DIST_MPI=${mpi_type} _UNUSED="none" From 9a984c150e37934fe0d91e32acf1b7768cd6c296 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Tue, 20 Aug 2024 09:47:02 -0600 Subject: [PATCH 11/13] bamboo: fix disabling MPI --- buildsys/bamboo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index 5273cdba..ca780122 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -460,7 +460,7 @@ getconfig() { elementsMiscEnv="${cc_environment}" depsStr="-r none" # Dependencies only needed for elements setConvenienceVars "$depsStr" - coreConfigStr="$corebaseoptions $coreMiscEnv --without-mpi" + coreConfigStr="$corebaseoptions $coreMiscEnv --disable-mpi" elementsConfigStr="${NOBUILD}" macroConfigStr="${NOBUILD}" externalelementConfigStr="${NOBUILD}" From 14373ca8c6475ba40f563f22d4c0222a6e55ad62 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Tue, 27 Aug 2024 11:54:52 -0600 Subject: [PATCH 12/13] bamboo: directory SST_DEPS_SRC_PRISTINE must exist --- buildsys/deps/include/depsDefinitions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buildsys/deps/include/depsDefinitions.sh b/buildsys/deps/include/depsDefinitions.sh index 809b7e5d..34f3a924 100755 --- a/buildsys/deps/include/depsDefinitions.sh +++ b/buildsys/deps/include/depsDefinitions.sh @@ -64,6 +64,10 @@ export SST_DEPS_SRC=$SST_BASE/sstDeps/src # be tar.gz files, zipfiles, or other archive formats. This can be # thought of as the "pre-staging" area for SST dependencies. export SST_DEPS_SRC_PRISTINE=${SST_DEPS_SRC}/pristine +if [[ ! -d "${SST_DEPS_SRC_PRISTINE}" ]]; then + echo "SST_DEPS_SRC_PRISTINE doesn't exist: ${SST_DEPS_SRC_PRISTINE}" + exit 1 +fi # Location where pristine source files are unarchived and patched as a # precursor to build and installation. This can be thought of as the From 9fdaedd62d4a94263c0933425e44fdfb89823b38 Mon Sep 17 00:00:00 2001 From: Eric Berquist Date: Fri, 9 Aug 2024 18:37:52 +0000 Subject: [PATCH 13/13] CI: run bamboo --- .github/workflows/ci.yml | 31 +++++++++ .github/workflows/pre-commit.yml | 16 +++-- .github/workflows/test.yml | 111 +++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e9899c17 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +--- +name: ci + +# yamllint disable-line rule:truthy +on: + push: + pull_request: + +concurrency: + # yamllint disable-line rule:line-length + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + prechecks: + uses: ./.github/workflows/pre-commit.yml + all-prechecks: + needs: [prechecks] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" + tests: + needs: [prechecks] + uses: ./.github/workflows/test.yml + all: + needs: [tests] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c2a1ed45..f64a609e 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,16 +1,20 @@ --- -# In the absence of using the pre-commit.ci service... name: pre-commit # yamllint disable-line rule:truthy on: - pull_request: - push: + workflow_call: + workflow_dispatch: + +concurrency: + # yamllint disable-line rule:line-length + group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..a6055b1d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,111 @@ +--- +# yamllint disable rule:line-length +name: run bamboo + +# yamllint disable-line rule:truthy +on: + workflow_call: + workflow_dispatch: + +concurrency: + group: test-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + run-bamboo: + strategy: + fail-fast: false + matrix: + os: + - macos-14 + - ubuntu-24.04 + build-type: + - sstmainline_config + - sstmainline_coreonly_config + - sstmainline_config_no_mpi + - sstmainline_coreonly_config_no_mpi + - sstmainline_config_make_dist_test + - sstmainline_config_core_make_dist_test + exclude: + - os: macos-14 + build-type: sstmainline_config + - os: macos-14 + build-type: sstmainline_coreonly_config + - os: macos-14 + build-type: sstmainline_config_make_dist_test + - os: macos-14 + build-type: sstmainline_config_core_make_dist_test + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: install system-level dependencies (macOS) + run: | + brew install \ + autoconf \ + automake \ + coreutils \ + doxygen \ + libtool \ + lmod \ + ncurses \ + pygments + python -m pip install blessings + # echo "source /opt/homebrew/opt/lmod/init/profile" >> ~/.bashrc + if: matrix.os == 'macos-14' + - name: install system-level dependencies (Ubuntu) + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -y update + sudo apt-get -y install \ + autoconf \ + automake \ + doxygen \ + libopenmpi-dev \ + libtool-bin \ + lmod \ + python3-blessings \ + python3-pygments + # echo "source /usr/share/lmod/lmod/init/bash" >> ~/.bashrc + if: matrix.os == 'ubuntu-24.04' + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.os }}-${{ matrix.build-type }} + verbose: 2 + - name: set env and prepare dir structure + run: | + # Keep this common in order to increase ccache effectiveness + # sst_deps_user_dir="${GITHUB_WORKSPACE}/../${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}_${GITHUB_RUN_ATTEMPT}" + sst_deps_user_dir="${GITHUB_WORKSPACE}/../sst-deps-user-dir" + mkdir -p "${sst_deps_user_dir}" + SST_DEPS_USER_DIR="$(realpath "${sst_deps_user_dir}")" + echo "SST_DEPS_USER_DIR=${SST_DEPS_USER_DIR}" >> "${GITHUB_ENV}" + - name: fetch elements dependencies + run: | + pristine="${SST_DEPS_USER_DIR}"/sstDeps/src/pristine + mkdir -p "${pristine}" + cd "${pristine}" + wget https://github.com/umd-memsys/DRAMSim2/archive/refs/tags/v2.2.2.tar.gz + - name: run bamboo + run: | + # needed for ncurses part of interactive sst-info + export TERM=dumb + MAKEFLAGS="-j$(nproc)" + export MAKEFLAGS + if [[ "$(uname)" == "Darwin" ]]; then + export PMIX_MCA_gds=hash + fi + export SST_DEPS_USER_MODE=1 + mkdir -p "${SST_DEPS_USER_DIR}/devel/trunk" + mv "${GITHUB_WORKSPACE}" "${SST_DEPS_USER_DIR}/devel/sqe" + cd "${SST_DEPS_USER_DIR}/devel/trunk" + ../sqe/buildsys/bamboo.sh ${{ matrix.build-type }} none none none none none + ccache -s -vv + - uses: actions/upload-artifact@v4 + with: + name: test-output-${{ matrix.os }}-${{ matrix.build-type }} + path: sst_test_outputs + compression-level: 9 + if: '!cancelled()' +# yamllint enable rule:line-length