Lassen (LLNL): Sources & Builds Off-Home (ECP-WarpX#4311)

Moving the source directories and build instructions for WarpX itself from `$HOME` to the per-user NFS directory `/usr/workspace/${USER}/lassen/src`, which has a larger default quota.
eebasso · Sep 21, 2023 · ef3fa05 · ef3fa05
1 parent 40e9c17
commit ef3fa05
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 36 deletions.
diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -41,14 +41,14 @@ Use the following commands to download the WarpX source code:
 
 .. code-block:: bash
 
-   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+   git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx
 
 We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
 Create it now:
 
 .. code-block:: bash
 
-   cp $HOME/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
+   cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
 
 .. dropdown:: Script Details
    :color: light
@@ -80,7 +80,7 @@ Finally, since lassen does not yet provide software modules for some of our depe
 
 .. code-block:: bash
 
-   bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
+   bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
    source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
 
 .. dropdown:: Script Details
@@ -99,7 +99,7 @@ Finally, since lassen does not yet provide software modules for some of our depe
 
    .. code-block:: bash
 
-      runNode bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
+      runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
 
    .. dropdown:: Script Details
       :color: light
@@ -113,7 +113,7 @@ Finally, since lassen does not yet provide software modules for some of our depe
 
    .. code-block:: bash
 
-      python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt
+      python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
 
 
 .. _building-lassen-compilation:
@@ -125,13 +125,13 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd /usr/workspace/${USER}/lassen/src/warpx
    rm -rf build_lassen
 
    cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
    cmake --build build_lassen -j 8
 
-The WarpX application executables are now in ``$HOME/src/warpx/build_lassen/bin/``.
+The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``.
 Additionally, the following commands will install WarpX as a Python module:
 
 .. code-block:: bash
@@ -155,7 +155,7 @@ If you already installed WarpX in the past and want to update it, start by getti
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd /usr/workspace/${USER}/lassen/src/warpx
 
    # read the output of this command - does it look ok?
    git status
@@ -174,7 +174,7 @@ And, if needed,
 - log out and into the system, activate the now updated environment profile as usual,
 - :ref:`execute the dependency install scripts <building-lassen-preparation>`.
 
-As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_lassen`` and rebuild WarpX.
+As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/lassen/src/warpx/build_lassen`` and rebuild WarpX.
 
 
 .. _running-cpp-lassen:

diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh
@@ -20,9 +20,11 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo
 
 # Remove old dependencies #####################################################
 #
+SRC_DIR="/usr/workspace/${USER}/lassen/src"
 SW_DIR="/usr/workspace/${USER}/lassen/gpu"
 rm -rf ${SW_DIR}
 mkdir -p ${SW_DIR}
+mkdir -p ${SRC_DIR}
 
 # remove common user mistakes in python, located in .local instead of a venv
 python3 -m pip uninstall -qq -y pywarpx
@@ -37,70 +39,70 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true
 build_dir=$(mktemp -d)
 
 # c-blosc (I/O compression)
-if [ -d $HOME/src/c-blosc ]
+if [ -d ${SRC_DIR}/c-blosc ]
 then
-  cd $HOME/src/c-blosc
+  cd ${SRC_DIR}/c-blosc
   git fetch --prune
   git checkout v1.21.1
   cd -
 else
-  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc
+  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc
 fi
-cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
+cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
 cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10
 
 # HDF5
-if [ -d $HOME/src/hdf5 ]
+if [ -d ${SRC_DIR}/hdf5 ]
 then
-  cd $HOME/src/hdf5
+  cd ${SRC_DIR}/hdf5
   git fetch --prune
   git checkout hdf5-1_14_1-2
   cd -
 else
-  git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git $HOME/src/hdf5
+  git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5
 fi
-cmake -S $HOME/src/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2
+cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2
 cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10
 
 # ADIOS2
-if [ -d $HOME/src/adios2 ]
+if [ -d ${SRC_DIR}/adios2 ]
 then
-  cd $HOME/src/adios2
+  cd ${SRC_DIR}/adios2
   git fetch --prune
   git checkout v2.8.3
   cd -
 else
-  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2
+  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2
 fi
-cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
+cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
 cmake --build ${build_dir}/adios2-lassen-build --target install -j 10
 
 # BLAS++ (for PSATD+RZ)
-if [ -d $HOME/src/blaspp ]
+if [ -d ${SRC_DIR}/blaspp ]
 then
-  cd $HOME/src/blaspp
+  cd ${SRC_DIR}/blaspp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp
+  git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp
 fi
-cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
+cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10
 
 # LAPACK++ (for PSATD+RZ)
-if [ -d $HOME/src/lapackpp ]
+if [ -d ${SRC_DIR}/lapackpp ]
 then
-  cd $HOME/src/lapackpp
+  cd ${SRC_DIR}/lapackpp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp
+  git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp
 fi
-CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so
+CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so
 cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10
 
 
@@ -124,7 +126,7 @@ echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt
 python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt
 
 # install or update WarpX dependencies such as picmistandard
-python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt
+python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt
 
 # for ML dependencies, see install_v100_ml.sh
 

diff --git a/Tools/machines/lassen-llnl/install_v100_ml.sh b/Tools/machines/lassen-llnl/install_v100_ml.sh
@@ -20,6 +20,9 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo
 
 # Remove old dependencies #####################################################
 #
+SRC_DIR="/usr/workspace/${USER}/lassen/src"
+mkdir -p ${SRC_DIR}
+
 # remove common user mistakes in python, located in .local instead of a venv
 python3 -m pip uninstall -qqq -y torch 2>/dev/null || true
 
@@ -29,21 +32,21 @@ python3 -m pip uninstall -qqq -y torch 2>/dev/null || true
 # for basic python dependencies, see install_v100_dependencies.sh
 
 # optional: for libEnsemble - WIP: issues with nlopt
-# python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt
+# python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt
 
 # optional: for pytorch
-if [ -d ${HOME}/src/pytorch ]
+if [ -d ${SRC_DIR}/pytorch ]
 then
-  cd ${HOME}/src/pytorch
+  cd ${SRC_DIR}/pytorch
   git fetch
   git checkout .
   git checkout v2.0.1
   git submodule update --init --recursive
   cd -
 else
-  git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git ${HOME}/src/pytorch
+  git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git ${SRC_DIR}/pytorch
 fi
-cd ${HOME}/src/pytorch
+cd ${SRC_DIR}/pytorch
 rm -rf build
 
 # see https://github.com/pytorch/pytorch/issues/97497#issuecomment-1499069641
@@ -63,4 +66,4 @@ cd -
 # optional: optimas dependencies (based on libEnsemble & ax->botorch->gpytorch->pytorch)
 #   commented because scikit-learn et al. compile > 2 hrs
 #   please run manually on a login node if needed
-#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt
+#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt
diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
@@ -10,6 +10,7 @@ module load cuda/12.0.0
 module load boost/1.70.0
 
 # optional: for openPMD support
+SRC_DIR="/usr/workspace/${USER}/lassen/src"
 SW_DIR="/usr/workspace/${USER}/lassen/gpu"
 export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH