Merge branch 'development' of https://github.com/ECP-WarpX/WarpX into…

… MultiFabSet

.clang-tidy

@@ -14,6 +14,9 @@ Checks: '-*,
         -cppcoreguidelines-narrowing-conversions,
         -cppcoreguidelines-no-malloc,
         -cppcoreguidelines-owning-memory,
+    google-build-explicit-make-pair,
+    google-build-namespaces,
+    google-global-names-in-headers,
     misc-const-correctness,
     misc-misleading-bidirectional,
     misc-misleading-identifier,

.github/workflows/cuda.yml

@@ -111,7 +111,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach c45770c9f9b2c5fa98c675a439c502e78912bf47 && cd -
+        cd ../amrex && git checkout --detach 23.09 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:

.github/workflows/macos.yml

@@ -13,6 +13,7 @@ jobs:
     if: github.event.pull_request.draft == false
     env:
       CXXFLAGS: "-Werror -Wno-error=pass-failed"
+      HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE
       # For macOS, Ninja is slower than the default:
       #CMAKE_GENERATOR: Ninja
       # setuptools/mp4py work-around, see

CMakeLists.txt

@@ -1,7 +1,7 @@
 # Preamble ####################################################################
 #
 cmake_minimum_required(VERSION 3.20.0)
-project(WarpX VERSION 23.08)
+project(WarpX VERSION 23.09)
 
 include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)
 

Docs/source/conf.py

@@ -79,9 +79,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'23.08'
+version = u'23.09'
 # The full version, including alpha/beta/rc tags.
-release = u'23.08'
+release = u'23.09'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

Docs/source/install/dependencies.rst

@@ -75,7 +75,7 @@ Conda (Linux/macOS/Windows)
 
       .. code-block:: bash
 
-         conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba ninja mpich pip virtualenv
+         conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv
          conda activate warpx-cpu-mpich-dev
 
          # compile WarpX with -DWarpX_MPI=ON
@@ -214,7 +214,7 @@ The `Advanced Package Tool (APT) <https://en.wikipedia.org/wiki/APT_(software)>`
       .. code-block:: bash
 
          sudo apt update
-         sudo apt install build-essential ccache cmake g++ git libfftw3-mpi-dev libfftw3-dev libhdf5-openmpi-dev libopenmpi-dev pkg-config python3 python3-matplotlib python3-numpy python3-pandas python3-pip python3-scipy python3-venv
+         sudo apt install build-essential ccache cmake g++ git libfftw3-mpi-dev libfftw3-dev libhdf5-openmpi-dev libopenmpi-dev pkg-config python3 python3-matplotlib python3-mpi4py python3-numpy python3-pandas python3-pip python3-scipy python3-venv
 
          # optional:
          # for CUDA, either install

Docs/source/install/hpc/lassen.rst

@@ -3,93 +3,178 @@
 Lassen (LLNL)
 =============
 
-The `Lassen V100 GPU cluster <https://hpc.llnl.gov/hardware/platforms/lassen>`_ is located at LLNL.
+The `Lassen V100 GPU cluster <https://hpc.llnl.gov/hardware/platforms/lassen>`__ is located at LLNL.
 
 
 Introduction
 ------------
 
 If you are new to this system, **please see the following resources**:
 
-* `LLNL user account <https://lc.llnl.gov/lorenz/mylc/mylc.cgi>`_
-* `Lassen user guide <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system>`_
-* Batch system: `LSF <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system#batch-system>`_
-* `Production directories <https://hpc.llnl.gov/hardware/file-systems>`_:
+* `LLNL user account <https://lc.llnl.gov/lorenz/mylc/mylc.cgi>`__ (login required)
+* `Lassen user guide <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system>`__
+* Batch system: `LSF <https://hpc.llnl.gov/training/tutorials/using-lcs-sierra-system#batch-system>`__
+* `Jupyter service <https://lc.llnl.gov/jupyter>`__ (`documentation <https://lc.llnl.gov/confluence/display/LC/JupyterHub+and+Jupyter+Notebook>`__, login required)
+* `Production directories <https://hpc.llnl.gov/hardware/file-systems>`__:
 
   * ``/p/gpfs1/$(whoami)``: personal directory on the parallel filesystem
   * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation.
 
 
-Installation
-------------
+Login
+-----
+
+.. note::
+
+   Lassen is currently transitioning to RHEL8.
+   During this transition, first SSH into lassen and then ``ssh eatoss4`` next to work with the updated RHEL8/TOSS4 nodes.
+
+   Approximately October 2023, the new software environment on these nodes will be the new default.
+
+
+.. _building-lassen-preparation:
+
+Preparation
+-----------
 
-Use the following commands to download the WarpX source code and switch to the correct branch:
+Use the following commands to download the WarpX source code:
 
 .. code-block:: bash
 
    git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
 
-We use the following modules and environments on the system (``$HOME/lassen_warpx.profile``).
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
+Create it now:
 
-.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_warpx.profile.example
-   :language: bash
-   :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_warpx.profile.example``.
+.. code-block:: bash
+
+   cp $HOME/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
+      :language: bash
+
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
+Enter the edit mode by typing ``i`` and edit line 2 to read:
+
+.. code-block:: bash
+
+   export proj="nsldt"
+
+Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+
+.. important::
+
+   Now, and as the first step on future logins to lassen, activate these environment settings:
+
+   .. code-block:: bash
+
+      source $HOME/lassen_v100_warpx.profile
+
+Finally, since lassen does not yet provide software modules for some of our dependencies, install them once:
+
+.. code-block:: bash
+
+   bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
+   source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
+      :language: bash
 
-We recommend to store the above lines in a file, such as ``$HOME/lassen_warpx.profile``, and load it into your shell after a login:
+.. dropdown:: AI/ML Dependencies (Optional)
+   :animate: fade-in-slide-down
+
+   If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
+   This will take a while and should be skipped if not needed.
+
+   .. code-block:: bash
+
+      runNode bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
+
+   .. dropdown:: Script Details
+      :color: light
+      :icon: info
+      :animate: fade-in-slide-down
+
+      .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
+         :language: bash
+
+   For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
+
+   .. code-block:: bash
+
+      python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt
+
+
+.. _building-lassen-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile the application executable:
 
 .. code-block:: bash
 
-   source $HOME/lassen_warpx.profile
+   cd $HOME/src/warpx
+   rm -rf build_lassen
 
-And since Lassen does not yet provide a module for them, install ADIOS2, BLAS++ and LAPACK++:
+   cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_lassen -j 8
+
+The WarpX application executables are now in ``$HOME/src/warpx/build_lassen/bin/``.
+Additionally, the following commands will install WarpX as a Python module:
 
 .. code-block:: bash
 
-   # c-blosc (I/O compression)
-   git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc
-   rm -rf src/c-blosc-lassen-build
-   cmake -S src/c-blosc -B src/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/c-blosc-1.21.1
-   cmake --build src/c-blosc-lassen-build --target install --parallel 16
-
-   # HDF5
-   git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5
-   rm -rf src/hdf5-lassen-build
-   cmake -S src/hdf5 -B src/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/hdf5-1.14.1.2
-   cmake --build src/hdf5-lassen-build --target install --parallel 16
-
-   # ADIOS2
-   git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2
-   rm -rf src/adios2-lassen-build
-   cmake -S src/adios2 -B src/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.8.3
-   cmake --build src/adios2-lassen-build --target install -j 16
-
-   # BLAS++ (for PSATD+RZ)
-   git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp
-   rm -rf src/blaspp-lassen-build
-   cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master
-   cmake --build src/blaspp-lassen-build --target install --parallel 16
-
-   # LAPACK++ (for PSATD+RZ)
-   git clone https://github.com/icl-utk-edu/lapackpp.git src/lapackpp
-   rm -rf src/lapackpp-lassen-build
-   CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so
-   cmake --build src/lapackpp-lassen-build --target install --parallel 16
-
-Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile:
+   rm -rf build_lassen_py
+
+   cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_lassen_py -j 8 --target pip_install
+
+Now, you can :ref:`submit lassen compute jobs <running-cpp-lassen>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit lassen jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-lassen>` or copy them to a location in ``$PROJWORK/$proj/``.
+
+
+.. _building-lassen-update:
+
+Update WarpX & Dependencies
+---------------------------
+
+If you already installed WarpX in the past and want to update it, start by getting the latest source code:
 
 .. code-block:: bash
 
    cd $HOME/src/warpx
 
-   rm -rf build_lassen
-   cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_DIMS="1;2;RZ;3" -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON
-   cmake --build build_lassen -j 10
+   # read the output of this command - does it look ok?
+   git status
+
+   # get the latest WarpX source code
+   git fetch
+   git pull
 
-The other :ref:`general compile-time options <building-cmake>` apply as usual.
+   # read the output of these commands - do they look ok?
+   git status
+   git log     # press q to exit
 
-**That's it!**
-WarpX executables for 1D, 2D, RZ and 3D are now in ``build_lassen/bin/`` and :ref:`can be run <running-cpp-lassen>` with the respective :ref:`example inputs files <usage-examples>`.
-Most people execute the binary directly or copy it out to a location in ``/p/gpfs1/$(whoami)``.
+And, if needed,
+
+- :ref:`update the lassen_v100_warpx.profile file <building-lassen-preparation>`,
+- log out and into the system, activate the now updated environment profile as usual,
+- :ref:`execute the dependency install scripts <building-lassen-preparation>`.
+
+As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_lassen`` and rebuild WarpX.
 
 
 .. _running-cpp-lassen:
@@ -146,3 +231,6 @@ Known System Issues
    .. code-block:: bash
 
       export OMPI_MCA_coll_ibm_skip_allgatherv=true
+
+As part of the same `CORAL acquisition program <https://doi.org/10.1109/SC.2018.00055>`__, Lassen is very similar to the design of Summit (OLCF).
+Thus, when encountering new issues it is worth checking also the :ref:`known Summit issues and work-arounds <building-summit-issues>`.