Merge https://github.com/ECP-WarpX/WarpX into extend-magnetostatic-so…

…lver

Docs/source/usage/parameters.rst

@@ -2472,7 +2472,10 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
 BackTransformed Diagnostics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``<diag_name>.diag_type = BackTransformed``. Additional options for this diagnostic include:
+``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``<diag_name>.diag_type = BackTransformed``. We support the following list of options from `Full Diagnostics`_
+    ``<diag_name>.format``, ``<diag_name>.openpmd_backend``, ``<diag_name>.dump_rz_modes``, ``<diag_name>.file_prefix``, ``<diag_name>.diag_lo``, ``<diag_name>.diag_hi``, ``<diag_name>.write_species``, ``<diag_name>.species``.
+
+    Additional options for this diagnostic include:
 
 * ``<diag_name>.num_snapshots_lab`` (`integer`)
     Only used when ``<diag_name>.diag_type`` is ``BackTransformed``.

Python/pywarpx/picmi.py

@@ -1673,6 +1673,12 @@ class Simulation(picmistandard.PICMI_Simulation):
 
     warpx_checkpoint_signals: list of strings
         Signals on which to write out a checkpoint
+
+    warpx_numprocs: list of ints (1 in 1D, 2 in 2D, 3 in 3D)
+        Domain decomposition on the coarsest level.
+        The domain will be chopped into the exact number of pieces in each dimension as specified by this parameter.
+        https://warpx.readthedocs.io/en/latest/usage/parameters.html#distribution-across-mpi-ranks-and-parallelization
+        https://warpx.readthedocs.io/en/latest/usage/domain_decomposition.html#simple-method
     """
 
     # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to
@@ -1716,6 +1722,7 @@ def init(self, kw):
 
         self.break_signals = kw.pop('warpx_break_signals', None)
         self.checkpoint_signals = kw.pop('warpx_checkpoint_signals', None)
+        self.numprocs = kw.pop('warpx_numprocs', None)
 
         self.inputs_initialized = False
         self.warpx_initialized = False
@@ -1766,6 +1773,8 @@ def initialize_inputs(self):
         pywarpx.warpx.break_signals = self.break_signals
         pywarpx.warpx.checkpoint_signals = self.checkpoint_signals
 
+        pywarpx.warpx.numprocs = self.numprocs
+
         particle_shape = self.particle_shape
         for s in self.species:
             if s.particle_shape is not None:
@@ -2161,6 +2170,7 @@ def initialize_inputs(self):
             variables = list(variables)
             variables.sort()
 
+        # species list
         if np.iterable(self.species):
             species_list = self.species
         else:
@@ -2342,6 +2352,46 @@ def initialize_inputs(self):
 
         self.set_write_dir()
 
+        # --- Use a set to ensure that fields don't get repeated.
+        variables = set()
+
+        if self.data_list is not None:
+            for dataname in self.data_list:
+                if dataname == 'position':
+                    # --- The positions are alway written out anyway
+                    pass
+                elif dataname == 'momentum':
+                    variables.add('ux')
+                    variables.add('uy')
+                    variables.add('uz')
+                elif dataname == 'weighting':
+                    variables.add('w')
+                elif dataname == 'fields':
+                    variables.add('Ex')
+                    variables.add('Ey')
+                    variables.add('Ez')
+                    variables.add('Bx')
+                    variables.add('By')
+                    variables.add('Bz')
+                elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']:
+                    variables.add(dataname)
+
+            # --- Convert the set to a sorted list so that the order
+            # --- is the same on all processors.
+            variables = list(variables)
+            variables.sort()
+
+        # species list
+        if np.iterable(self.species):
+            species_list = self.species
+        else:
+            species_list = [self.species]
+
+        for specie in species_list:
+            diag = pywarpx.Bucket.Bucket(self.name + '.' + specie.name,
+                                         variables = variables)
+            self.diagnostic._species_dict[specie.name] = diag
+
 
 class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase):
     """

Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp

@@ -5,6 +5,7 @@
  * License: BSD-3-Clause-LBNL
  */
 #include "HardEdgedPlasmaLens.H"
+#include "Utils/Parser/ParserUtils.H"
 
 #include <AMReX_ParmParse.H>
 #include <AMReX_REAL.H>
@@ -25,8 +26,8 @@ HardEdgedPlasmaLens::AddElement (amrex::ParmParse & pp_element, amrex::ParticleR
 
     amrex::ParticleReal dEdx = 0._prt;
     amrex::ParticleReal dBdx = 0._prt;
-    pp_element.query("dEdx", dEdx);
-    pp_element.query("dBdx", dBdx);
+    utils::parser::queryWithParser(pp_element, "dEdx", dEdx);
+    utils::parser::queryWithParser(pp_element, "dBdx", dBdx);
 
     h_dEdx.push_back(dEdx);
     h_dBdx.push_back(dBdx);

Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp

@@ -5,6 +5,7 @@
  * License: BSD-3-Clause-LBNL
  */
 #include "HardEdgedQuadrupole.H"
+#include "Utils/Parser/ParserUtils.H"
 
 #include <AMReX_ParmParse.H>
 #include <AMReX_REAL.H>
@@ -25,8 +26,8 @@ HardEdgedQuadrupole::AddElement (amrex::ParmParse & pp_element, amrex::ParticleR
 
     amrex::ParticleReal dEdx = 0._prt;
     amrex::ParticleReal dBdx = 0._prt;
-    pp_element.query("dEdx", dEdx);
-    pp_element.query("dBdx", dBdx);
+    utils::parser::queryWithParser(pp_element, "dEdx", dEdx);
+    utils::parser::queryWithParser(pp_element, "dBdx", dBdx);
 
     h_dEdx.push_back(dEdx);
     h_dBdx.push_back(dBdx);

Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp

@@ -5,6 +5,7 @@
  * License: BSD-3-Clause-LBNL
  */
 #include "LatticeElementBase.H"
+#include "Utils/Parser/ParserUtils.H"
 
 #include <AMReX_ParmParse.H>
 #include <AMReX_REAL.H>
@@ -21,7 +22,7 @@ LatticeElementBase::AddElementBase (amrex::ParmParse & pp_element, amrex::Partic
 {
     // Read in the length of the element and save the start and end, and update z_location
     amrex::ParticleReal ds;
-    pp_element.get("ds", ds);
+    utils::parser::getWithParser(pp_element, "ds", ds);
 
     h_zs.push_back(z_location);
     z_location += ds;

Source/Evolve/WarpXEvolve.cpp

@@ -216,7 +216,7 @@ WarpX::Evolve (int numsteps)
         // Resample particles
         // +1 is necessary here because value of step seen by user (first step is 1) is different than
         // value of step in code (first step is 0)
-        mypc->doResampling(istep[0]+1);
+        mypc->doResampling(istep[0]+1, verbose);
 
         if (num_mirrors>0){
             applyMirrors(cur_time);

Source/Particles/MultiParticleContainer.H

@@ -179,7 +179,7 @@ public:
     *
     * @param[in] timestep the current timestep.
     */
-    void doResampling (int timestep);
+    void doResampling (int timestep, bool verbose);
 
 #ifdef WARPX_QED
     /** If Schwinger process is activated, this function is called at every

Source/Particles/MultiParticleContainer.cpp

@@ -942,14 +942,14 @@ MultiParticleContainer::doCollisions ( Real cur_time, amrex::Real dt )
     collisionhandler->doCollisions(cur_time, dt, this);
 }
 
-void MultiParticleContainer::doResampling (const int timestep)
+void MultiParticleContainer::doResampling (const int timestep, const bool verbose)
 {
     for (auto& pc : allcontainers)
     {
         // do_resampling can only be true for PhysicalParticleContainers
         if (!pc->do_resampling){ continue; }
 
-        pc->resample(timestep);
+        pc->resample(timestep, verbose);
     }
 }
 

Source/Particles/PhysicalParticleContainer.H

@@ -307,7 +307,7 @@ public:
     *
     * @param[in] timestep the current timestep.
     */
-    void resample (int timestep) override final;
+    void resample (int timestep, bool verbose=true) override final;
 
 #ifdef WARPX_QED
     //Functions decleared in WarpXParticleContainer.H

Source/Particles/PhysicalParticleContainer.cpp

@@ -2949,7 +2949,7 @@ PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector ()
     return plasma_injector.get();
 }
 
-void PhysicalParticleContainer::resample (const int timestep)
+void PhysicalParticleContainer::resample (const int timestep, const bool verbose)
 {
     // In heavily load imbalanced simulations, MPI processes with few particles will spend most of
     // the time at the MPI synchronization in TotalNumberOfParticles(). Having two profiler entries
@@ -2967,7 +2967,11 @@ void PhysicalParticleContainer::resample (const int timestep)
     WARPX_PROFILE_VAR_START(blp_resample_actual);
     if (m_resampler.triggered(timestep, global_numparts))
     {
-        amrex::Print() << Utils::TextMsg::Info("Resampling " + species_name);
+        if (verbose) {
+            amrex::Print() << Utils::TextMsg::Info(
+                "Resampling " + species_name + " at step " + std::to_string(timestep)
+            );
+        }
         for (int lev = 0; lev <= maxLevel(); lev++)
         {
             for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti)
@@ -2977,7 +2981,6 @@ void PhysicalParticleContainer::resample (const int timestep)
         }
     }
     WARPX_PROFILE_VAR_STOP(blp_resample_actual);
-
 }
 
 

Source/Particles/WarpXParticleContainer.H

@@ -381,7 +381,7 @@ public:
      * override the method for every derived class. Note that in practice this function is never
      * called because resample() is only called for PhysicalParticleContainers.
      */
-    virtual void resample (const int /*timestep*/) {}
+    virtual void resample (const int /*timestep*/, bool /*verbose*/) {}
 
     /**
      * When using runtime components, AMReX requires to touch all tiles