From 8e3d9400c4a1c8f5cf440bcc5c53e56c9de0eaef Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 31 Aug 2023 17:35:55 -0700 Subject: [PATCH 1/5] PICMI: LabFrameParticleDiagnostic w/ Species (#4254) Support the `species=...` argument in `PICMI_LabFrameParticleDiagnostic`. --- Python/pywarpx/picmi.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index d5d00fb3090..0417a260f6d 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2161,6 +2161,7 @@ def initialize_inputs(self): variables = list(variables) variables.sort() + # species list if np.iterable(self.species): species_list = self.species else: @@ -2342,6 +2343,46 @@ def initialize_inputs(self): self.set_write_dir() + # --- Use a set to ensure that fields don't get repeated. + variables = set() + + if self.data_list is not None: + for dataname in self.data_list: + if dataname == 'position': + # --- The positions are alway written out anyway + pass + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + variables.add(dataname) + + # --- Convert the set to a sorted list so that the order + # --- is the same on all processors. + variables = list(variables) + variables.sort() + + # species list + if np.iterable(self.species): + species_list = self.species + else: + species_list = [self.species] + + for specie in species_list: + diag = pywarpx.Bucket.Bucket(self.name + '.' + specie.name, + variables = variables) + self.diagnostic._species_dict[specie.name] = diag + class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ From 01f69c657c401b83fd795686a48a61e45b1a5b6e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 1 Sep 2023 12:05:54 -0700 Subject: [PATCH 2/5] PICMI: warpx.numprocs (#4255) Adding a parameter for `warpx.numprocs` in PICMI. https://warpx.readthedocs.io/en/latest/usage/parameters.html#distribution-across-mpi-ranks-and-parallelization https://warpx.readthedocs.io/en/latest/usage/domain_decomposition.html#simple-method --- Python/pywarpx/picmi.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 0417a260f6d..d1f5136bcd9 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1673,6 +1673,12 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_checkpoint_signals: list of strings Signals on which to write out a checkpoint + + warpx_numprocs: list of ints (1 in 1D, 2 in 2D, 3 in 3D) + Domain decomposition on the coarsest level. + The domain will be chopped into the exact number of pieces in each dimension as specified by this parameter. + https://warpx.readthedocs.io/en/latest/usage/parameters.html#distribution-across-mpi-ranks-and-parallelization + https://warpx.readthedocs.io/en/latest/usage/domain_decomposition.html#simple-method """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -1716,6 +1722,7 @@ def init(self, kw): self.break_signals = kw.pop('warpx_break_signals', None) self.checkpoint_signals = kw.pop('warpx_checkpoint_signals', None) + self.numprocs = kw.pop('warpx_numprocs', None) self.inputs_initialized = False self.warpx_initialized = False @@ -1766,6 +1773,8 @@ def initialize_inputs(self): pywarpx.warpx.break_signals = self.break_signals pywarpx.warpx.checkpoint_signals = self.checkpoint_signals + pywarpx.warpx.numprocs = self.numprocs + particle_shape = self.particle_shape for s in self.species: if s.particle_shape is not None: From 0272129f4fa88d083a65f1b302d029a1810d43cf Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 4 Sep 2023 05:29:03 -0700 Subject: [PATCH 3/5] Print resampling message only if `verbose` is on (#4264) * only print resampling message if `verbose` is on * clang-tidy fix Only use `const` qualification in function definitions, not in function declarations. --- Source/Evolve/WarpXEvolve.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 4 ++-- Source/Particles/PhysicalParticleContainer.H | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 9 ++++++--- Source/Particles/WarpXParticleContainer.H | 2 +- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 73ccc164676..067cd9a11b6 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -216,7 +216,7 @@ WarpX::Evolve (int numsteps) // Resample particles // +1 is necessary here because value of step seen by user (first step is 1) is different than // value of step in code (first step is 0) - mypc->doResampling(istep[0]+1); + mypc->doResampling(istep[0]+1, verbose); if (num_mirrors>0){ applyMirrors(cur_time); diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 4c3aeca4579..fecd39137d7 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -179,7 +179,7 @@ public: * * @param[in] timestep the current timestep. */ - void doResampling (int timestep); + void doResampling (int timestep, bool verbose); #ifdef WARPX_QED /** If Schwinger process is activated, this function is called at every diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 1a02cabb816..b9802fa448b 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -942,14 +942,14 @@ MultiParticleContainer::doCollisions ( Real cur_time, amrex::Real dt ) collisionhandler->doCollisions(cur_time, dt, this); } -void MultiParticleContainer::doResampling (const int timestep) +void MultiParticleContainer::doResampling (const int timestep, const bool verbose) { for (auto& pc : allcontainers) { // do_resampling can only be true for PhysicalParticleContainers if (!pc->do_resampling){ continue; } - pc->resample(timestep); + pc->resample(timestep, verbose); } } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index c4ad257c95f..ca04bc071a8 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -307,7 +307,7 @@ public: * * @param[in] timestep the current timestep. */ - void resample (int timestep) override final; + void resample (int timestep, bool verbose=true) override final; #ifdef WARPX_QED //Functions decleared in WarpXParticleContainer.H diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 958c800ffc0..1358011b9b0 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2949,7 +2949,7 @@ PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector () return plasma_injector.get(); } -void PhysicalParticleContainer::resample (const int timestep) +void PhysicalParticleContainer::resample (const int timestep, const bool verbose) { // In heavily load imbalanced simulations, MPI processes with few particles will spend most of // the time at the MPI synchronization in TotalNumberOfParticles(). Having two profiler entries @@ -2967,7 +2967,11 @@ void PhysicalParticleContainer::resample (const int timestep) WARPX_PROFILE_VAR_START(blp_resample_actual); if (m_resampler.triggered(timestep, global_numparts)) { - amrex::Print() << Utils::TextMsg::Info("Resampling " + species_name); + if (verbose) { + amrex::Print() << Utils::TextMsg::Info( + "Resampling " + species_name + " at step " + std::to_string(timestep) + ); + } for (int lev = 0; lev <= maxLevel(); lev++) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) @@ -2977,7 +2981,6 @@ void PhysicalParticleContainer::resample (const int timestep) } } WARPX_PROFILE_VAR_STOP(blp_resample_actual); - } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c58b51fa6fb..86691ef8cc2 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -381,7 +381,7 @@ public: * override the method for every derived class. Note that in practice this function is never * called because resample() is only called for PhysicalParticleContainers. */ - virtual void resample (const int /*timestep*/) {} + virtual void resample (const int /*timestep*/, bool /*verbose*/) {} /** * When using runtime components, AMReX requires to touch all tiles From bae7530ff16afd0d348aac4e7c5257ea4d053cfe Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 5 Sep 2023 00:55:19 -0700 Subject: [PATCH 4/5] Use input parameter parser for lattice elements (#4258) --- .../LatticeElements/HardEdgedPlasmaLens.cpp | 5 +++-- .../LatticeElements/HardEdgedQuadrupole.cpp | 5 +++-- .../LatticeElements/LatticeElementBase.cpp | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp index 1c609d697e6..9c8726a9c4b 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp @@ -5,6 +5,7 @@ * License: BSD-3-Clause-LBNL */ #include "HardEdgedPlasmaLens.H" +#include "Utils/Parser/ParserUtils.H" #include #include @@ -25,8 +26,8 @@ HardEdgedPlasmaLens::AddElement (amrex::ParmParse & pp_element, amrex::ParticleR amrex::ParticleReal dEdx = 0._prt; amrex::ParticleReal dBdx = 0._prt; - pp_element.query("dEdx", dEdx); - pp_element.query("dBdx", dBdx); + utils::parser::queryWithParser(pp_element, "dEdx", dEdx); + utils::parser::queryWithParser(pp_element, "dBdx", dBdx); h_dEdx.push_back(dEdx); h_dBdx.push_back(dBdx); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp index cdb6502ecef..51bcf7a2497 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp @@ -5,6 +5,7 @@ * License: BSD-3-Clause-LBNL */ #include "HardEdgedQuadrupole.H" +#include "Utils/Parser/ParserUtils.H" #include #include @@ -25,8 +26,8 @@ HardEdgedQuadrupole::AddElement (amrex::ParmParse & pp_element, amrex::ParticleR amrex::ParticleReal dEdx = 0._prt; amrex::ParticleReal dBdx = 0._prt; - pp_element.query("dEdx", dEdx); - pp_element.query("dBdx", dBdx); + utils::parser::queryWithParser(pp_element, "dEdx", dEdx); + utils::parser::queryWithParser(pp_element, "dBdx", dBdx); h_dEdx.push_back(dEdx); h_dBdx.push_back(dBdx); diff --git a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp index 6ef318057ff..248db59aaf6 100644 --- a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp +++ b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp @@ -5,6 +5,7 @@ * License: BSD-3-Clause-LBNL */ #include "LatticeElementBase.H" +#include "Utils/Parser/ParserUtils.H" #include #include @@ -21,7 +22,7 @@ LatticeElementBase::AddElementBase (amrex::ParmParse & pp_element, amrex::Partic { // Read in the length of the element and save the start and end, and update z_location amrex::ParticleReal ds; - pp_element.get("ds", ds); + utils::parser::getWithParser(pp_element, "ds", ds); h_zs.push_back(z_location); z_location += ds; From 7eada29a6981a06651a90a4e46ca0eed1a2e7676 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Tue, 5 Sep 2023 01:51:04 -0700 Subject: [PATCH 5/5] make explicit the options from full diags also available in BTD (#4256) * make explicit the options from full diags also available in BTD * EOL --- Docs/source/usage/parameters.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 177e08090cd..4514e6f3fb0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2472,7 +2472,10 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a BackTransformed Diagnostics ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``.diag_type = BackTransformed``. Additional options for this diagnostic include: +``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``.diag_type = BackTransformed``. We support the following list of options from `Full Diagnostics`_ + ``.format``, ``.openpmd_backend``, ``.dump_rz_modes``, ``.file_prefix``, ``.diag_lo``, ``.diag_hi``, ``.write_species``, ``.species``. + + Additional options for this diagnostic include: * ``.num_snapshots_lab`` (`integer`) Only used when ``.diag_type`` is ``BackTransformed``.