From c1ec1024f633cd5ed11a7164cf653a8afc3103e3 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Sat, 16 Dec 2023 18:33:42 -0800 Subject: [PATCH 1/6] Docs: rename and spruce up boundary conditions section (#4537) * Docs: rename and spruce up boundary conditions section --- Docs/source/index.rst | 2 +- Docs/source/theory/PML.rst | 242 ---------------- Docs/source/theory/boundary_conditions.rst | 303 +++++++++++++++++++++ 3 files changed, 304 insertions(+), 243 deletions(-) delete mode 100644 Docs/source/theory/PML.rst create mode 100644 Docs/source/theory/boundary_conditions.rst diff --git a/Docs/source/index.rst b/Docs/source/index.rst index e5841da1f2d..9f2d2ff038e 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -109,7 +109,7 @@ Theory theory/intro theory/pic theory/amr - theory/PML + theory/boundary_conditions theory/boosted_frame theory/input_output theory/collisions diff --git a/Docs/source/theory/PML.rst b/Docs/source/theory/PML.rst deleted file mode 100644 index d60be931c5f..00000000000 --- a/Docs/source/theory/PML.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. _theory-bc: - -Boundary conditions -=================== - -Open boundary condition for electromagnetic waves -------------------------------------------------- - -For the TE case, the original Berenger’s Perfectly Matched Layer (PML) writes - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{\partial H_{z}}{\partial y}\label{PML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{\partial H_{z}}{\partial x}\label{PML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{\partial E_{y}}{\partial x}\label{PML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{\partial E_{x}}{\partial y}\label{PML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{PML_def_5}\end{aligned} - -This can be generalized to - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z}\label{APML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z}\label{APML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y}\label{APML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x}\label{APML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{APML_def_5}\end{aligned} - -For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, -this system reduces to the Berenger PML medium, while adding the additional -constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` -leads to the system of Maxwell equations in vacuum. - -.. _theory-bc-propa-plane-wave: - -Propagation of a Plane Wave in an APML Medium -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) -and pulsation :math:`\omega` propagating in the APML medium with an -angle :math:`\varphi` relative to the x axis - -.. math:: - - \begin{aligned} - E_{x} = & -E_{0}\sin \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_1} \\ - E_{y} = & E_{0}\cos \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_2} \\ - H_{zx} = & H_{zx0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_AMPL_def_3} \\ - H_{zy} = & H_{zy0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_4}\end{aligned} - -where :math:`\alpha` and\ :math:`\beta` are two complex constants to -be determined. - -Introducing (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_def_2] <#Plane_wave_APML_def_2>`__), -(`[Plane_wave_AMPL_def_3] <#Plane_wave_AMPL_def_3>`__) and (`[Plane_wave_APML_def_4] <#Plane_wave_APML_def_4>`__) -into (`[APML_def_1] <#APML_def_1>`__), (`[APML_def_2] <#APML_def_2>`__), (`[APML_def_3] <#APML_def_3>`__) -and (`[APML_def_4] <#APML_def_4>`__) gives - -.. math:: - - \begin{aligned} - \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = & \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_1} \\ - \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = & \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_2} \\ - \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = & \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi \label{Plane_wave_APML_1_3} \\ - \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = & \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi \label{Plane_wave_APML_1_4}\end{aligned} - -Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using (`[Plane_wave_APML_1_1] <#Plane_wave_APML_1_1>`__) -and (`[Plane_wave_APML_1_2] <#Plane_wave_APML_1_2>`__), we get - -.. math:: - - \begin{aligned} - \beta = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}}\label{Plane_wave_APML_beta_of_g} \\ - \alpha = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}}\label{Plane_wave_APML_alpha_of_g}\end{aligned} - -Adding :math:`H_{zx0}` and :math:`H_{zy0}` from (`[Plane_wave_APML_1_3] <#Plane_wave_APML_1_3>`__) -and (`[Plane_wave_APML_1_4] <#Plane_wave_APML_1_4>`__) and substituting the expressions -for :math:`\alpha` and :math:`\beta` from (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__) -and (`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) yields - -.. math:: - - \begin{aligned} - \frac{1}{Z} = & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber \\ - + & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi\end{aligned} - -If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then - -.. math:: - - \begin{aligned} - Z = & \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}}\label{APML_impedance}\end{aligned} - -which is the impedance of vacuum. Hence, like the PML, given some -restrictions on the parameters, the APML does not generate any reflection -at any angle and any frequency. As for the PML, this property is not -retained after discretization, as shown subsequently in this paper. - -Calling :math:`\psi` any component of the field and :math:`\psi _{0}` -its magnitude, we get from (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__), -(`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) and (`[APML_impedance] <#APML_impedance>`__) that - -.. math:: - - \label{Plane_wave_absorption} - \psi =\psi _{0}e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x}e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y} - -We assume that we have an APML layer of thickness :math:`\delta` (measured -along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` -and :math:`c_{y}=c.` Using (`[Plane_wave_absorption] <#Plane_wave_absorption>`__), we determine -that the coefficient of reflection given by this layer is - -.. math:: - - \begin{aligned} - R_{APML}\left( \theta \right) = & e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta }\nonumber \\ - = & e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }\end{aligned} - -which happens to be the same as the PML theoretical coefficient of -reflection if we assume :math:`c_{x}=c`. Hence, it follows that for -the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` -seems to be of no interest. However, although this conclusion is true -at the infinitesimal limit, it does not hold for the discretized counterpart. - -Discretization -~~~~~~~~~~~~~~ - -.. math:: - - \begin{aligned} - \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = & \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} \\ - % - \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = & - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n}_{j+1/2,k+1/2,l}}{2} = & - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n}_{j+1/2,k+1/2,l}}{2} = & \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - c_x = & c e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta x}{1-e^{-\sigma_x\Delta t}} \\ - c_y = & c e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta y}{1-e^{-\sigma_y\Delta t}} \\ - c^*_x = & c e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta x}{1-e^{-\sigma^*_x\Delta t}} \\ - c^*_y = & c e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta y}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. _theory-bc-pec: - -Perfect Electrical Conductor ----------------------------- - -This boundary can be used to model a dielectric or metallic surface. -For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic -field are set to 0. In the guard-cell region, the tangential electric field is set equal and -opposite to the respective field component in the mirror location across the PEC -boundary, and the normal electric field is set equal to the field component in the -mirror location in the domain across the PEC boundary. Similarly, the tangential -(and normal) magnetic field components are set equal (and opposite) to the respective -magnetic field components in the mirror locations across the PEC boundary. - -The PEC boundary condition also impacts the deposition of charge and current density. -On the boundary the charge density and parallel current density is set to zero. If -a reflecting boundary condition is used for the particles, density overlapping -with the PEC will be reflected back into the domain (for both charge and current -density). If absorbing boundaries are used, an image charge (equal weight but -opposite charge) is considered in the mirror location accross the boundary, and -the density from that charge is also deposited in the simulation domain. The -figure below shows the effect of this. The left boundary is absorbing while -the right boundary is reflecting. - -.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png - :alt: PEC boundary deposition - :width: 80% diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst new file mode 100644 index 00000000000..395b072ccbe --- /dev/null +++ b/Docs/source/theory/boundary_conditions.rst @@ -0,0 +1,303 @@ +.. _theory-bc: + +Boundary conditions +=================== + +.. _theory-bc-PML: + +Perfectly Matched Layer: open boundary condition for electromagnetic waves +-------------------------------------------------------------------------- + +For the transverse electric (TE) case, the original Berenger’s Perfectly Matched Layer (PML) paper :cite:p:`bc-Berengerjcp94` writes + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{\partial H_{z}}{\partial y} + :label: PML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{\partial H_{z}}{\partial x} + :label: PML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{\partial E_{y}}{\partial x} + :label: PML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{\partial E_{x}}{\partial y} + :label: PML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: PML_def_5 + +This can be generalized to + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z} + :label: APML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z} + :label: APML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y} + :label: APML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x} + :label: APML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: APML_def_5 + +For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, +this system reduces to the Berenger PML medium, while adding the additional +constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` +leads to the system of Maxwell equations in vacuum. + +.. _theory-bc-propa-plane-wave: + +Propagation of a Plane Wave in an APML Medium +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) +and pulsation :math:`\omega` propagating in the APML medium with an +angle :math:`\varphi` relative to the x axis + +.. math:: + E_{x} = -E_{0}\sin \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_1 + +.. math:: + E_{y} = E_{0}\cos \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_2 + +.. math:: + H_{zx} = H_{zx0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_AMPL_def_3 + +.. math:: + H_{zy} = H_{zy0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_4 + +where :math:`\alpha` and :math:`\beta` are two complex constants to +be determined. + +Introducing Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_def_2`), +(:eq:`Plane_wave_AMPL_def_3`) and (:eq:`Plane_wave_APML_def_4`) +into Eqs. (:eq:`APML_def_1`), (:eq:`APML_def_2`), (:eq:`APML_def_3`) +and (:eq:`APML_def_4`) gives + +.. math:: + \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_1 + +.. math:: + \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_2 + +.. math:: + \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi + :label: Plane_wave_APML_1_3 + +.. math:: + \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi + :label: Plane_wave_APML_1_4 + +Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using Eqs. (:eq:`Plane_wave_APML_1_1`) +and (:eq:`Plane_wave_APML_1_2`), we get + +.. math:: + \beta = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}} + :label: Plane_wave_APML_beta_of_g + +.. math:: + \alpha = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}} + :label: Plane_wave_APML_alpha_of_g + +Adding :math:`H_{zx0}` and :math:`H_{zy0}` from Eqs. (:eq:`Plane_wave_APML_1_3`) +and (:eq:`Plane_wave_APML_1_4`) and substituting the expressions +for :math:`\alpha` and :math:`\beta` from Eqs. (:eq:`Plane_wave_APML_beta_of_g`) +and (:eq:`Plane_wave_APML_alpha_of_g`) yields + +.. math:: + + \begin{aligned} + \frac{1}{Z} & = \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber + \\ + & + \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi + \end{aligned} + +If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then + +.. math:: + Z = \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}} + :label: APML_impedance + +which is the impedance of vacuum. Hence, like the PML, given some +restrictions on the parameters, the APML does not generate any reflection +at any angle and any frequency. As for the PML, this property is not +retained after discretization, as shown subsequently. + +Calling :math:`\psi` any component of the field and :math:`\psi _{0}` +its magnitude, we get from Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_beta_of_g`), +(:eq:`Plane_wave_APML_alpha_of_g`) and (:eq:`APML_impedance`) that + +.. math:: + \psi =\psi _{0} \: e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x} e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y}. + :label: Plane_wave_absorption + +We assume that we have an APML layer of thickness :math:`\delta` (measured +along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` +and :math:`c_{y}=c.` Using (:eq:`Plane_wave_absorption`), we determine +that the coefficient of reflection given by this layer is + +.. math:: + + \begin{aligned} + R_{\mathrm{APML}}\left( \theta \right) & = e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta },\nonumber + \\ + & = e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }, + \end{aligned} + +which happens to be the same as the PML theoretical coefficient of +reflection if we assume :math:`c_{x}=c`. Hence, it follows that for +the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` +seems to be of no interest. However, although this conclusion is true +at the infinitesimal limit, it does not hold for the discretized counterpart. + +Discretization +~~~~~~~~~~~~~~ + +In the following we set :math:`\varepsilon_0 = \mu_0 = 1`. We discretize Eqs. (:eq:`PML_def_1`), (:eq:`PML_def_2`), (:eq:`PML_def_3`), and (:eq:`PML_def_4`) to obtain + +.. math:: + \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} + +.. math:: + \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} + +and this can be solved to obtain the following leapfrog integration equations + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we account for higher order :math:`\Delta t` terms, a better approximation is given by + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +More generally, this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we set + +.. math:: + + \begin{aligned} + c_x & = c \: e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta t}{1-e^{-\sigma_x\Delta t}} \\ + c_y & = c \: e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta t}{1-e^{-\sigma_y\Delta t}} \\ + c^*_x & = c \: e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta t}{1-e^{-\sigma^*_x\Delta t}} \\ + c^*_y & = c \: e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta t}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} + +then this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] + \end{aligned} + +When the generalized conductivities are zero, the update equations are + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +as expected. + +.. _theory-bc-pec: + +Perfect Electrical Conductor +---------------------------- + +This boundary can be used to model a dielectric or metallic surface. +For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic +field are set to 0. In the guard-cell region, the tangential electric field is set equal and +opposite to the respective field component in the mirror location across the PEC +boundary, and the normal electric field is set equal to the field component in the +mirror location in the domain across the PEC boundary. Similarly, the tangential +(and normal) magnetic field components are set equal (and opposite) to the respective +magnetic field components in the mirror locations across the PEC boundary. + +The PEC boundary condition also impacts the deposition of charge and current density. +On the boundary the charge density and parallel current density is set to zero. If +a reflecting boundary condition is used for the particles, density overlapping +with the PEC will be reflected back into the domain (for both charge and current +density). If absorbing boundaries are used, an image charge (equal weight but +opposite charge) is considered in the mirror location accross the boundary, and +the density from that charge is also deposited in the simulation domain. :numref:`fig_PEC_boundary_deposition` +shows the effect of this. The left boundary is absorbing while +the right boundary is reflecting. + +.. _fig_PEC_boundary_deposition: + +.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png + :alt: Plot of PEC boundary current deposition showing current vs position along the ``x``-axis. + :width: 100% + + PEC boundary current deposition along the ``x``-axis. The left boundary is absorbing while the right boundary is reflecting. + +.. bibliography:: + :keyprefix: bc- From 9af087d70c5104b8c7150ef8a01aee66e885bf56 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Sun, 17 Dec 2023 11:13:11 -0800 Subject: [PATCH 2/6] Spruce up kinetic_fluid_hybrid_model.rst (#4534) --- Docs/source/refs.bib | 22 ++++++++++--------- .../theory/kinetic_fluid_hybrid_model.rst | 6 ++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 65545dfca87..8d001cf971e 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -60,20 +60,22 @@ @article{Turner2013 year = {2013} } -@article{winske2022hybrid, -archivePrefix = {arXiv}, -author = {D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -eprint = {2204.01676}, -journal = {arXiv}, -primaryClass = {physics.plasm-ph}, -title = {{Hybrid codes (massless electron fluid)}}, -year = {2022} +@Inbook{WinskeInBook2023, +author = {Winske, Dan and Karimabadi, Homa and Le, Ari Yitzchak and Omidi, Nojan Nick and Roytershteyn, Vadim and Stanier, Adam John}, +bookTitle = {Space and Astrophysical Plasma Simulation: Methods, Algorithms, and Applications}, +doi = {10.1007/978-3-031-11870-8_3}, +editor = {B{\"u}chner, J{\"o}rg}, +isbn = {978-3-031-11870-8}, +pages = {63--91}, +publisher = {Springer International Publishing}, +title = {{Hybrid-Kinetic Approach: Massless Electrons}}, +year = {2023} } -@incollection{NIELSON1976, +@incollection{Nielson1976, author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, -doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, +doi = {10.1016/B978-0-12-460816-0.50015-4}, editor = {John Killeen}, issn = {0076-6860}, pages = {367--388}, diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 69797352bee..55bce704fbb 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -19,7 +19,7 @@ of light. Many authors have described variations of the kinetic ion & fluid electron model, generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`c-winske2022hybrid`. +in WarpX follows the outline from :cite:t:`kfhm-WinskeInBook2023`. This description follows mostly from that reference. Model @@ -29,7 +29,7 @@ The basic justification for the hybrid model is that the system to which it is applied is dominated by ion kinetics, with ions moving much slower than electrons and photons. In this scenario two critical approximations can be made, namely, neutrality (:math:`n_e=n_i`) and the Maxwell-Ampere equation can be simplified by -neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, +neglecting the displacement current term :cite:p:`kfhm-Nielson1976`, giving, .. math:: @@ -168,4 +168,4 @@ The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. .. bibliography:: - :keyprefix: c- + :keyprefix: kfhm- From 8c307c712f26a47c6775eb45b30e3b71c023936f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 17 Dec 2023 13:07:34 -0800 Subject: [PATCH 3/6] Doc: CUDA 11.7+ (#4538) We know of a few compiler issues for C++17 CUDA code in CUDA <11.7 for a bit, updating Summit and general recommendations. --- Docs/source/install/dependencies.rst | 2 +- Tools/machines/summit-olcf/summit_warpx.profile.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 7bd6c7dc8d5..a939db840da 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -18,7 +18,7 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.0+ (11.3+ recommended) `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support - `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index 5d88bb9aeed..e41521b5815 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -11,7 +11,7 @@ module load nano # required dependencies module load cmake/3.20.2 module load gcc/9.3.0 -module load cuda/11.3.1 +module load cuda/11.7.1 # optional: faster re-builds module load ccache From 244ec1cf43bab800d107440ea730389c8486a617 Mon Sep 17 00:00:00 2001 From: Andrei Berceanu Date: Sun, 17 Dec 2023 23:09:42 +0200 Subject: [PATCH 4/6] Add updated instructions for WarpX on Karolina (#4477) * Add updated instructions for WarpX on Karolina Use spack for installing all dependencies, and the new slurm scheduler. * Update Tools/machines/karolina-it4i/install_dependencies.sh End for loop Co-authored-by: Axel Huebl --- Docs/source/install/hpc/karolina.rst | 195 +++++------------- Tools/machines/karolina-it4i/cleanup.sh | 3 + .../karolina-it4i/install_cpu_dependencies.sh | 141 ------------- .../karolina-it4i/install_dependencies.sh | 48 +++++ .../karolina-it4i/install_gpu_dependencies.sh | 141 ------------- .../karolina_cpu_warpx.profile.example | 61 ------ .../machines/karolina-it4i/karolina_gpu.qsub | 29 --- .../karolina-it4i/karolina_gpu.sbatch | 39 ++++ .../karolina_gpu_warpx.profile.example | 65 ------ .../karolina_warpx.profile.example | 69 +++++++ .../karolina-it4i/spack-karolina-cuda.yaml | 80 +++++++ 11 files changed, 293 insertions(+), 578 deletions(-) create mode 100755 Tools/machines/karolina-it4i/cleanup.sh delete mode 100755 Tools/machines/karolina-it4i/install_cpu_dependencies.sh create mode 100755 Tools/machines/karolina-it4i/install_dependencies.sh delete mode 100755 Tools/machines/karolina-it4i/install_gpu_dependencies.sh delete mode 100644 Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example delete mode 100644 Tools/machines/karolina-it4i/karolina_gpu.qsub create mode 100644 Tools/machines/karolina-it4i/karolina_gpu.sbatch delete mode 100644 Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example create mode 100644 Tools/machines/karolina-it4i/karolina_warpx.profile.example create mode 100644 Tools/machines/karolina-it4i/spack-karolina-cuda.yaml diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 7a8bf561986..fffb917df1c 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -12,83 +12,60 @@ Introduction If you are new to this system, **please see the following resources**: * `IT4I user guide `__ -* Batch system: `PBS `__ +* Batch system: `SLURM `__ * Jupyter service: not provided/documented (yet) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scratch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (10TB default) + * ``/mnt/proj/``: per-project work directory, used for long term data storage (20TB default) .. _building-karolina-preparation: -Preparation ------------ - -Use the following commands to download the WarpX source code: +Installation +------------ -.. code-block:: bash +We show how to install from scratch all the dependencies using `Spack `__. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx +For size reasons it is not advisable to install WarpX in the ``$HOME`` directory, it should be installed in the "work directory". For this purpose we set an environment variable ``$WORK`` with the path to the "work directory". On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. -.. tab-set:: - - .. tab-item:: A100 GPUs - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash +Profile file +^^^^^^^^^^^^ - export proj="DD-23-83" +One can use the pre-prepared ``karolina_warpx.profile`` script below, +which you can copy to ``${HOME}/karolina_warpx.profile``, edit as required and then ``source``. - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. important:: + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_warpx.profile.example + :language: bash + :caption: Copy the contents of this file to ``${HOME}/karolina_warpx.profile``. - Now, and as the first step on future logins to Karolina, activate these environment settings: +To have the environment activated on every login, add the following line to ``${HOME}/.bashrc``: - .. code-block:: bash - - source $HOME/karolina_gpu_warpx.profile - - Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once: - - .. code-block:: bash +.. code-block:: bash - bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh - source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate + source $HOME/karolina_warpx.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +To install the ``spack`` environment and Python packages: - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh - :language: bash +.. code-block:: bash + bash $WORK/src/warpx/Tools/machines/karolina-it4i/install_dependencies.sh - .. tab-item:: CPU Nodes +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - CPU usage is documentation is TODO. + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_dependencies.sh + :language: bash .. _building-karolina-compilation: @@ -98,117 +75,53 @@ Compilation Use the following :ref:`cmake commands ` to compile the application executable: -.. tab-set:: - - .. tab-item:: A100 GPUs - - .. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_gpu - - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu -j 12 - - The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: - - .. code-block:: bash - - rm -rf build_gpu_py - - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu_py -j 12 --target pip_install - - .. tab-item:: CPU Nodes - - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu + cd $WORK/src/warpx + rm -rf build_gpu - cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu -j 12 + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 48 - The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: +The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu_py + cd $WORK/src/warpx + rm -rf build_gpu_py - cmake -S . -B build_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu_py -j 12 --target pip_install + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. -.. _building-karolina-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. - - .. _running-cpp-karolina: Running ------- -.. tab-set:: +The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#SBATCH --nodes=`` accordingly) on the supercomputer Karolina at IT4I. +This partition has up to `72 nodes `__. +Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. - .. tab-item:: A100 (40GB) GPUs +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. +Note that we run one MPI rank per GPU. - The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I. - This partition as up to `72 nodes `__. - Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. +.. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.sbatch + :language: bash + :caption: You can copy this file from ``$WORK/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.sbatch``. - Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. - Note that we run one MPI rank per GPU. - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub - :language: bash - :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``. - - To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run - - .. code-block:: bash - - qsub karolina_gpu.qsub - - to submit the job. +To run a simulation, copy the lines above to a file ``karolina_gpu.sbatch`` and run +.. code-block:: bash - .. tab-item:: CPU Nodes + sbatch karolina_gpu.sbatch - CPU usage is documentation is TODO. +to submit the job. .. _post-processing-karolina: diff --git a/Tools/machines/karolina-it4i/cleanup.sh b/Tools/machines/karolina-it4i/cleanup.sh new file mode 100755 index 00000000000..b4d46edd1e4 --- /dev/null +++ b/Tools/machines/karolina-it4i/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ${HOME}/.spack ${WORK}/src/warpx ${WORK}/spack ${HOME}/.local/lib/python3.11 diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh deleted file mode 100755 index 2695435738f..00000000000 --- a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_cpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_cpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/cpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-cpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-cpu-build --target install --parallel 16 -rm -rf $HOME/src/adios2-cpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-cpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-cpu-build -Duse_openmp=ON -Dcpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-cpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-cpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-cpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-cpu -python3 -m venv ${SW_DIR}/venvs/warpx-cpu -source ${SW_DIR}/venvs/warpx-cpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh new file mode 100755 index 00000000000..c9c084d5aa7 --- /dev/null +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ################################# +# +set -eu -o pipefail + +# Check: ########################################################## +# +# Was karolina_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your karolina_warpx.profile file!" + echo "Please edit its line 2 to continue!" + return +fi + +# download and activate spack +# this might take about ~ 1 hour +if [ ! -d "$WORK/spack" ] +then + git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack + source $WORK/spack/share/spack/setup-env.sh + + # create and activate the spack environment + spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml + spack env activate warpx-karolina-cuda + spack install +fi + +# Python ########################################################## +# +python -m pip install --user --upgrade pandas +python -m pip install --user --upgrade matplotlib +# optional +#python -m pip install --user --upgrade yt + +# install or update WarpX dependencies +python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade lasy + +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# python -m pip install --user --upgrade -r $WORK/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/karolina-it4i/install_gpu_dependencies.sh deleted file mode 100755 index a03bb008816..00000000000 --- a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_gpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-gpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-gpu-build --target install --parallel 12 -rm -rf $HOME/src/adios2-gpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/blaspp-gpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/lapackpp-gpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-gpu -python3 -m venv ${SW_DIR}/venvs/warpx-gpu -source ${SW_DIR}/venvs/warpx-gpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example deleted file mode 100644 index c0a3ed53ee3..00000000000 --- a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example +++ /dev/null @@ -1,61 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module load GCCcore/11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load OpenMPI/4.1.4-GCC-11.3.0 - -# optional: for QED support with detailed tables -module load Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -module load OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#module load ccache - -# optional: for Python bindings or libEnsemble -module load Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/cpu/venvs/warpx-cpu" ] -then - source ${HOME}/sw/karolina/cpu/venvs/warpx-cpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -#alias getNode="salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -#alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.qsub b/Tools/machines/karolina-it4i/karolina_gpu.qsub deleted file mode 100644 index 274184ed1ca..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu.qsub +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Authors: Axel Huebl, Andrei Berceanu -# License: BSD-3-Clause-LBNL - -#PBS -q qgpu -#PBS -N WarpX -# Use two full nodes, 8 GPUs per node, 16 GPUs total -#PBS -l select=2:ncpus=128:ngpus=8:mpiprocs=8:ompthreads=16,walltime=00:10:00 -#PBS -A - -cd ${PBS_O_WORKDIR} - -# executable & inputs file or python interpreter & PICMI script here -EXE=./warpx.rz -INPUTS=inputs_rz - -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 - -# run -mpirun -np ${PBS_NP} bash -c " - export CUDA_VISIBLE_DEVICES=\${OMPI_COMM_WORLD_LOCAL_RANK}; - ${EXE} ${INPUTS}" \ - > output.txt diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch new file mode 100644 index 00000000000..94af0fa5b0e --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -0,0 +1,39 @@ +#!/bin/bash -l + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +#SBATCH --account= +#SBATCH --partition=qgpu +#SBATCH --time=00:10:00 +#SBATCH --job-name=WarpX +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus-per-node=8 +#SBATCH --gpu-bind=single:1 + +#SBATCH --mail-type=ALL +# change me! +#SBATCH --mail-user=someone@example.com +#SBATCH --chdir=/scratch/project//it4i-/runs/warpx + +#SBATCH -o stdout_%j +#SBATCH -e stderr_%j + +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=16 + +# set user rights to u=rwx;g=r-x;o=--- +umask 0027 + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx.rz +INPUTS=./inputs_rz + +# run +srun -K1 ${EXE} ${INPUTS} diff --git a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example deleted file mode 100644 index 174598acaac..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example +++ /dev/null @@ -1,65 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module purge -ml GCCcore/11.3.0 -ml CUDA/11.7.0 -ml OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -ml CMake/3.23.1-GCCcore-11.3.0 - -# optional: for QED support with detailed tables -ml Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -ml OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#ml ccache - -# optional: for Python bindings or libEnsemble -ml Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/gpu/venvs/warpx-gpu" ] -then - source ${HOME}/sw/karolina/gpu/venvs/warpx-gpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -alias getNode="qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00 -I" -# an alias to run a command on a batch node for up to 1hr -# usage: runNode -alias runNode='echo -e "#!/bin/bash\nmpirun -n 4 $1" | qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00' - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example new file mode 100644 index 00000000000..5fbd144300f --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -0,0 +1,69 @@ +# please set your project account, ie DD-N-N +export proj="" # change me! + +# Name and Path of this Script ################### (DO NOT change!) +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) + +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file!" + echo "Please edit its line 2 to continue!" + return +fi + +# set env variable storing the path to the work directory +# please check if your project ID belongs to proj1, proj2, proj3 etc +export WORK="/mnt/proj/${proj,,}/${USER}" # change me! +mkdir -p WORK + +# clone warpx +# you can also clone your own fork here, eg git@github.com:/WarpX.git +if [ ! -d "$WORK/src/warpx" ] +then + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx +fi + +# load required modules +module purge +module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + +# download and activate spack +if [ ! -d "$WORK/spack" ] +then + echo "WARNING: Spack installation not detected, please run install_dependencies.sh" +else + # activate the spack environment + source $WORK/spack/share/spack/setup-env.sh + spack env activate warpx-karolina-cuda +fi + +# Text Editor for Tools ########################## (edit this line) +# examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" +#export EDITOR="nano" # change me! + +# allocate an interactive shell for one hour +# usage: getNode 2 # allocates two interactive nodes (default: 1) +function getNode() { + if [ -z "$1" ] ; then + numNodes=1 + else + numNodes=$1 + fi + export OMP_NUM_THREADS=16 + srun --time=1:00:00 --nodes=$numNodes --ntasks=$((8 * $numNodes)) --ntasks-per-node=8 --cpus-per-task=16 --exclusive --gpus-per-node=8 -p qgpu -A $proj --pty bash +} + +# Environment ##################################################### +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH="8.0" +export SCRATCH="/scratch/project/${proj,,}/${USER}" + +# optimize CPU microarchitecture for AMD EPYC 7763 (zen3) +export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml new file mode 100644 index 00000000000..1cb6a4ac209 --- /dev/null +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -0,0 +1,80 @@ +spack: + specs: +# Karolina's openssl version is deprecated + - openssl certs=system + - pkgconfig + - ccache + - cmake@3.26.5 + - cuda@11.7.0 + - openmpi@4.1.4 +atomics + - fftw + - hdf5@1.14.0 + - adios2@2.9.2 ~mgard + - blaspp + - lapackpp + - boost@1.81.0 +program_options +atomic ~python + - python@3.11.6 + - py-pip + - py-setuptools + - py-wheel + - py-cython + - py-mpi4py + - py-numpy@1.24.2 + - openpmd-api@0.15.2 +python + - py-periodictable@1.5.0 + - py-h5py +# optional +# - py-libensemble +nlopt + + packages: + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + buildable: False + cuda: + externals: + - spec: cuda@11.7.0 + modules: + - CUDA/11.7.0 + buildable: False + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.4 +atomics +cuda %gcc@11.3.0 + modules: + - OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + libfabric: + externals: + - spec: libfabric@1.15.1 %gcc@11.3.0 + modules: + - libfabric/1.15.1-GCCcore-11.3.0 + buildable: False + all: + target: [zen3] + compiler: [gcc@11.3.0] + variants: +mpi ~fortran +cuda cuda_arch=80 + providers: + mpi: [openmpi@4.1.4] + cuda: [cuda@11.7.0] + + compilers: + - compiler: + modules: [GCCcore/11.3.0] + operating_system: centos7 + paths: + cc: /apps/all/GCCcore/11.3.0/bin/gcc + cxx: /apps/all/GCCcore/11.3.0/bin/g++ + f77: /apps/all/GCCcore/11.3.0/bin/gfortran + fc: /apps/all/GCCcore/11.3.0/bin/gfortran + spec: gcc@=11.3.0 + target: x86_64 + flags: {} + environment: {} + extra_rpaths: [] + + view: true + concretizer: + reuse: false + unify: true From e8f54f2986033ef6e981861760aea1414ef53d47 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 23:49:25 -0800 Subject: [PATCH 5/6] [pre-commit.ci] pre-commit autoupdate (#4539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.13.0 → 5.13.2](https://github.com/pycqa/isort/compare/5.13.0...5.13.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 21577b0604a..35c38ac1ad6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.13.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) From e460e608a303b642023b2c8418bf5ae2ec64a5f5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 19 Dec 2023 12:11:04 +0100 Subject: [PATCH 6/6] AMReX/pyAMReX/PICSAR: Weekly Update (#4541) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ff50edc428e..2874b9dea7c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - + cd ../amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index ceba435d251..40eb6cd4802 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = ef38229189e3213f992a2e89dbe304fb49db9287 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 2934c686c4d..7fa7162891a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = ef38229189e3213f992a2e89dbe304fb49db9287 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 2b044f3a485..bc04048e2fe 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "ecaa46d0be4b5c79b8806e48e3469000d8bb7252" +set(WarpX_amrex_branch "ef38229189e3213f992a2e89dbe304fb49db9287" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 57e378c26a0..9bb9cf427cd 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "0b2d3f6b160991834534164b7391080fabc48ddb" +set(WarpX_pyamrex_branch "91bbbbab13a0651781b6830f64d362dc3d15a3ea" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 8da087155ba..8ddca959861 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - +cd amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets