Skip to content

Commit

Permalink
core: ghosts: Removed redundant GhostCommunicator::num
Browse files Browse the repository at this point in the history
  • Loading branch information
fweik committed Oct 22, 2019
1 parent 33a3bba commit 1283c40
Show file tree
Hide file tree
Showing 9 changed files with 36 additions and 47 deletions.
3 changes: 2 additions & 1 deletion src/core/cells.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,8 @@ void cells_resort_particles(int global_flag) {
}

ghost_communicator(&cell_structure.exchange_ghosts_comm, GHOSTTRANS_PARTNUM);
ghost_communicator(&cell_structure.exchange_ghosts_comm, GHOSTTRANS_POSITION | GHOSTTRANS_PROPRTS);
ghost_communicator(&cell_structure.exchange_ghosts_comm,
GHOSTTRANS_POSITION | GHOSTTRANS_PROPRTS);

/* Particles are now sorted, but Verlet lists are invalid
and p_old has to be reset. */
Expand Down
45 changes: 16 additions & 29 deletions src/core/domain_decomposition.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ using Utils::get_linear_index;
#include "event.hpp"

#include <boost/mpi/collectives.hpp>
#include <boost/range/algorithm/reverse.hpp>

/** Returns pointer to the cell which corresponds to the position if the
* position is in the nodes spatial domain otherwise a nullptr pointer.
Expand Down Expand Up @@ -344,28 +345,18 @@ void dd_prepare_comm(GhostCommunicator *comm, int data_parts,
* communication types GHOST_SEND <-> GHOST_RECV.
*/
void dd_revert_comm_order(GhostCommunicator *comm) {
int i, j, nlist2;
GhostCommunication tmp;

/* revert order */
for (i = 0; i < (comm->num / 2); i++) {
tmp = comm->comm[i];
comm->comm[i] = comm->comm[comm->num - i - 1];
comm->comm[comm->num - i - 1] = tmp;
}
boost::reverse(comm->comm);

/* exchange SEND/RECV */
for (i = 0; i < comm->num; i++) {
if (comm->comm[i].type == GHOST_SEND)
comm->comm[i].type = GHOST_RECV;
else if (comm->comm[i].type == GHOST_RECV)
comm->comm[i].type = GHOST_SEND;
else if (comm->comm[i].type == GHOST_LOCL) {
nlist2 = comm->comm[i].part_lists.size() / 2;
for (j = 0; j < nlist2; j++) {
auto tmplist = comm->comm[i].part_lists[j];
comm->comm[i].part_lists[j] = comm->comm[i].part_lists[j + nlist2];
comm->comm[i].part_lists[j + nlist2] = tmplist;
}
for (auto &c : comm->comm) {
if (c.type == GHOST_SEND)
c.type = GHOST_RECV;
else if (c.type == GHOST_RECV)
c.type = GHOST_SEND;
else if (c.type == GHOST_LOCL) {
boost::reverse(c.part_lists);
}
}
}
Expand All @@ -374,22 +365,18 @@ void dd_revert_comm_order(GhostCommunicator *comm) {
* poststore
*/
void dd_assign_prefetches(GhostCommunicator *comm) {
int cnt;

for (cnt = 0; cnt < comm->num; cnt += 2) {
if (comm->comm[cnt].type == GHOST_RECV &&
comm->comm[cnt + 1].type == GHOST_SEND) {
comm->comm[cnt].type |= GHOST_PREFETCH | GHOST_PSTSTORE;
comm->comm[cnt + 1].type |= GHOST_PREFETCH | GHOST_PSTSTORE;
for (auto it = comm->comm.begin(); it != comm->comm.end(); it += 2) {
auto next = std::next(it);
if (it->type == GHOST_RECV && next->type == GHOST_SEND) {
it->type |= GHOST_PREFETCH | GHOST_PSTSTORE;
next->type |= GHOST_PREFETCH | GHOST_PSTSTORE;
}
}
}

/** update the 'shift' member of those GhostCommunicators, which use
* that value to speed up the folding process of its ghost members
* (see \ref dd_prepare_comm for the original), i.e. all which have
* GHOSTTRANS_POSSHFTD or'd into 'data_parts' upon execution of \ref
* dd_prepare_comm.
* (see \ref dd_prepare_comm for the original).
*/
void dd_update_communicators_w_boxl(const Utils::Vector3i &grid) {
int cnt = 0;
Expand Down
6 changes: 4 additions & 2 deletions src/core/electrostatics_magnetostatics/icc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ int iccp3m_iteration(const ParticleRange &particles,

force_calc_iccp3m(particles, ghost_particles); /* Calculate electrostatic
forces (SR+LR) excluding source source interaction*/
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);

double diff = 0;

Expand Down Expand Up @@ -172,7 +173,8 @@ int iccp3m_iteration(const ParticleRange &particles,
}
} /* cell particles */
/* Update charges on ghosts. */
ghost_communicator(&cell_structure.exchange_ghosts_comm, GHOSTTRANS_PROPRTS);
ghost_communicator(&cell_structure.exchange_ghosts_comm,
GHOSTTRANS_PROPRTS);

iccp3m_cfg.citeration++;

Expand Down
3 changes: 2 additions & 1 deletion src/core/forces.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,8 @@ void force_calc(CellStructure &cell_structure) {
#endif

// Communication Step: ghost forces
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);

// should be pretty late, since it needs to zero out the total force
comfixed.apply(comm_cart, particles);
Expand Down
11 changes: 4 additions & 7 deletions src/core/ghosts.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,6 @@ bool ghosts_have_bonds = false;

void prepare_comm(GhostCommunicator *gcr, int num) {
assert(gcr);
gcr->num = num;
gcr->comm.resize(num);
for (auto &ghost_comm : gcr->comm)
ghost_comm.shift.fill(0.0);
Expand Down Expand Up @@ -409,8 +408,8 @@ void ghost_communicator(GhostCommunicator *gcr, int data_parts) {
if (ghosts_have_v && (data_parts & GHOSTTRANS_POSITION))
data_parts |= GHOSTTRANS_MOMENTUM;

for (int n = 0; n < gcr->num; n++) {
GhostCommunication &ghost_comm = gcr->comm[n];
for (auto it = gcr->comm.begin(); it != gcr->comm.end(); ++it) {
GhostCommunication &ghost_comm = *it;
int const comm_type = ghost_comm.type & GHOST_JOBMASK;

if (comm_type == GHOST_LOCL) {
Expand All @@ -434,8 +433,7 @@ void ghost_communicator(GhostCommunicator *gcr, int data_parts) {
} else if (prefetch) {
/* we do not send this time, let's look for a prefetch */
auto prefetch_ghost_comm =
std::find_if(std::next(gcr->comm.begin(), n + 1), gcr->comm.end(),
is_prefetchable);
std::find_if(std::next(it), gcr->comm.end(), is_prefetchable);
if (prefetch_ghost_comm != gcr->comm.end())
prepare_send_buffer(send_buffer, *prefetch_ghost_comm, data_parts);
}
Expand Down Expand Up @@ -498,8 +496,7 @@ void ghost_communicator(GhostCommunicator *gcr, int data_parts) {
* prefetch send. */
/* find previous action where we recv and which has PSTSTORE set */
auto poststore_ghost_comm = std::find_if(
std::make_reverse_iterator(std::next(gcr->comm.begin(), n)),
gcr->comm.rend(), is_poststorable);
std::make_reverse_iterator(it), gcr->comm.rend(), is_poststorable);

if (poststore_ghost_comm != gcr->comm.rend()) {
assert(recv_buffer.size() ==
Expand Down
3 changes: 0 additions & 3 deletions src/core/ghosts.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,6 @@ struct GhostCommunication {

/** Properties for a ghost communication. A ghost communication is defined */
struct GhostCommunicator {
/** number of communication steps. */
int num;

/** List of ghost communications. */
std::vector<GhostCommunication> comm;
};
Expand Down
6 changes: 4 additions & 2 deletions src/core/rattle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ void correct_pos_shake(ParticleRange const &particles) {
init_correction_vector(cell_structure.local_cells().particles());
repeat_ = 0;
compute_pos_corr_vec(&repeat_, cell_structure.local_cells().particles());
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);
app_pos_correction(cell_structure.local_cells().particles());
/**Ghost Positions Update*/
ghost_communicator(&cell_structure.exchange_ghosts_comm,
Expand Down Expand Up @@ -291,7 +292,8 @@ void correct_vel_shake() {
init_correction_vector(particles);
repeat_ = 0;
compute_vel_corr_vec(&repeat_, cell_structure.local_cells().particles());
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);
apply_vel_corr(particles);
ghost_communicator(&cell_structure.exchange_ghosts_comm,
GHOSTTRANS_POSITION);
Expand Down
3 changes: 2 additions & 1 deletion src/core/virtual_sites/VirtualSitesRelative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ void VirtualSitesRelative::update_vel(Particle &p) const {
// Distribute forces that have accumulated on virtual particles to the
// associated real particles
void VirtualSitesRelative::back_transfer_forces_and_torques() const {
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);
init_forces_ghosts(cell_structure.ghost_cells().particles());

// Iterate over all the particles in the local cells
Expand Down
3 changes: 2 additions & 1 deletion src/core/virtual_sites/lb_inertialess_tracers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,8 @@ void ParticleVelocitiesFromLB_CPU() {
// real particles
// This could be solved by keeping a backup of the local forces before this
// operation is attempted
ghost_communicator(&cell_structure.collect_ghost_force_comm, GHOSTTRANS_FORCE);
ghost_communicator(&cell_structure.collect_ghost_force_comm,
GHOSTTRANS_FORCE);

// Transfer to velocity field
for (int c = 0; c < local_cells.n; c++) {
Expand Down

0 comments on commit 1283c40

Please sign in to comment.