From d6fa63a5c4d0b780f47ac0956f9f12a0eea389f3 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 17:22:52 +0000 Subject: [PATCH 001/101] Add function to compute submap indices --- cpp/dolfinx/common/IndexMap.cpp | 274 +++++++++++++++++++++++++++++ cpp/dolfinx/common/IndexMap.h | 4 + python/dolfinx/wrappers/common.cpp | 12 +- 3 files changed, 289 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 9b3ee431e1b..dd811f5e85a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,9 +14,39 @@ #include #include +#include + using namespace dolfinx; using namespace dolfinx::common; +std::stringstream ss; + +// A function for printing vectors +template +std::ostream &operator<<(std::ostream &os, + const std::vector &vector) +{ + os << "{ "; + for (auto v : vector) + { + os << v << " "; + } + os << "}"; + return os; +} + +// A function for printing maps +template +std::ostream &operator<<(std::ostream &os, + const std::map &map) +{ + os << "{ "; + for (const auto &[k, v] : map) + os << k << ": " << v << " "; + os << "}"; + return os; +} + namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -32,6 +62,224 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } + +std::tuple, std::vector, + std::vector> +compute_submap_indices(const dolfinx::common::IndexMap& imap, + std::span indices) +{ + // --- Step 1 ---: Send ghost indices in `indices` to their owners + // and receive indices owned by this process that are in `indices` + // on other processes + + const MPI_Comm comm = imap.comm(); + // TODO Should these be spans? + const std::vector& src = imap.src(); + const std::vector& dest = imap.dest(); + const std::vector& ghosts = imap.ghosts(); + const std::vector& ghost_owners = imap.owners(); + // ss << "src = " << src << "\n"; + // ss << "dest = " << dest << "\n"; + + // Create neighbourhood comm (ghost -> owner) + MPI_Comm comm0; + int ierr = MPI_Dist_graph_create_adjacent( + comm, dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), src.data(), + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); + dolfinx::MPI::check_error(comm, ierr); + + // Create lookup array to determine if an index is in the sub-map + std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); + for (auto v : indices) + is_in_submap[v] = 1; + // ss << "is_in_submap = " << is_in_submap << "\n"; + + // Pack ghosts indices that this process wants to include in the + // sub-map + std::vector> send_data(src.size()); + for (std::size_t i = 0; i < ghosts.size(); ++i) + { + auto it = std::lower_bound(src.begin(), src.end(), ghost_owners[i]); + assert(it != src.end() and *it == ghost_owners[i]); + int r = std::distance(src.begin(), it); + // Send ghost index if it is in the submap, else send -1 + if (is_in_submap[imap.size_local() + i]) + send_data[r].push_back(ghosts[i]); + else + send_data[r].push_back(-1); + } + // ss << "send_data = " << send_data << "\n"; + + // Count number of indices to send per dest + std::vector send_sizes; + std::transform(send_data.begin(), send_data.end(), + std::back_inserter(send_sizes), + [](auto& d) { return d.size(); }); + + // ss << "send_sizes = " << send_sizes << "\n"; + + // Send how many indices I ghost to each owner, and receive how many + // of my indices other ranks ghost + std::vector recv_sizes; + recv_sizes.resize(dest.size(), 0); + send_sizes.reserve(1); + recv_sizes.reserve(1); + ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, + recv_sizes.data(), 1, MPI_INT32_T, comm0); + dolfinx::MPI::check_error(comm, ierr); + + // ss << "recv_sizes = " << recv_sizes << "\n"; + + // Prepare displacement vectors + std::vector send_disp, recv_disp; + send_disp.resize(src.size() + 1, 0); + recv_disp.resize(dest.size() + 1, 0); + std::partial_sum(send_sizes.begin(), send_sizes.end(), + std::next(send_disp.begin())); + std::partial_sum(recv_sizes.begin(), recv_sizes.end(), + std::next(recv_disp.begin())); + + // ss << "send_disp = " << send_disp << "\n"; + // ss << "recv_disp = " << recv_disp << "\n"; + + // Send ghost indices to owner, and receive indices owned by this process + // that are ghosts on others + std::vector send_indices, recv_indices; + for (auto& d : send_data) + send_indices.insert(send_indices.end(), d.begin(), d.end()); + + // ss << "send_indices = " << send_indices << "\n"; + + recv_indices.resize(recv_disp.back()); + ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), + send_disp.data(), MPI_INT64_T, + recv_indices.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT64_T, comm0); + dolfinx::MPI::check_error(comm, ierr); + + // ss << "recv_indices = " << recv_indices << "\n"; + + // Free the communicator + ierr = MPI_Comm_free(&comm0); + dolfinx::MPI::check_error(comm, ierr); + + // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. + // indices owned by this process that are in `indices` on other processes) to + // their owner in the submap. This is required since not all indices in + // `recv_indices` will be in `indices` on this process, and thus other + // processes must own them in the submap. + + // Create a map from (global) indices owned by this process that are ghosted + // on others to processes that can own them. + // FIXME Avoid map + const int rank = dolfinx::MPI::rank(comm); + // A map from (global) indices in `recv_indices` to a list of processes that + // can own the index in the submap. + std::map> + global_idx_to_possible_owner; + const std::array local_range = imap.local_range(); + for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) + { + for (int j = recv_disp[i]; j < recv_disp[i + 1]; ++j) + { + std::int64_t idx = recv_indices[j]; + // Check that the index is in the submap + if (idx != -1) + { + // Compute the local index + std::int32_t idx_local = idx - local_range[0]; + assert(idx_local >= 0); + assert(idx_local < local_range[1]); + + // Check if index is in the submap on this process. If so, this process + // remains its owner in the submap. Otherwise, add the process that sent + // it to the list of possible owners. + if (is_in_submap[idx_local]) + global_idx_to_possible_owner[idx].push_back(rank); + else + global_idx_to_possible_owner[idx].push_back(dest[i]); + } + } + } + + // ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner << + // "\n"; + + // Choose the submap owner for each index in `recv_indices` + std::vector send_owners; + send_owners.reserve(recv_indices.size()); + for (auto idx : recv_indices) + { + // Check the index is in the submap, otherwise send -1 + if (idx != -1) + { + // Choose new owner randomly for load balancing + const std::vector& possible_owners + = global_idx_to_possible_owner[idx]; + const int random_index = std::rand() % possible_owners.size(); + send_owners.push_back(possible_owners[random_index]); + } + else + send_owners.push_back(-1); + } + + // ss << "send_owners = " << send_owners << "\n"; + + // Create neighbourhood comm (owner -> ghost) + MPI_Comm comm1; + ierr = MPI_Dist_graph_create_adjacent( + comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); + dolfinx::MPI::check_error(comm, ierr); + + // Send the data + std::vector recv_owners(send_disp.back()); + ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT32_T, + recv_owners.data(), send_sizes.data(), + send_disp.data(), MPI_INT32_T, comm1); + dolfinx::MPI::check_error(comm, ierr); + + // Free the communicator + ierr = MPI_Comm_free(&comm1); + dolfinx::MPI::check_error(comm, ierr); + + // ss << "recv_owners = " << recv_owners << "\n"; + + // Local indices (w.r.t. original map) owned by this process in the submap + std::vector submap_owned; + // Local indices (w.r.t. original map) ghosted by this process in the submap + std::vector submap_ghost; + // The owners of the submap ghost indices (process submap_ghost_owners[i] owns + // index submap_ghost[i]) + std::vector submap_ghost_owners; + for (std::int32_t v : indices) + if (v < imap.size_local()) + submap_owned.push_back(v); + + for (int i = 0; i < send_indices.size(); ++i) + { + std::int32_t local_idx = imap.size_local() + i; + std::int64_t global_idx = send_indices[i]; + std::int32_t owner = recv_owners[i]; + + // Check if index is in the submap + if (global_idx >= 0) + { + if (owner == rank) + { + submap_owned.push_back(local_idx); + } + else + { + submap_ghost.push_back(local_idx); + submap_ghost_owners.push_back(owner); + } + } + } + + return {submap_owned, submap_ghost, submap_ghost_owners}; +} } // namespace //----------------------------------------------------------------------------- @@ -631,6 +879,32 @@ IndexMap::create_submap(std::span indices) const } } //----------------------------------------------------------------------------- +// std::tuple> +void +IndexMap::create_submap_conn(std::span indices) const +{ + const int rank = dolfinx::MPI::rank(_comm.comm()); + const int comm_size = dolfinx::MPI::size(_comm.comm()); + ss << "Rank " << rank << ":\n"; + + // TODO Maybe submap_ghost (at least) should return global indices? + auto [submap_owned, submap_ghost, submap_ghost_owners] + = compute_submap_indices(*this, indices); + + ss << "submap_owned = " << submap_owned << "\n"; + ss << "submap_ghost = " << submap_ghost << "\n"; + ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; + + for (int i = 0; i < comm_size; ++i) + { + if (i == rank) + { + std::cout << ss.str() << "\n"; + } + MPI_Barrier(_comm.comm()); + } +} +//----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const { const std::int64_t offset = _local_range[0]; diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 9e4a0ee131a..596135e9152 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -189,6 +189,10 @@ class IndexMap std::pair> create_submap(std::span indices) const; + // std::tuple> + void + create_submap_conn(std::span indices) const; + /// @todo Aim to remove this function? /// /// @brief Compute map from each local (owned) index to the set of diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 5b940b78609..29fab610b36 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -143,7 +143,17 @@ void common(py::module& m) std::span(entities.data(), entities.size())); return std::pair(std::move(map), as_pyarray(std::move(ghosts))); }, - py::arg("entities")); + py::arg("entities")) + .def("create_submap_conn", + [](const dolfinx::common::IndexMap& self, + const py::array_t& indices) + { + // auto [map, submap_to_map] = + self.create_submap_conn( + std::span(indices.data(), indices.size())); + // return std::pair(std::move(map), as_pyarray(std::move(submap_to_map))); + }, + py::arg("indices")); // dolfinx::common::Timer py::class_>( From 07fcad637f34988858a87edbf0cf93f1e9a052f9 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 17:46:50 +0000 Subject: [PATCH 002/101] Compute submap offset --- cpp/dolfinx/common/IndexMap.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index dd811f5e85a..564370f9655 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -23,8 +23,7 @@ std::stringstream ss; // A function for printing vectors template -std::ostream &operator<<(std::ostream &os, - const std::vector &vector) +std::ostream& operator<<(std::ostream& os, const std::vector& vector) { os << "{ "; for (auto v : vector) @@ -37,11 +36,10 @@ std::ostream &operator<<(std::ostream &os, // A function for printing maps template -std::ostream &operator<<(std::ostream &os, - const std::map &map) +std::ostream& operator<<(std::ostream& os, const std::map& map) { os << "{ "; - for (const auto &[k, v] : map) + for (const auto& [k, v] : map) os << k << ": " << v << " "; os << "}"; return os; @@ -880,8 +878,7 @@ IndexMap::create_submap(std::span indices) const } //----------------------------------------------------------------------------- // std::tuple> -void -IndexMap::create_submap_conn(std::span indices) const +void IndexMap::create_submap_conn(std::span indices) const { const int rank = dolfinx::MPI::rank(_comm.comm()); const int comm_size = dolfinx::MPI::size(_comm.comm()); @@ -895,6 +892,16 @@ IndexMap::create_submap_conn(std::span indices) const ss << "submap_ghost = " << submap_ghost << "\n"; ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; + // Compute submap offset for this rank + std::int64_t submap_local_size = submap_owned.size(); + ss << "submap_local_size = " << submap_local_size << "\n"; + std::int64_t submap_offset = 0; + int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, + MPI_SUM, _comm.comm()); + dolfinx::MPI::check_error(_comm.comm(), ierr); + + ss << "submap_offset = " << submap_offset << "\n"; + for (int i = 0; i < comm_size; ++i) { if (i == rank) From 1e2e82738699d0eb82fae55df25bad08b9bff446 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 17:48:57 +0000 Subject: [PATCH 003/101] Compute submap sources --- cpp/dolfinx/common/IndexMap.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 564370f9655..25faa7d7269 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -902,6 +902,16 @@ void IndexMap::create_submap_conn(std::span indices) const ss << "submap_offset = " << submap_offset << "\n"; + // Get submap source ranks + std::vector submap_src(submap_ghost_owners.begin(), + submap_ghost_owners.end()); + std::sort(submap_src.begin(), submap_src.end()); + submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), + submap_src.end()); + submap_src.shrink_to_fit(); + + ss << "submap_src = " << submap_src << "\n"; + for (int i = 0; i < comm_size; ++i) { if (i == rank) From 8ab92f2fb053775439801d40eb1d22913c7bcfa9 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:04:54 +0000 Subject: [PATCH 004/101] Fix type --- cpp/dolfinx/common/IndexMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 25faa7d7269..a16077223d3 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -255,7 +255,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, if (v < imap.size_local()) submap_owned.push_back(v); - for (int i = 0; i < send_indices.size(); ++i) + for (std::size_t i = 0; i < send_indices.size(); ++i) { std::int32_t local_idx = imap.size_local() + i; std::int64_t global_idx = send_indices[i]; From 5343edba5a53b3ff0cd161ed221d68af9984346c Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:10:45 +0000 Subject: [PATCH 005/101] Compute submap dest ranks --- cpp/dolfinx/common/IndexMap.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index a16077223d3..6c5a0c72f13 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -912,6 +912,14 @@ void IndexMap::create_submap_conn(std::span indices) const ss << "submap_src = " << submap_src << "\n"; + // Compute submap destination ranks + // FIXME Can NBX call be avoided by using O^r_p and O^g_p? + std::vector submap_dest + = dolfinx::MPI::compute_graph_edges_nbx(_comm.comm(), submap_src); + std::sort(submap_dest.begin(), submap_dest.end()); + + ss << "submap_dest = " << submap_dest << "\n"; + for (int i = 0; i < comm_size; ++i) { if (i == rank) From 6cda53fef5be60e1f0f27aa6d369c2131c3cf786 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:12:01 +0000 Subject: [PATCH 006/101] Document --- cpp/dolfinx/common/IndexMap.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 596135e9152..797a163c83e 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -189,6 +189,8 @@ class IndexMap std::pair> create_submap(std::span indices) const; + /// @brief TODO + /// @param[in] indices // std::tuple> void create_submap_conn(std::span indices) const; From 21f5da93ba3c42dfe366c5f8673cd47cde71047e Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:23:27 +0000 Subject: [PATCH 007/101] Compute submap ghost indices --- cpp/dolfinx/common/IndexMap.cpp | 148 ++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 6c5a0c72f13..44ca1f71f3e 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -278,6 +278,143 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, return {submap_owned, submap_ghost, submap_ghost_owners}; } +std::vector compute_submap_ghost_indices( + const MPI_Comm& comm, const std::vector& submap_src, + const std::vector& submap_dest, + const std::vector& submap_owned, + const std::vector& submap_ghosts_global, + const std::vector& submap_ghost_owners, + const int submap_offset, + const dolfinx::common::IndexMap& imap) // NOTE: This is the original imap! +{ + // --- Step 1 ---: Send submap global ghost indices (indexed w.r.t. original + // imap) to owning rank + + std::vector recv_indices; + std::vector send_disp, recv_disp; + std::vector send_sizes, recv_sizes; + { + // Create neighbourhood comm (ghost -> owner) + MPI_Comm comm0; + int ierr = MPI_Dist_graph_create_adjacent( + comm, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, + submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, + false, &comm0); + dolfinx::MPI::check_error(comm, ierr); + + // Pack ghosts indices + std::vector> send_data(submap_src.size()); + std::vector> pos_to_ghost(submap_src.size()); + for (std::size_t i = 0; i < submap_ghosts_global.size(); ++i) + { + auto it = std::lower_bound(submap_src.begin(), submap_src.end(), + submap_ghost_owners[i]); + assert(it != submap_src.end() and *it == submap_ghost_owners[i]); + int r = std::distance(submap_src.begin(), it); + send_data[r].push_back(submap_ghosts_global[i]); + pos_to_ghost[r].push_back(i); + } + + // Count number of ghosts per dest + std::transform(send_data.begin(), send_data.end(), + std::back_inserter(send_sizes), + [](auto& d) { return d.size(); }); + + // Build send buffer and ghost position to send buffer position + std::vector send_indices; + for (auto& d : send_data) + send_indices.insert(send_indices.end(), d.begin(), d.end()); + + // Send how many indices I ghost to each owner, and receive how many + // of my indices other ranks ghost + recv_sizes.resize(submap_dest.size(), 0); + send_sizes.reserve(1); + recv_sizes.reserve(1); + ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, + recv_sizes.data(), 1, MPI_INT32_T, comm0); + dolfinx::MPI::check_error(comm, ierr); + + // Prepare displacement vectors + send_disp.resize(submap_src.size() + 1, 0); + recv_disp.resize(submap_dest.size() + 1, 0); + std::partial_sum(send_sizes.begin(), send_sizes.end(), + std::next(send_disp.begin())); + std::partial_sum(recv_sizes.begin(), recv_sizes.end(), + std::next(recv_disp.begin())); + + // Send ghost indices to owner, and receive indices + recv_indices.resize(recv_disp.back()); + ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), + send_disp.data(), MPI_INT64_T, + recv_indices.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT64_T, comm0); + dolfinx::MPI::check_error(comm, ierr); + + ierr = MPI_Comm_free(&comm0); + dolfinx::MPI::check_error(comm, ierr); + } + + // --- Step 2 ---: For each received index, compute the submap global index + + std::vector send_gidx; + send_gidx.reserve(recv_indices.size()); + const std::array local_range = imap.local_range(); + for (auto idx : recv_indices) + { + std::int32_t idx_local = idx - local_range[0]; + assert(idx_local >= 0); + assert(idx_local < local_range[1]); + + // Could avoid search by creating look-up array + auto it + = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx_local); + assert(it != submap_owned.end() and *it == idx_local); + std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); + send_gidx.push_back(idx_local_submap + submap_offset); + } + + // ss << "send_gidx = " << send_gidx << "\n"; + + // --- Step 3 ---: Send submap global indices to process that ghost them + + // Create neighbourhood comm (owner -> ghost) + MPI_Comm comm1; + int ierr = MPI_Dist_graph_create_adjacent( + comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, + submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, + false, &comm1); + dolfinx::MPI::check_error(comm, ierr); + + // Send indices to ghosting ranks + std::vector recv_gidx(send_disp.back()); + ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT64_T, recv_gidx.data(), + send_sizes.data(), send_disp.data(), + MPI_INT64_T, comm1); + dolfinx::MPI::check_error(comm, ierr); + + ierr = MPI_Comm_free(&comm1); + dolfinx::MPI::check_error(comm, ierr); + + // ss << "recv_gidx = " << recv_gidx << "\n"; + + // // --- Step 4---: Unpack received data + + std::vector ghost_submap_gidx; + for (std::size_t i = 0; i < send_disp.size() - 1; ++i) + { + for (int j = send_disp[i]; j < send_disp[i + 1]; ++j) + { + std::int64_t idx = recv_gidx[j]; + assert(idx >= 0); + ghost_submap_gidx.push_back(idx); + } + } + + // ss << "ghost_submap_gidx = " << ghost_submap_gidx << "\n"; + + return ghost_submap_gidx; +} } // namespace //----------------------------------------------------------------------------- @@ -920,6 +1057,17 @@ void IndexMap::create_submap_conn(std::span indices) const ss << "submap_dest = " << submap_dest << "\n"; + // Compute the global indices (w.r.t. the submap) of the submap ghosts + std::vector submap_ghost_global(submap_ghost.size()); + this->local_to_global(submap_ghost, submap_ghost_global); + // FIXME Maybe return submap_owned to as global to simplify + // compute_submap_ghost_indices? + auto submap_ghost_gidxs = compute_submap_ghost_indices( + _comm.comm(), submap_src, submap_dest, submap_owned, submap_ghost_global, + submap_ghost_owners, submap_offset, *this); + + ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; + for (int i = 0; i < comm_size; ++i) { if (i == rank) From 027be30c818e0e188f0d5d6c31c60502ca09f0dd Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:28:37 +0000 Subject: [PATCH 008/101] Simplify --- cpp/dolfinx/common/IndexMap.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 44ca1f71f3e..e9d62502416 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -304,7 +304,6 @@ std::vector compute_submap_ghost_indices( // Pack ghosts indices std::vector> send_data(submap_src.size()); - std::vector> pos_to_ghost(submap_src.size()); for (std::size_t i = 0; i < submap_ghosts_global.size(); ++i) { auto it = std::lower_bound(submap_src.begin(), submap_src.end(), @@ -312,7 +311,6 @@ std::vector compute_submap_ghost_indices( assert(it != submap_src.end() and *it == submap_ghost_owners[i]); int r = std::distance(submap_src.begin(), it); send_data[r].push_back(submap_ghosts_global[i]); - pos_to_ghost[r].push_back(i); } // Count number of ghosts per dest From 8965bc2ca5c9838a59dd6f22a2989cd2eab02098 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:32:43 +0000 Subject: [PATCH 009/101] Compute sub imap to imap --- cpp/dolfinx/common/IndexMap.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index e9d62502416..671ccd96d45 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -1066,6 +1066,21 @@ void IndexMap::create_submap_conn(std::span indices) const ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; + // Create sub-index map + dolfinx::common::IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, + submap_ghost_owners); + + // Create a map from (local) indices in the submap to the corresponding + // (local) index in the original map + std::vector sub_imap_to_imap; + sub_imap_to_imap.reserve(submap_owned.size() + submap_ghost.size()); + sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_owned.begin(), + submap_owned.end()); + sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), + submap_ghost.end()); + + ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; + for (int i = 0; i < comm_size; ++i) { if (i == rank) From 9bf0137d9cac0c0ba17e4e433edcf265984f9643 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 1 Nov 2023 18:48:31 +0000 Subject: [PATCH 010/101] Return imap and map --- cpp/dolfinx/common/IndexMap.cpp | 11 +++++------ cpp/dolfinx/common/IndexMap.h | 3 +-- python/dolfinx/wrappers/common.cpp | 9 +++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 671ccd96d45..dc7de44ff12 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -1012,8 +1012,8 @@ IndexMap::create_submap(std::span indices) const } } //----------------------------------------------------------------------------- -// std::tuple> -void IndexMap::create_submap_conn(std::span indices) const +std::pair> +IndexMap::create_submap_conn(std::span indices) const { const int rank = dolfinx::MPI::rank(_comm.comm()); const int comm_size = dolfinx::MPI::size(_comm.comm()); @@ -1066,10 +1066,6 @@ void IndexMap::create_submap_conn(std::span indices) const ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; - // Create sub-index map - dolfinx::common::IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, - submap_ghost_owners); - // Create a map from (local) indices in the submap to the corresponding // (local) index in the original map std::vector sub_imap_to_imap; @@ -1089,6 +1085,9 @@ void IndexMap::create_submap_conn(std::span indices) const } MPI_Barrier(_comm.comm()); } + + return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), + std::move(sub_imap_to_imap)}; } //----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 797a163c83e..94ef56a65de 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -191,8 +191,7 @@ class IndexMap /// @brief TODO /// @param[in] indices - // std::tuple> - void + std::pair> create_submap_conn(std::span indices) const; /// @todo Aim to remove this function? diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 29fab610b36..eaedf571001 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -144,14 +144,15 @@ void common(py::module& m) return std::pair(std::move(map), as_pyarray(std::move(ghosts))); }, py::arg("entities")) - .def("create_submap_conn", + .def( + "create_submap_conn", [](const dolfinx::common::IndexMap& self, const py::array_t& indices) { - // auto [map, submap_to_map] = - self.create_submap_conn( + auto [map, submap_to_map] = self.create_submap_conn( std::span(indices.data(), indices.size())); - // return std::pair(std::move(map), as_pyarray(std::move(submap_to_map))); + return std::pair(std::move(map), + as_pyarray(std::move(submap_to_map))); }, py::arg("indices")); From cbf36d054c161e7226269a95d1cfb02637bfd108 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Mon, 6 Nov 2023 11:00:59 +0000 Subject: [PATCH 011/101] Fix typos --- python/dolfinx/wrappers/common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index d185950366f..6faaeba9be4 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -145,7 +145,7 @@ void common(nb::module_& m) return std::pair(std::move(map), dolfinx_wrappers::as_nbarray(std::move(ghosts))); }, - np::arg("entities")) + nb::arg("entities")) .def( "create_submap_conn", [](const dolfinx::common::IndexMap& self, @@ -156,7 +156,7 @@ void common(nb::module_& m) return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( std::move(submap_to_map))); }, - py::arg("indices")); + nb::arg("indices")); // dolfinx::common::Timer nb::class_(m, "Timer", "Timer class") From 2654251bf2b974d79d0896c9f8cc84d99adef841 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Mon, 6 Nov 2023 12:15:08 +0000 Subject: [PATCH 012/101] Add test --- python/test/unit/common/test_index_map.py | 73 +++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 51fea1bb5e5..2c1d4a53ad0 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -107,3 +107,76 @@ def test_index_map_ghost_lifetime(): assert np.array_equal(ghosts, map_ghosts) del map assert np.array_equal(ghosts, map_ghosts) + + +# Test create_submap_conn. The diagram illustrates the case with four +# processes. Original map numbering and connectivity (G indicates a ghost +# index): +# Global Rank 0 Rank 1 Rank 2 Rank 3 +# 1 - 0 1 - 0 +# | / | | / | +# 3 - 2 3G - 2 0 - 2G +# | / | | / | +# 5 - 4 3G - 1 0 - 2G +# | / | | / | +# 7 - 6 3G - 1 0 - 3G +# | / | | / | +# 9 - 8 2 - 1 +# We now create a submap of the "upper triangular" parts (i.e. to +# get the following): +# Global Rank 0 Rank 1 Rank 2 Rank 3 +# 1 - 0 1 - 0 +# | / | / +# 2 - 3 2G 0 - 1 +# | / | / +# 4 - 5 2G 0 - 1 +# | / | / +# 6 - 8 2G 0 - 2 +# | / | / +# 7 1 +def test_create_submap_connected(): + comm = MPI.COMM_WORLD + + if comm.size == 1: + return + + if comm.rank == 0: + local_size = 3 + ghosts = np.array([local_size], dtype=np.int64) + owners = np.array([1], dtype=np.int32) + submap_indices = np.array([0, 1, 3], dtype=np.int32) + elif comm.rank == comm.size - 1: + local_size = 3 + ghosts = np.array([2 * comm.rank], dtype=np.int64) + owners = np.array([comm.rank - 1], dtype=np.int32) + submap_indices = np.array([0, 3, 2], dtype=np.int32) + else: + local_size = 2 + ghosts = np.array([2 * comm.rank, 2 * comm.rank + 3], dtype=np.int64) + owners = np.array([comm.rank - 1, comm.rank + 1], dtype=np.int32) + submap_indices = np.array([0, 2, 3], dtype=np.int32) + + imap = dolfinx.common.IndexMap(comm, local_size, ghosts, owners) + sub_imap, sub_imap_to_imap = imap.create_submap_conn(submap_indices) + + if comm.rank == 0: + assert sub_imap.size_local == 2 + assert np.array_equal(sub_imap.ghosts, [2]) + assert np.array_equal(sub_imap.owners, [comm.rank + 1]) + assert np.array_equal(sub_imap_to_imap, [0, 1, 3]) + elif comm.rank == comm.size - 1: + assert sub_imap.size_local == 3 + assert np.array_equal(sub_imap.ghosts, []) + assert np.array_equal(sub_imap.owners, []) + assert np.array_equal(sub_imap_to_imap, [0, 2, 3]) + else: + assert sub_imap.size_local == 2 + assert np.array_equal(sub_imap.ghosts, [2 * (comm.rank + 1)]) + assert np.array_equal(sub_imap.owners, [comm.rank + 1]) + assert np.array_equal(sub_imap_to_imap, [0, 2, 3]) + + global_indices = sub_imap.local_to_global( + np.arange(sub_imap.size_local + sub_imap.num_ghosts, + dtype=np.int32)) + assert np.array_equal( + global_indices, np.arange(comm.rank * 2, comm.rank * 2 + 3)) From 53648ba0c7afc292fbbbff9cbd82fd92e0a7b942 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 11:43:13 +0000 Subject: [PATCH 013/101] Use new create submap in topology creation --- cpp/dolfinx/mesh/Topology.cpp | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index f6283f08ffc..326c658fdc8 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1195,23 +1195,33 @@ mesh::create_subtopology(const Topology& topology, int dim, // Get the vertices in the sub-topology owned by this process auto map0 = topology.index_map(0); assert(map0); - std::vector subvertices0 = common::compute_owned_indices( - compute_incident_entities(topology, subentities, dim, 0), *map0); + // std::vector subvertices0 = common::compute_owned_indices( + // compute_incident_entities(topology, subentities, dim, 0), *map0); + + // // Create map from the vertices in the sub-topology to the vertices in the + // // parent topology, and an index map + // std::shared_ptr submap0; + // { + // std::pair> map_data + // = map0->create_submap(subvertices0); + // submap0 = std::make_shared(std::move(map_data.first)); + + // // Add ghost vertices to the map + // subvertices0.reserve(submap0->size_local() + submap0->num_ghosts()); + // std::transform(map_data.second.begin(), map_data.second.end(), + // std::back_inserter(subvertices0), + // [offset = map0->size_local()](std::int32_t vertex_index) + // { return offset + vertex_index; }); + // } - // Create map from the vertices in the sub-topology to the vertices in the - // parent topology, and an index map std::shared_ptr submap0; + std::vector subvertices0; { std::pair> map_data - = map0->create_submap(subvertices0); + = map0->create_submap_conn( + compute_incident_entities(topology, subentities, dim, 0)); submap0 = std::make_shared(std::move(map_data.first)); - - // Add ghost vertices to the map - subvertices0.reserve(submap0->size_local() + submap0->num_ghosts()); - std::transform(map_data.second.begin(), map_data.second.end(), - std::back_inserter(subvertices0), - [offset = map0->size_local()](std::int32_t vertex_index) - { return offset + vertex_index; }); + subvertices0 = std::move(map_data.second); } // Sub-topology vertex-to-vertex connectivity (identity) From b6723c91699d2b73b3cce09beeec3c70c6ec4fd2 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 12:04:26 +0000 Subject: [PATCH 014/101] Use new create submap in geometry creation --- cpp/dolfinx/mesh/Geometry.h | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index b2c55815cae..9679b1201f3 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -328,24 +328,34 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, // Get the sub-geometry dofs owned by this process auto x_index_map = geometry.index_map(); assert(x_index_map); - auto subx_to_x_dofmap - = common::compute_owned_indices(sub_x_dofs, *x_index_map); + // auto subx_to_x_dofmap + // = common::compute_owned_indices(sub_x_dofs, *x_index_map); + // std::shared_ptr sub_x_dof_index_map; + // { + // std::pair> map_data + // = x_index_map->create_submap(subx_to_x_dofmap); + // sub_x_dof_index_map + // = std::make_shared(std::move(map_data.first)); + + // // Create a map from the dofs in the sub-geometry to the geometry + // subx_to_x_dofmap.reserve(sub_x_dof_index_map->size_local() + // + sub_x_dof_index_map->num_ghosts()); + // std::transform(map_data.second.begin(), map_data.second.end(), + // std::back_inserter(subx_to_x_dofmap), + // [offset = x_index_map->size_local()](auto x_dof_index) + // { return offset + x_dof_index; }); + // } + std::shared_ptr sub_x_dof_index_map; + std::vector subx_to_x_dofmap; { std::pair> map_data - = x_index_map->create_submap(subx_to_x_dofmap); - sub_x_dof_index_map - = std::make_shared(std::move(map_data.first)); - - // Create a map from the dofs in the sub-geometry to the geometry - subx_to_x_dofmap.reserve(sub_x_dof_index_map->size_local() - + sub_x_dof_index_map->num_ghosts()); - std::transform(map_data.second.begin(), map_data.second.end(), - std::back_inserter(subx_to_x_dofmap), - [offset = x_index_map->size_local()](auto x_dof_index) - { return offset + x_dof_index; }); + = x_index_map->create_submap_conn(sub_x_dofs); + sub_x_dof_index_map = std::make_shared(std::move(map_data.first)); + subx_to_x_dofmap = std::move(map_data.second); } + // Create sub-geometry coordinates std::span x = geometry.x(); std::int32_t sub_num_x_dofs = subx_to_x_dofmap.size(); From 6fcdaba5f75b6718b1dde5672a41a2cd8c23b6fa Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 12:04:58 +0000 Subject: [PATCH 015/101] format --- cpp/dolfinx/mesh/Geometry.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index 9679b1201f3..2ad91682874 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -350,12 +350,12 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, std::vector subx_to_x_dofmap; { std::pair> map_data - = x_index_map->create_submap_conn(sub_x_dofs); - sub_x_dof_index_map = std::make_shared(std::move(map_data.first)); + = x_index_map->create_submap_conn(sub_x_dofs); + sub_x_dof_index_map + = std::make_shared(std::move(map_data.first)); subx_to_x_dofmap = std::move(map_data.second); } - // Create sub-geometry coordinates std::span x = geometry.x(); std::int32_t sub_num_x_dofs = subx_to_x_dofmap.size(); From b14939bad48caf070f4c786689ae6fb498a7fa5e Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 15:44:14 +0000 Subject: [PATCH 016/101] Start on posisble fix --- cpp/dolfinx/common/IndexMap.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 7dbf136c984..5ce98e6fc93 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -276,6 +276,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } + // TODO Move return {submap_owned, submap_ghost, submap_ghost_owners}; } std::vector compute_submap_ghost_indices( @@ -371,6 +372,22 @@ std::vector compute_submap_ghost_indices( send_gidx.push_back(idx_local_submap + submap_offset); } + // TODO POSSIBLE FIX + // // TODO Convert recv_indices or submap_owned? + // std::vector recv_indices_local(recv_indices.size()); + // imap.global_to_local(recv_indices, recv_indices_local); + // ss << "recv_indices_local = " << recv_indices_local << "\n"; + // for (auto idx : recv_indices_local) + // { + // // TODO Check submap owned is always sorted + // // Could avoid search by creating look-up array + // auto it + // = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); + // assert(it != submap_owned.end() and *it == idx); + // std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); + // send_gidx.push_back(idx_local_submap + submap_offset); + // } + // ss << "send_gidx = " << send_gidx << "\n"; // --- Step 3 ---: Send submap global indices to process that ghost them @@ -1055,9 +1072,17 @@ IndexMap::create_submap_conn(std::span indices) const ss << "submap_dest = " << submap_dest << "\n"; + // std::vector submap_owned_global(submap_owned.size()); + // this->local_to_global(submap_owned, submap_owned_global); + + // ss << "submap_owned_global = " << submap_owned_global << "\n"; + // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); this->local_to_global(submap_ghost, submap_ghost_global); + + // ss << "submap_ghost_global = " << submap_ghost_global << "\n"; + // FIXME Maybe return submap_owned to as global to simplify // compute_submap_ghost_indices? auto submap_ghost_gidxs = compute_submap_ghost_indices( @@ -1086,7 +1111,8 @@ IndexMap::create_submap_conn(std::span indices) const MPI_Barrier(_comm.comm()); } - return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), + return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, + submap_ghost_owners), std::move(sub_imap_to_imap)}; } //----------------------------------------------------------------------------- @@ -1378,7 +1404,7 @@ std::vector IndexMap::shared_indices() const assert(idx < range[1]); return idx - range[0]; }); - + // Sort and remove duplicates std::sort(shared.begin(), shared.end()); shared.erase(std::unique(shared.begin(), shared.end()), shared.end()); From 490efe1d89043488c09f81d33995194a61c25fc4 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 17:13:29 +0000 Subject: [PATCH 017/101] Fix computation of global index when vertices change owner --- cpp/dolfinx/common/IndexMap.cpp | 37 +++++++++++---------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 5ce98e6fc93..d28eca6d499 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -357,37 +357,23 @@ std::vector compute_submap_ghost_indices( std::vector send_gidx; send_gidx.reserve(recv_indices.size()); - const std::array local_range = imap.local_range(); - for (auto idx : recv_indices) + // NOTE: Received indices are owned by this process in the submap, but not + // necessarily in the original imap, so we must use global_to_local to convert + // rather than rather than subtracting the local_range[0] + // TODO Convert recv_indices or submap_owned? + std::vector recv_indices_local(recv_indices.size()); + imap.global_to_local(recv_indices, recv_indices_local); + ss << "recv_indices_local = " << recv_indices_local << "\n"; + for (auto idx : recv_indices_local) { - std::int32_t idx_local = idx - local_range[0]; - assert(idx_local >= 0); - assert(idx_local < local_range[1]); - + // TODO Check submap owned is always sorted // Could avoid search by creating look-up array - auto it - = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx_local); - assert(it != submap_owned.end() and *it == idx_local); + auto it = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); + assert(it != submap_owned.end() and *it == idx); std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); send_gidx.push_back(idx_local_submap + submap_offset); } - // TODO POSSIBLE FIX - // // TODO Convert recv_indices or submap_owned? - // std::vector recv_indices_local(recv_indices.size()); - // imap.global_to_local(recv_indices, recv_indices_local); - // ss << "recv_indices_local = " << recv_indices_local << "\n"; - // for (auto idx : recv_indices_local) - // { - // // TODO Check submap owned is always sorted - // // Could avoid search by creating look-up array - // auto it - // = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); - // assert(it != submap_owned.end() and *it == idx); - // std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); - // send_gidx.push_back(idx_local_submap + submap_offset); - // } - // ss << "send_gidx = " << send_gidx << "\n"; // --- Step 3 ---: Send submap global indices to process that ghost them @@ -1110,6 +1096,7 @@ IndexMap::create_submap_conn(std::span indices) const } MPI_Barrier(_comm.comm()); } + ss = std::stringstream(); return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), From ff8426a478a1f32f73337f14e29f6d0e3fd2f748 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 7 Nov 2023 17:33:03 +0000 Subject: [PATCH 018/101] Mention indices must be sorted --- cpp/dolfinx/common/IndexMap.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 94ef56a65de..1121d78441f 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -191,6 +191,7 @@ class IndexMap /// @brief TODO /// @param[in] indices + /// @pre `indices` must be sorted and contain no duplicates. std::pair> create_submap_conn(std::span indices) const; From e105934bbae3cebc651369ab39e4f05787a7f700 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 8 Nov 2023 15:05:34 +0000 Subject: [PATCH 019/101] Fix computaion of local indices --- cpp/dolfinx/common/IndexMap.cpp | 68 ++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index d28eca6d499..56c8e093e45 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -76,8 +76,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, const std::vector& dest = imap.dest(); const std::vector& ghosts = imap.ghosts(); const std::vector& ghost_owners = imap.owners(); - // ss << "src = " << src << "\n"; - // ss << "dest = " << dest << "\n"; + ss << "src = " << src << "\n"; + ss << "dest = " << dest << "\n"; // Create neighbourhood comm (ghost -> owner) MPI_Comm comm0; @@ -90,7 +90,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); for (auto v : indices) is_in_submap[v] = 1; - // ss << "is_in_submap = " << is_in_submap << "\n"; + ss << "is_in_submap = " << is_in_submap << "\n"; // Pack ghosts indices that this process wants to include in the // sub-map @@ -106,7 +106,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, else send_data[r].push_back(-1); } - // ss << "send_data = " << send_data << "\n"; + ss << "send_data = " << send_data << "\n"; // Count number of indices to send per dest std::vector send_sizes; @@ -114,7 +114,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::back_inserter(send_sizes), [](auto& d) { return d.size(); }); - // ss << "send_sizes = " << send_sizes << "\n"; + ss << "send_sizes = " << send_sizes << "\n"; // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost @@ -126,7 +126,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_sizes.data(), 1, MPI_INT32_T, comm0); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_sizes = " << recv_sizes << "\n"; + ss << "recv_sizes = " << recv_sizes << "\n"; // Prepare displacement vectors std::vector send_disp, recv_disp; @@ -137,8 +137,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); - // ss << "send_disp = " << send_disp << "\n"; - // ss << "recv_disp = " << recv_disp << "\n"; + ss << "send_disp = " << send_disp << "\n"; + ss << "recv_disp = " << recv_disp << "\n"; // Send ghost indices to owner, and receive indices owned by this process // that are ghosts on others @@ -146,7 +146,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, for (auto& d : send_data) send_indices.insert(send_indices.end(), d.begin(), d.end()); - // ss << "send_indices = " << send_indices << "\n"; + ss << "send_indices = " << send_indices << "\n"; recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), @@ -155,7 +155,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_indices = " << recv_indices << "\n"; + ss << "recv_indices = " << recv_indices << "\n"; // Free the communicator ierr = MPI_Comm_free(&comm0); @@ -200,8 +200,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } - // ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner << - // "\n"; + ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner + << "\n"; // Choose the submap owner for each index in `recv_indices` std::vector send_owners; @@ -221,7 +221,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_owners.push_back(-1); } - // ss << "send_owners = " << send_owners << "\n"; + ss << "send_owners = " << send_owners << "\n"; // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; @@ -242,7 +242,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_owners = " << recv_owners << "\n"; + ss << "recv_owners = " << recv_owners << "\n"; // Local indices (w.r.t. original map) owned by this process in the submap std::vector submap_owned; @@ -255,9 +255,13 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, if (v < imap.size_local()) submap_owned.push_back(v); + // FIXME Could just create when making send_indices + std::vector send_indices_local(send_indices.size()); + imap.global_to_local(send_indices, send_indices_local); + for (std::size_t i = 0; i < send_indices.size(); ++i) { - std::int32_t local_idx = imap.size_local() + i; + std::int32_t local_idx = send_indices_local[i]; std::int64_t global_idx = send_indices[i]; std::int32_t owner = recv_owners[i]; @@ -363,7 +367,7 @@ std::vector compute_submap_ghost_indices( // TODO Convert recv_indices or submap_owned? std::vector recv_indices_local(recv_indices.size()); imap.global_to_local(recv_indices, recv_indices_local); - ss << "recv_indices_local = " << recv_indices_local << "\n"; + // ss << "recv_indices_local = " << recv_indices_local << "\n"; for (auto idx : recv_indices_local) { // TODO Check submap owned is always sorted @@ -1022,6 +1026,17 @@ IndexMap::create_submap_conn(std::span indices) const const int comm_size = dolfinx::MPI::size(_comm.comm()); ss << "Rank " << rank << ":\n"; + // ss << "indices = "; + // for (auto idx : indices) + // { + // ss << idx << " "; + // } + // ss << "\n"; + + // std::vector indices_global(indices.size()); + // this->local_to_global(indices, indices_global); + // ss << "indices_global = " << indices_global << "\n"; + // TODO Maybe submap_ghost (at least) should return global indices? auto [submap_owned, submap_ghost, submap_ghost_owners] = compute_submap_indices(*this, indices); @@ -1030,15 +1045,24 @@ IndexMap::create_submap_conn(std::span indices) const ss << "submap_ghost = " << submap_ghost << "\n"; ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; + // std::vector submap_owned_global(submap_owned.size()); + // this->local_to_global(submap_owned, submap_owned_global); + // std::vector submap_ghost_global(submap_ghost.size()); + // this->local_to_global(submap_ghost, submap_ghost_global); + + // ss << "submap_owned_global = " << submap_owned_global << "\n"; + // ss << "submap_ghost_global = " << submap_ghost_global << "\n"; + // ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; + // Compute submap offset for this rank std::int64_t submap_local_size = submap_owned.size(); - ss << "submap_local_size = " << submap_local_size << "\n"; + // ss << "submap_local_size = " << submap_local_size << "\n"; std::int64_t submap_offset = 0; int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, MPI_SUM, _comm.comm()); dolfinx::MPI::check_error(_comm.comm(), ierr); - ss << "submap_offset = " << submap_offset << "\n"; + // ss << "submap_offset = " << submap_offset << "\n"; // Get submap source ranks std::vector submap_src(submap_ghost_owners.begin(), @@ -1048,7 +1072,7 @@ IndexMap::create_submap_conn(std::span indices) const submap_src.end()); submap_src.shrink_to_fit(); - ss << "submap_src = " << submap_src << "\n"; + // ss << "submap_src = " << submap_src << "\n"; // Compute submap destination ranks // FIXME Can NBX call be avoided by using O^r_p and O^g_p? @@ -1056,7 +1080,7 @@ IndexMap::create_submap_conn(std::span indices) const = dolfinx::MPI::compute_graph_edges_nbx(_comm.comm(), submap_src); std::sort(submap_dest.begin(), submap_dest.end()); - ss << "submap_dest = " << submap_dest << "\n"; + // ss << "submap_dest = " << submap_dest << "\n"; // std::vector submap_owned_global(submap_owned.size()); // this->local_to_global(submap_owned, submap_owned_global); @@ -1075,7 +1099,7 @@ IndexMap::create_submap_conn(std::span indices) const _comm.comm(), submap_src, submap_dest, submap_owned, submap_ghost_global, submap_ghost_owners, submap_offset, *this); - ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; + // ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; // Create a map from (local) indices in the submap to the corresponding // (local) index in the original map @@ -1086,7 +1110,7 @@ IndexMap::create_submap_conn(std::span indices) const sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), submap_ghost.end()); - ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; + // ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; for (int i = 0; i < comm_size; ++i) { From 070f995ebd95d2d012d6aea4cfad99715e2d9b7b Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 8 Nov 2023 16:35:58 +0000 Subject: [PATCH 020/101] Add temporary fix --- cpp/dolfinx/common/IndexMap.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 56c8e093e45..001e98eae0a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -214,8 +214,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Choose new owner randomly for load balancing const std::vector& possible_owners = global_idx_to_possible_owner[idx]; - const int random_index = std::rand() % possible_owners.size(); - send_owners.push_back(possible_owners[random_index]); + // const int random_index = std::rand() % possible_owners.size(); + // send_owners.push_back(possible_owners[random_index]); + + // FIXME TEMPORARILY CHOOSE THE FIRST PROCESS, MAKE RANDOM + send_owners.push_back(possible_owners[0]); } else send_owners.push_back(-1); @@ -367,6 +370,19 @@ std::vector compute_submap_ghost_indices( // TODO Convert recv_indices or submap_owned? std::vector recv_indices_local(recv_indices.size()); imap.global_to_local(recv_indices, recv_indices_local); + + // ss << "recv_indices_local = " << recv_indices_local << "\n"; + + // for (int i = 0; i < dolfinx::MPI::size(comm); ++i) + // { + // if (i == dolfinx::MPI::rank(comm)) + // { + // std::cout << ss.str() << "\n"; + // } + // MPI_Barrier(comm); + // } + // ss = std::stringstream(); + // ss << "recv_indices_local = " << recv_indices_local << "\n"; for (auto idx : recv_indices_local) { From f36d0d8b657527a7925db82cdb07c4bcf1683dd8 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 8 Nov 2023 17:04:10 +0000 Subject: [PATCH 021/101] Comment cout --- cpp/dolfinx/common/IndexMap.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 001e98eae0a..ed6de77f387 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -1039,7 +1039,7 @@ std::pair> IndexMap::create_submap_conn(std::span indices) const { const int rank = dolfinx::MPI::rank(_comm.comm()); - const int comm_size = dolfinx::MPI::size(_comm.comm()); + // const int comm_size = dolfinx::MPI::size(_comm.comm()); ss << "Rank " << rank << ":\n"; // ss << "indices = "; @@ -1128,15 +1128,15 @@ IndexMap::create_submap_conn(std::span indices) const // ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; - for (int i = 0; i < comm_size; ++i) - { - if (i == rank) - { - std::cout << ss.str() << "\n"; - } - MPI_Barrier(_comm.comm()); - } - ss = std::stringstream(); + // for (int i = 0; i < comm_size; ++i) + // { + // if (i == rank) + // { + // std::cout << ss.str() << "\n"; + // } + // MPI_Barrier(_comm.comm()); + // } + // ss = std::stringstream(); return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), From 5f0d043bb0a7167e9f56de6b319f01a445f390f0 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 8 Nov 2023 17:04:58 +0000 Subject: [PATCH 022/101] Add TODO --- python/test/unit/common/test_index_map.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 2c1d4a53ad0..7423087ea23 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -109,6 +109,8 @@ def test_index_map_ghost_lifetime(): assert np.array_equal(ghosts, map_ghosts) +# TODO: Add test for case where more than one two process shares an index +# whose owner changes in the submap # Test create_submap_conn. The diagram illustrates the case with four # processes. Original map numbering and connectivity (G indicates a ghost # index): From 4b54e1cbf1b79f3d0366c89d59b61676a9f42906 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 9 Nov 2023 12:11:30 +0000 Subject: [PATCH 023/101] Sort submap owned --- cpp/dolfinx/common/IndexMap.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index ed6de77f387..6644d5b4f88 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -282,6 +282,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } } + std::sort(submap_owned.begin(), submap_owned.end()); // TODO Move return {submap_owned, submap_ghost, submap_ghost_owners}; From 69bb7db4b1a21cc7781e2291741988250d8dff66 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 10 Nov 2023 18:45:27 +0000 Subject: [PATCH 024/101] Debugging --- cpp/dolfinx/common/IndexMap.cpp | 89 +++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 6644d5b4f88..05c05fdfb54 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -76,8 +76,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, const std::vector& dest = imap.dest(); const std::vector& ghosts = imap.ghosts(); const std::vector& ghost_owners = imap.owners(); - ss << "src = " << src << "\n"; - ss << "dest = " << dest << "\n"; + // ss << "src = " << src << "\n"; + // ss << "dest = " << dest << "\n"; // Create neighbourhood comm (ghost -> owner) MPI_Comm comm0; @@ -90,7 +90,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); for (auto v : indices) is_in_submap[v] = 1; - ss << "is_in_submap = " << is_in_submap << "\n"; + // ss << "is_in_submap = " << is_in_submap << "\n"; // Pack ghosts indices that this process wants to include in the // sub-map @@ -106,7 +106,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, else send_data[r].push_back(-1); } - ss << "send_data = " << send_data << "\n"; + // ss << "send_data = " << send_data << "\n"; // Count number of indices to send per dest std::vector send_sizes; @@ -114,7 +114,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::back_inserter(send_sizes), [](auto& d) { return d.size(); }); - ss << "send_sizes = " << send_sizes << "\n"; + // ss << "send_sizes = " << send_sizes << "\n"; // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost @@ -126,7 +126,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_sizes.data(), 1, MPI_INT32_T, comm0); dolfinx::MPI::check_error(comm, ierr); - ss << "recv_sizes = " << recv_sizes << "\n"; + // ss << "recv_sizes = " << recv_sizes << "\n"; // Prepare displacement vectors std::vector send_disp, recv_disp; @@ -137,8 +137,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); - ss << "send_disp = " << send_disp << "\n"; - ss << "recv_disp = " << recv_disp << "\n"; + // ss << "send_disp = " << send_disp << "\n"; + // ss << "recv_disp = " << recv_disp << "\n"; // Send ghost indices to owner, and receive indices owned by this process // that are ghosts on others @@ -146,7 +146,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, for (auto& d : send_data) send_indices.insert(send_indices.end(), d.begin(), d.end()); - ss << "send_indices = " << send_indices << "\n"; + // ss << "send_indices = " << send_indices << "\n"; recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), @@ -155,7 +155,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); - ss << "recv_indices = " << recv_indices << "\n"; + // ss << "recv_indices = " << recv_indices << "\n"; // Free the communicator ierr = MPI_Comm_free(&comm0); @@ -200,8 +200,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } - ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner - << "\n"; + // ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner + // << "\n"; // Choose the submap owner for each index in `recv_indices` std::vector send_owners; @@ -224,7 +224,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_owners.push_back(-1); } - ss << "send_owners = " << send_owners << "\n"; + // ss << "send_owners = " << send_owners << "\n"; // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; @@ -245,7 +245,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); - ss << "recv_owners = " << recv_owners << "\n"; + // ss << "recv_owners = " << recv_owners << "\n"; // Local indices (w.r.t. original map) owned by this process in the submap std::vector submap_owned; @@ -284,6 +284,19 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } std::sort(submap_owned.begin(), submap_owned.end()); + // std::vector perm(submap_ghost.size()); + // std::iota(perm.begin(), perm.end(), 0); + // std::sort(perm.begin(), perm.end(), [&](int i, int j){return submap_ghost[i] < submap_ghost[j];}); + + // std::vector submap_ghost_sorted(submap_ghost.size()); + // std::vector submap_ghost_owners_sorted(submap_ghost_owners.size()); + // for (std::size_t i = 0; i < submap_ghost.size(); ++i) + // { + // submap_ghost_sorted[i] = submap_ghost[perm[i]]; + // submap_ghost_owners_sorted[i] = submap_ghost_owners[perm[i]]; + // } + // return {submap_owned, submap_ghost_sorted, submap_ghost_owners_sorted}; + // TODO Move return {submap_owned, submap_ghost, submap_ghost_owners}; } @@ -390,6 +403,7 @@ std::vector compute_submap_ghost_indices( // TODO Check submap owned is always sorted // Could avoid search by creating look-up array auto it = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); + // auto it = std::find(submap_owned.begin(), submap_owned.end(), idx); assert(it != submap_owned.end() and *it == idx); std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); send_gidx.push_back(idx_local_submap + submap_offset); @@ -1043,16 +1057,18 @@ IndexMap::create_submap_conn(std::span indices) const // const int comm_size = dolfinx::MPI::size(_comm.comm()); ss << "Rank " << rank << ":\n"; - // ss << "indices = "; - // for (auto idx : indices) - // { - // ss << idx << " "; - // } - // ss << "\n"; + ss << "indices = "; + for (auto idx : indices) + { + ss << idx << " "; + } + ss << "\n"; + + ss << "this->size_local() = " << this->size_local() << "\n"; - // std::vector indices_global(indices.size()); - // this->local_to_global(indices, indices_global); - // ss << "indices_global = " << indices_global << "\n"; + std::vector indices_global(indices.size()); + this->local_to_global(indices, indices_global); + ss << "indices_global = " << indices_global << "\n"; // TODO Maybe submap_ghost (at least) should return global indices? auto [submap_owned, submap_ghost, submap_ghost_owners] @@ -1062,24 +1078,15 @@ IndexMap::create_submap_conn(std::span indices) const ss << "submap_ghost = " << submap_ghost << "\n"; ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; - // std::vector submap_owned_global(submap_owned.size()); - // this->local_to_global(submap_owned, submap_owned_global); - // std::vector submap_ghost_global(submap_ghost.size()); - // this->local_to_global(submap_ghost, submap_ghost_global); - - // ss << "submap_owned_global = " << submap_owned_global << "\n"; - // ss << "submap_ghost_global = " << submap_ghost_global << "\n"; - // ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; - // Compute submap offset for this rank std::int64_t submap_local_size = submap_owned.size(); - // ss << "submap_local_size = " << submap_local_size << "\n"; + ss << "submap_local_size = " << submap_local_size << "\n"; std::int64_t submap_offset = 0; int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, MPI_SUM, _comm.comm()); dolfinx::MPI::check_error(_comm.comm(), ierr); - // ss << "submap_offset = " << submap_offset << "\n"; + ss << "submap_offset = " << submap_offset << "\n"; // Get submap source ranks std::vector submap_src(submap_ghost_owners.begin(), @@ -1101,14 +1108,22 @@ IndexMap::create_submap_conn(std::span indices) const // std::vector submap_owned_global(submap_owned.size()); // this->local_to_global(submap_owned, submap_owned_global); - // ss << "submap_owned_global = " << submap_owned_global << "\n"; - // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); this->local_to_global(submap_ghost, submap_ghost_global); - // ss << "submap_ghost_global = " << submap_ghost_global << "\n"; + ss << "submap_ghost_global = " << submap_ghost_global << "\n"; + + // for (int i = 0; i < comm_size; ++i) + // { + // if (i == rank) + // { + // std::cout << ss.str() << "\n"; + // } + // MPI_Barrier(_comm.comm()); + // } + // ss = std::stringstream(); // FIXME Maybe return submap_owned to as global to simplify // compute_submap_ghost_indices? From 06dab30fe4e369911a831d59b5ba5fa1936f97b8 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Mon, 13 Nov 2023 16:18:29 +0000 Subject: [PATCH 025/101] Fix bug where ghosts ordering got the wrong submap index --- cpp/dolfinx/common/IndexMap.cpp | 52 ++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 05c05fdfb54..35c374a688d 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -312,7 +312,11 @@ std::vector compute_submap_ghost_indices( // --- Step 1 ---: Send submap global ghost indices (indexed w.r.t. original // imap) to owning rank + // ss << "submap_src = " << submap_src << "\n"; + // ss << "submap_dest = " << submap_dest << "\n"; + std::vector recv_indices; + std::vector ghost_perm; std::vector send_disp, recv_disp; std::vector send_sizes, recv_sizes; { @@ -326,6 +330,7 @@ std::vector compute_submap_ghost_indices( // Pack ghosts indices std::vector> send_data(submap_src.size()); + std::vector> ghost_perm_dict(submap_src.size()); for (std::size_t i = 0; i < submap_ghosts_global.size(); ++i) { auto it = std::lower_bound(submap_src.begin(), submap_src.end(), @@ -333,8 +338,11 @@ std::vector compute_submap_ghost_indices( assert(it != submap_src.end() and *it == submap_ghost_owners[i]); int r = std::distance(submap_src.begin(), it); send_data[r].push_back(submap_ghosts_global[i]); + ghost_perm_dict[r].push_back(i); } + // ss << "ghost_perm_dict = " << ghost_perm_dict << "\n"; + // Count number of ghosts per dest std::transform(send_data.begin(), send_data.end(), std::back_inserter(send_sizes), @@ -344,6 +352,10 @@ std::vector compute_submap_ghost_indices( std::vector send_indices; for (auto& d : send_data) send_indices.insert(send_indices.end(), d.begin(), d.end()); + for (auto& p : ghost_perm_dict) + ghost_perm.insert(ghost_perm.end(), p.begin(), p.end()); + + // ss << "send_indices =" << send_indices << "\n"; // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost @@ -362,6 +374,9 @@ std::vector compute_submap_ghost_indices( std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); + // ss << "send_sizes = " << send_sizes << "\n"; + // ss << "recv_sizes = " << recv_sizes << "\n"; + // Send ghost indices to owner, and receive indices recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), @@ -385,6 +400,8 @@ std::vector compute_submap_ghost_indices( std::vector recv_indices_local(recv_indices.size()); imap.global_to_local(recv_indices, recv_indices_local); + // ss << "recv_indices = " << recv_indices << "\n"; + // ss << "recv_indices_local = " << recv_indices_local << "\n"; // for (int i = 0; i < dolfinx::MPI::size(comm); ++i) @@ -434,18 +451,13 @@ std::vector compute_submap_ghost_indices( // ss << "recv_gidx = " << recv_gidx << "\n"; + // ss << "ghost_perm = " << ghost_perm << "\n"; + // // --- Step 4---: Unpack received data - std::vector ghost_submap_gidx; - for (std::size_t i = 0; i < send_disp.size() - 1; ++i) - { - for (int j = send_disp[i]; j < send_disp[i + 1]; ++j) - { - std::int64_t idx = recv_gidx[j]; - assert(idx >= 0); - ghost_submap_gidx.push_back(idx); - } - } + std::vector ghost_submap_gidx(submap_ghosts_global); + for (std::size_t i = 0; i < recv_gidx.size(); ++i) + ghost_submap_gidx[ghost_perm[i]] = recv_gidx[i]; // ss << "ghost_submap_gidx = " << ghost_submap_gidx << "\n"; @@ -1054,7 +1066,7 @@ std::pair> IndexMap::create_submap_conn(std::span indices) const { const int rank = dolfinx::MPI::rank(_comm.comm()); - // const int comm_size = dolfinx::MPI::size(_comm.comm()); + const int comm_size = dolfinx::MPI::size(_comm.comm()); ss << "Rank " << rank << ":\n"; ss << "indices = "; @@ -1144,15 +1156,15 @@ IndexMap::create_submap_conn(std::span indices) const // ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; - // for (int i = 0; i < comm_size; ++i) - // { - // if (i == rank) - // { - // std::cout << ss.str() << "\n"; - // } - // MPI_Barrier(_comm.comm()); - // } - // ss = std::stringstream(); + for (int i = 0; i < comm_size; ++i) + { + if (i == rank) + { + std::cout << ss.str() << "\n"; + } + MPI_Barrier(_comm.comm()); + } + ss = std::stringstream(); return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), From d11f1843eb5a87e0a84d4f608389807f550da448 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 14 Nov 2023 14:32:42 +0000 Subject: [PATCH 026/101] Tidy --- cpp/dolfinx/common/IndexMap.cpp | 203 ++++------------------ cpp/dolfinx/common/IndexMap.h | 13 +- cpp/dolfinx/mesh/Topology.cpp | 2 + python/dolfinx/wrappers/common.cpp | 6 +- python/test/unit/common/test_index_map.py | 52 +++--- 5 files changed, 74 insertions(+), 202 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index ed9dba0ae08..e5be8086404 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,37 +14,9 @@ #include #include -#include - using namespace dolfinx; using namespace dolfinx::common; -std::stringstream ss; - -// A function for printing vectors -template -std::ostream& operator<<(std::ostream& os, const std::vector& vector) -{ - os << "{ "; - for (auto v : vector) - { - os << v << " "; - } - os << "}"; - return os; -} - -// A function for printing maps -template -std::ostream& operator<<(std::ostream& os, const std::map& map) -{ - os << "{ "; - for (const auto& [k, v] : map) - os << k << ": " << v << " "; - os << "}"; - return os; -} - namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -61,6 +33,10 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } +// A helper function that, given an index map and a set of local indices (can be +// owned or ghost but must be unique and sorted), returns the owned, ghost and +// ghost owners in the submap. All indices are local and with respect to the +// original index map. std::tuple, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, @@ -76,8 +52,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, const std::vector& dest = imap.dest(); const std::vector& ghosts = imap.ghosts(); const std::vector& ghost_owners = imap.owners(); - // ss << "src = " << src << "\n"; - // ss << "dest = " << dest << "\n"; // Create neighbourhood comm (ghost -> owner) MPI_Comm comm0; @@ -90,9 +64,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); for (auto v : indices) is_in_submap[v] = 1; - // ss << "is_in_submap = " << is_in_submap << "\n"; - // Pack ghosts indices that this process wants to include in the + // Pack ghost indices that this process wants to include in the // sub-map std::vector> send_data(src.size()); for (std::size_t i = 0; i < ghosts.size(); ++i) @@ -106,7 +79,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, else send_data[r].push_back(-1); } - // ss << "send_data = " << send_data << "\n"; // Count number of indices to send per dest std::vector send_sizes; @@ -114,8 +86,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::back_inserter(send_sizes), [](auto& d) { return d.size(); }); - // ss << "send_sizes = " << send_sizes << "\n"; - // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost std::vector recv_sizes; @@ -126,8 +96,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_sizes.data(), 1, MPI_INT32_T, comm0); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_sizes = " << recv_sizes << "\n"; - // Prepare displacement vectors std::vector send_disp, recv_disp; send_disp.resize(src.size() + 1, 0); @@ -137,17 +105,12 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); - // ss << "send_disp = " << send_disp << "\n"; - // ss << "recv_disp = " << recv_disp << "\n"; - // Send ghost indices to owner, and receive indices owned by this process // that are ghosts on others std::vector send_indices, recv_indices; for (auto& d : send_data) send_indices.insert(send_indices.end(), d.begin(), d.end()); - // ss << "send_indices = " << send_indices << "\n"; - recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, @@ -155,8 +118,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_indices = " << recv_indices << "\n"; - // Free the communicator ierr = MPI_Comm_free(&comm0); dolfinx::MPI::check_error(comm, ierr); @@ -164,11 +125,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. // indices owned by this process that are in `indices` on other processes) to // their owner in the submap. This is required since not all indices in - // `recv_indices` will be in `indices` on this process, and thus other - // processes must own them in the submap. + // `recv_indices` will necessarily be in `indices` on this process, and thus + // other processes must own them in the submap. - // Create a map from (global) indices owned by this process that are ghosted - // on others to processes that can own them. + // Create a map from (global) owned shared indices owned to processes that can + // own them. // FIXME Avoid map const int rank = dolfinx::MPI::rank(comm); // A map from (global) indices in `recv_indices` to a list of processes that @@ -200,9 +161,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } - // ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner - // << "\n"; - // Choose the submap owner for each index in `recv_indices` std::vector send_owners; send_owners.reserve(recv_indices.size()); @@ -211,7 +169,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Check the index is in the submap, otherwise send -1 if (idx != -1) { - // Choose new owner randomly for load balancing + // TODO Choose new owner randomly for load balancing const std::vector& possible_owners = global_idx_to_possible_owner[idx]; // const int random_index = std::rand() % possible_owners.size(); @@ -224,8 +182,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_owners.push_back(-1); } - // ss << "send_owners = " << send_owners << "\n"; - // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; ierr = MPI_Dist_graph_create_adjacent( @@ -245,8 +201,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_owners = " << recv_owners << "\n"; - // Local indices (w.r.t. original map) owned by this process in the submap std::vector submap_owned; // Local indices (w.r.t. original map) ghosted by this process in the submap @@ -254,6 +208,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // The owners of the submap ghost indices (process submap_ghost_owners[i] owns // index submap_ghost[i]) std::vector submap_ghost_owners; + + // Add owned indices to submap_owned for (std::int32_t v : indices) if (v < imap.size_local()) submap_owned.push_back(v); @@ -262,6 +218,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector send_indices_local(send_indices.size()); imap.global_to_local(send_indices, send_indices_local); + // Loop over ghost indices (in the original map) and add to submap_owned + // if the owning process has decided this process to be the submap owner. + // Else, add the index and its (possibly new) owner to submap_ghost and + // submap_ghost_owners respectively. for (std::size_t i = 0; i < send_indices.size(); ++i) { std::int32_t local_idx = send_indices_local[i]; @@ -284,22 +244,12 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } std::sort(submap_owned.begin(), submap_owned.end()); - // std::vector perm(submap_ghost.size()); - // std::iota(perm.begin(), perm.end(), 0); - // std::sort(perm.begin(), perm.end(), [&](int i, int j){return submap_ghost[i] < submap_ghost[j];}); - - // std::vector submap_ghost_sorted(submap_ghost.size()); - // std::vector submap_ghost_owners_sorted(submap_ghost_owners.size()); - // for (std::size_t i = 0; i < submap_ghost.size(); ++i) - // { - // submap_ghost_sorted[i] = submap_ghost[perm[i]]; - // submap_ghost_owners_sorted[i] = submap_ghost_owners[perm[i]]; - // } - // return {submap_owned, submap_ghost_sorted, submap_ghost_owners_sorted}; - // TODO Move return {submap_owned, submap_ghost, submap_ghost_owners}; } + +// Helper function to compute the submap indices of ghosts. +// NOTE: submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( const MPI_Comm& comm, const std::vector& submap_src, const std::vector& submap_dest, @@ -309,12 +259,8 @@ std::vector compute_submap_ghost_indices( const int submap_offset, const dolfinx::common::IndexMap& imap) // NOTE: This is the original imap! { - // --- Step 1 ---: Send submap global ghost indices (indexed w.r.t. original - // imap) to owning rank - - // ss << "submap_src = " << submap_src << "\n"; - // ss << "submap_dest = " << submap_dest << "\n"; - + // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning + // rank std::vector recv_indices; std::vector ghost_perm; std::vector send_disp, recv_disp; @@ -341,8 +287,6 @@ std::vector compute_submap_ghost_indices( ghost_perm_dict[r].push_back(i); } - // ss << "ghost_perm_dict = " << ghost_perm_dict << "\n"; - // Count number of ghosts per dest std::transform(send_data.begin(), send_data.end(), std::back_inserter(send_sizes), @@ -355,8 +299,6 @@ std::vector compute_submap_ghost_indices( for (auto& p : ghost_perm_dict) ghost_perm.insert(ghost_perm.end(), p.begin(), p.end()); - // ss << "send_indices =" << send_indices << "\n"; - // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost recv_sizes.resize(submap_dest.size(), 0); @@ -374,10 +316,8 @@ std::vector compute_submap_ghost_indices( std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); - // ss << "send_sizes = " << send_sizes << "\n"; - // ss << "recv_sizes = " << recv_sizes << "\n"; - - // Send ghost indices to owner, and receive indices + // Send ghost indices to owner, and receive indices owned by this process + // that are shared with others in the submap recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, @@ -395,39 +335,21 @@ std::vector compute_submap_ghost_indices( send_gidx.reserve(recv_indices.size()); // NOTE: Received indices are owned by this process in the submap, but not // necessarily in the original imap, so we must use global_to_local to convert - // rather than rather than subtracting the local_range[0] + // rather than subtracting local_range[0] // TODO Convert recv_indices or submap_owned? std::vector recv_indices_local(recv_indices.size()); imap.global_to_local(recv_indices, recv_indices_local); - // ss << "recv_indices = " << recv_indices << "\n"; - - // ss << "recv_indices_local = " << recv_indices_local << "\n"; - - // for (int i = 0; i < dolfinx::MPI::size(comm); ++i) - // { - // if (i == dolfinx::MPI::rank(comm)) - // { - // std::cout << ss.str() << "\n"; - // } - // MPI_Barrier(comm); - // } - // ss = std::stringstream(); - - // ss << "recv_indices_local = " << recv_indices_local << "\n"; + // Compute submap global index for (auto idx : recv_indices_local) { - // TODO Check submap owned is always sorted // Could avoid search by creating look-up array auto it = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); - // auto it = std::find(submap_owned.begin(), submap_owned.end(), idx); assert(it != submap_owned.end() and *it == idx); std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); send_gidx.push_back(idx_local_submap + submap_offset); } - // ss << "send_gidx = " << send_gidx << "\n"; - // --- Step 3 ---: Send submap global indices to process that ghost them // Create neighbourhood comm (owner -> ghost) @@ -449,18 +371,12 @@ std::vector compute_submap_ghost_indices( ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); - // ss << "recv_gidx = " << recv_gidx << "\n"; - - // ss << "ghost_perm = " << ghost_perm << "\n"; - - // // --- Step 4---: Unpack received data + // --- Step 4---: Unpack received data std::vector ghost_submap_gidx(submap_ghosts_global); for (std::size_t i = 0; i < recv_gidx.size(); ++i) ghost_submap_gidx[ghost_perm[i]] = recv_gidx[i]; - // ss << "ghost_submap_gidx = " << ghost_submap_gidx << "\n"; - return ghost_submap_gidx; } } // namespace @@ -1070,41 +986,19 @@ IndexMap::create_submap(std::span indices) const std::pair> IndexMap::create_submap_conn(std::span indices) const { - const int rank = dolfinx::MPI::rank(_comm.comm()); - const int comm_size = dolfinx::MPI::size(_comm.comm()); - ss << "Rank " << rank << ":\n"; - - ss << "indices = "; - for (auto idx : indices) - { - ss << idx << " "; - } - ss << "\n"; - - ss << "this->size_local() = " << this->size_local() << "\n"; - - std::vector indices_global(indices.size()); - this->local_to_global(indices, indices_global); - ss << "indices_global = " << indices_global << "\n"; - - // TODO Maybe submap_ghost (at least) should return global indices? + // Compute the owned, ghost, and ghost owners of submap indices. + // NOTE: All indices are local and numbered w.r.t. the original (this) index + // map auto [submap_owned, submap_ghost, submap_ghost_owners] = compute_submap_indices(*this, indices); - ss << "submap_owned = " << submap_owned << "\n"; - ss << "submap_ghost = " << submap_ghost << "\n"; - ss << "submap_ghost_owners = " << submap_ghost_owners << "\n"; - // Compute submap offset for this rank std::int64_t submap_local_size = submap_owned.size(); - ss << "submap_local_size = " << submap_local_size << "\n"; std::int64_t submap_offset = 0; int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, MPI_SUM, _comm.comm()); dolfinx::MPI::check_error(_comm.comm(), ierr); - ss << "submap_offset = " << submap_offset << "\n"; - // Get submap source ranks std::vector submap_src(submap_ghost_owners.begin(), submap_ghost_owners.end()); @@ -1113,43 +1007,20 @@ IndexMap::create_submap_conn(std::span indices) const submap_src.end()); submap_src.shrink_to_fit(); - // ss << "submap_src = " << submap_src << "\n"; - // Compute submap destination ranks - // FIXME Can NBX call be avoided by using O^r_p and O^g_p? + // NOTE: The call to NBX can be avoided by a two-step communication process + // TODO: Can NBX call be avoided by using O^r_p and O^g_p? std::vector submap_dest = dolfinx::MPI::compute_graph_edges_nbx(_comm.comm(), submap_src); std::sort(submap_dest.begin(), submap_dest.end()); - // ss << "submap_dest = " << submap_dest << "\n"; - - // std::vector submap_owned_global(submap_owned.size()); - // this->local_to_global(submap_owned, submap_owned_global); - // ss << "submap_owned_global = " << submap_owned_global << "\n"; // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); this->local_to_global(submap_ghost, submap_ghost_global); - - ss << "submap_ghost_global = " << submap_ghost_global << "\n"; - - // for (int i = 0; i < comm_size; ++i) - // { - // if (i == rank) - // { - // std::cout << ss.str() << "\n"; - // } - // MPI_Barrier(_comm.comm()); - // } - // ss = std::stringstream(); - - // FIXME Maybe return submap_owned to as global to simplify - // compute_submap_ghost_indices? auto submap_ghost_gidxs = compute_submap_ghost_indices( _comm.comm(), submap_src, submap_dest, submap_owned, submap_ghost_global, submap_ghost_owners, submap_offset, *this); - // ss << "submap_ghost_gidxs = " << submap_ghost_gidxs << "\n"; - // Create a map from (local) indices in the submap to the corresponding // (local) index in the original map std::vector sub_imap_to_imap; @@ -1159,18 +1030,6 @@ IndexMap::create_submap_conn(std::span indices) const sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), submap_ghost.end()); - // ss << "sub_imap_to_imap = " << sub_imap_to_imap << "\n"; - - for (int i = 0; i < comm_size; ++i) - { - if (i == rank) - { - std::cout << ss.str() << "\n"; - } - MPI_Barrier(_comm.comm()); - } - ss = std::stringstream(); - return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), std::move(sub_imap_to_imap)}; diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 94660ba6c55..4f95500d419 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -189,9 +189,18 @@ class IndexMap std::pair> create_submap(std::span indices) const; - /// @brief TODO - /// @param[in] indices + /// @brief Create a new index map from a subset of indices in this index + /// map, changing the owner of indices that are included on other processes + /// but not on the owning process. + /// + /// This can be used when, for instance, creating a submesh to ensure + /// that all vertices have exactly one owner and are connected to at least + /// one cell on the owning process. + /// @param[in] indices Local indices to include in the new index map (owned + /// and ghost) /// @pre `indices` must be sorted and contain no duplicates. + /// @return The (i) new index map and (ii) a map from local indices in the + /// submap to local indices in the original (this) map std::pair> create_submap_conn(std::span indices) const; diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index 1ee1b247149..0ef376a2eb8 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1196,6 +1196,8 @@ mesh::create_subtopology(const Topology& topology, int dim, auto map0 = topology.index_map(0); assert(map0); + // Create map from the vertices in the sub-topology to the vertices in the + // parent topology, and an index map std::shared_ptr submap0; std::vector subvertices0; { diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 6faaeba9be4..43d88a94968 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -138,14 +138,14 @@ void common(nb::module_& m) "create_submap", [](const dolfinx::common::IndexMap& self, nb::ndarray, nb::c_contig> - entities) + indices) { auto [map, ghosts] = self.create_submap( - std::span(entities.data(), entities.size())); + std::span(indices.data(), indices.size())); return std::pair(std::move(map), dolfinx_wrappers::as_nbarray(std::move(ghosts))); }, - nb::arg("entities")) + nb::arg("indices")) .def( "create_submap_conn", [](const dolfinx::common::IndexMap& self, diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 7423087ea23..392c2897b93 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -111,32 +111,34 @@ def test_index_map_ghost_lifetime(): # TODO: Add test for case where more than one two process shares an index # whose owner changes in the submap -# Test create_submap_conn. The diagram illustrates the case with four -# processes. Original map numbering and connectivity (G indicates a ghost -# index): -# Global Rank 0 Rank 1 Rank 2 Rank 3 -# 1 - 0 1 - 0 -# | / | | / | -# 3 - 2 3G - 2 0 - 2G -# | / | | / | -# 5 - 4 3G - 1 0 - 2G -# | / | | / | -# 7 - 6 3G - 1 0 - 3G -# | / | | / | -# 9 - 8 2 - 1 -# We now create a submap of the "upper triangular" parts (i.e. to -# get the following): -# Global Rank 0 Rank 1 Rank 2 Rank 3 -# 1 - 0 1 - 0 -# | / | / -# 2 - 3 2G 0 - 1 -# | / | / -# 4 - 5 2G 0 - 1 -# | / | / -# 6 - 8 2G 0 - 2 -# | / | / -# 7 1 def test_create_submap_connected(): + """ + Test create_submap_conn. The diagram illustrates the case with four + processes. Original map numbering and connectivity (G indicates a ghost + index): + Global Rank 0 Rank 1 Rank 2 Rank 3 + 1 - 0 1 - 0 + | / | | / | + 3 - 2 3G - 2 0 - 2G + | / | | / | + 5 - 4 3G - 1 0 - 2G + | / | | / | + 7 - 6 3G - 1 0 - 3G + | / | | / | + 9 - 8 2 - 1 + We now create a submap of the "upper triangular" parts to + get the following: + Global Rank 0 Rank 1 Rank 2 Rank 3 + 1 - 0 1 - 0 + | / | / + 2 - 3 2G 0 - 1 + | / | / + 4 - 5 2G 0 - 1 + | / | / + 6 - 8 2G 0 - 2 + | / | / + 7 1 + """ comm = MPI.COMM_WORLD if comm.size == 1: From 4bf992b5b7bfbd232d71d021f8c9dc897590ae74 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 14 Nov 2023 15:30:30 +0000 Subject: [PATCH 027/101] Improve test --- python/test/unit/fem/test_assemble_submesh.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/test/unit/fem/test_assemble_submesh.py b/python/test/unit/fem/test_assemble_submesh.py index b3ca0022891..939056d4b59 100644 --- a/python/test/unit/fem/test_assemble_submesh.py +++ b/python/test/unit/fem/test_assemble_submesh.py @@ -32,8 +32,7 @@ def assemble(mesh, space, k): dofs = fem.locate_dofs_topological(V, facet_dim, facets) bc_func = fem.Function(V) - # TODO Interpolate when issue #2126 has been resolved - bc_func.x.array[:] = 1.0 + bc_func.interpolate(lambda x: np.sin(x[0])) bc = fem.dirichletbc(bc_func, dofs) From ce5ecafc23786985e24c13c045fe8e4dd66857fc Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 10:17:25 +0000 Subject: [PATCH 028/101] Remove TODO --- cpp/dolfinx/common/IndexMap.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index e5be8086404..8568dcf98ad 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -244,7 +244,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } std::sort(submap_owned.begin(), submap_owned.end()); - // TODO Move return {submap_owned, submap_ghost, submap_ghost_owners}; } From 083ab0e70d50a80ab4a7d7abc075128f19ccfab0 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 11:09:24 +0000 Subject: [PATCH 029/101] Don't move --- cpp/dolfinx/common/IndexMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 8568dcf98ad..e11da84fd4d 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -1031,7 +1031,7 @@ IndexMap::create_submap_conn(std::span indices) const return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), - std::move(sub_imap_to_imap)}; + sub_imap_to_imap}; } //----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const From 2973b089181fc7349f7e1f9a42117561c2bda9a0 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 12:12:20 +0000 Subject: [PATCH 030/101] Add back moves for now --- cpp/dolfinx/common/IndexMap.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index e11da84fd4d..c617c2bbebf 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -244,7 +244,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } std::sort(submap_owned.begin(), submap_owned.end()); - return {submap_owned, submap_ghost, submap_ghost_owners}; + return {std::move(submap_owned), std::move(submap_ghost), + std::move(submap_ghost_owners)}; } // Helper function to compute the submap indices of ghosts. @@ -1031,7 +1032,7 @@ IndexMap::create_submap_conn(std::span indices) const return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, submap_ghost_owners), - sub_imap_to_imap}; + std::move(sub_imap_to_imap)}; } //----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const From beb86f78468f717d2050f461b75a473896667942 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 12:21:35 +0000 Subject: [PATCH 031/101] Update cpp/dolfinx/common/IndexMap.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jørgen Schartum Dokken --- cpp/dolfinx/common/IndexMap.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c617c2bbebf..342e7724fac 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -33,10 +33,15 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } -// A helper function that, given an index map and a set of local indices (can be -// owned or ghost but must be unique and sorted), returns the owned, ghost and -// ghost owners in the submap. All indices are local and with respect to the -// original index map. +/// Given a subset of indices in an index map, compute owned indices, ghost indices and owners of ghosts. +/// +/// A helper function that, given an index map and a set of local indices (can be +/// owned or ghost but must be unique and sorted), returns the owned, ghost and +/// ghost owners in the submap. All indices are local and with respect to the +/// original index map. +/// @param[in] imap Index map. +/// @param[in] indices List of entity indices (indices local to the process). +/// @pre `indices` must be sorted and unique. std::tuple, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, From 04b88b5bfd2990d82c8d0834c26cbcf80a541a68 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 12:26:32 +0000 Subject: [PATCH 032/101] Improve docs --- cpp/dolfinx/common/IndexMap.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 342e7724fac..bbcd6dfda4c 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -33,15 +33,14 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } -/// Given a subset of indices in an index map, compute owned indices, ghost indices and owners of ghosts. -/// -/// A helper function that, given an index map and a set of local indices (can be -/// owned or ghost but must be unique and sorted), returns the owned, ghost and -/// ghost owners in the submap. All indices are local and with respect to the -/// original index map. -/// @param[in] imap Index map. +/// Given an index map and a subset of local indices (can be owned or ghost but +/// must be unique and sorted), computes the owned, ghost and ghost owners in +/// the submap. +/// @param[in] imap An index map. /// @param[in] indices List of entity indices (indices local to the process). -/// @pre `indices` must be sorted and unique. +/// @pre `indices` must be sorted and unique. +/// @return The owned, ghost, and ghost owners in the submap. All indices are +/// local and with respect to the original index map. std::tuple, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, From f5123fc96308010b9f80c67430ea408a6fb1c975 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 13:31:34 +0000 Subject: [PATCH 033/101] Use span --- cpp/dolfinx/common/IndexMap.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index bbcd6dfda4c..eb98c94d5d6 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -51,11 +51,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // on other processes const MPI_Comm comm = imap.comm(); - // TODO Should these be spans? - const std::vector& src = imap.src(); - const std::vector& dest = imap.dest(); - const std::vector& ghosts = imap.ghosts(); - const std::vector& ghost_owners = imap.owners(); + std::span src = imap.src(); + std::span dest = imap.dest(); + std::span ghosts = imap.ghosts(); + std::span ghost_owners = imap.owners(); // Create neighbourhood comm (ghost -> owner) MPI_Comm comm0; From 87e3ea91264078516672d41fd4e3094ced20bcf6 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 13:35:59 +0000 Subject: [PATCH 034/101] Get comm from imap --- cpp/dolfinx/common/IndexMap.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index eb98c94d5d6..c757abd7c26 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -251,10 +251,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::move(submap_ghost_owners)}; } -// Helper function to compute the submap indices of ghosts. +/// Computes the global indices of ghosts in a submap. // NOTE: submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( - const MPI_Comm& comm, const std::vector& submap_src, + const std::vector& submap_src, const std::vector& submap_dest, const std::vector& submap_owned, const std::vector& submap_ghosts_global, @@ -264,6 +264,7 @@ std::vector compute_submap_ghost_indices( { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning // rank + const MPI_Comm comm = imap.comm(); std::vector recv_indices; std::vector ghost_perm; std::vector send_disp, recv_disp; @@ -1021,7 +1022,7 @@ IndexMap::create_submap_conn(std::span indices) const std::vector submap_ghost_global(submap_ghost.size()); this->local_to_global(submap_ghost, submap_ghost_global); auto submap_ghost_gidxs = compute_submap_ghost_indices( - _comm.comm(), submap_src, submap_dest, submap_owned, submap_ghost_global, + submap_src, submap_dest, submap_owned, submap_ghost_global, submap_ghost_owners, submap_offset, *this); // Create a map from (local) indices in the submap to the corresponding From fecf9e26c69ce82938f65310f29b6a26b224036b Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:01:19 +0000 Subject: [PATCH 035/101] Improve docs --- cpp/dolfinx/common/IndexMap.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c757abd7c26..2498aca426b 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -252,7 +252,14 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } /// Computes the global indices of ghosts in a submap. -// NOTE: submap_owned must be sorted and contain no repeated indices +/// @param[in] submap_src The submap source ranks +/// @param[in] submap_dest The submap destination ranks +/// @param[in] submap_owned Owned submap indices (local w.r.t. original index map) +/// @param[in] submap_ghosts_global Ghost submap indices (global w.r.t. original index map) +/// @param[in] submap_ghost_owners The ranks that own the ghosts in the submap +/// @param[in] submap_offset The global offset for this rank in the submap +/// @param[in] imap The original index map +/// @pre submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( const std::vector& submap_src, const std::vector& submap_dest, From af0fbbe130842f99f654e1c58b0052f10d430464 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:19:58 +0000 Subject: [PATCH 036/101] Use std::for_each --- cpp/dolfinx/common/IndexMap.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2498aca426b..3d2b9a57948 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -65,8 +65,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Create lookup array to determine if an index is in the sub-map std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); - for (auto v : indices) - is_in_submap[v] = 1; + std::for_each(indices.begin(), indices.end(), + [&is_in_submap](int i) { is_in_submap[i] = 1; }); // Pack ghost indices that this process wants to include in the // sub-map @@ -254,15 +254,16 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, /// Computes the global indices of ghosts in a submap. /// @param[in] submap_src The submap source ranks /// @param[in] submap_dest The submap destination ranks -/// @param[in] submap_owned Owned submap indices (local w.r.t. original index map) -/// @param[in] submap_ghosts_global Ghost submap indices (global w.r.t. original index map) +/// @param[in] submap_owned Owned submap indices (local w.r.t. original index +/// map) +/// @param[in] submap_ghosts_global Ghost submap indices (global w.r.t. original +/// index map) /// @param[in] submap_ghost_owners The ranks that own the ghosts in the submap /// @param[in] submap_offset The global offset for this rank in the submap /// @param[in] imap The original index map /// @pre submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( - const std::vector& submap_src, - const std::vector& submap_dest, + const std::vector& submap_src, const std::vector& submap_dest, const std::vector& submap_owned, const std::vector& submap_ghosts_global, const std::vector& submap_ghost_owners, From c10ff3a27f475ceec2a986ffcee4d5fa69c5cdbe Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:28:14 +0000 Subject: [PATCH 037/101] Use std::size_t --- cpp/dolfinx/common/IndexMap.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 3d2b9a57948..1b777c69439 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -75,7 +75,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, { auto it = std::lower_bound(src.begin(), src.end(), ghost_owners[i]); assert(it != src.end() and *it == ghost_owners[i]); - int r = std::distance(src.begin(), it); + std::size_t r = std::distance(src.begin(), it); // Send ghost index if it is in the submap, else send -1 if (is_in_submap[imap.size_local() + i]) send_data[r].push_back(ghosts[i]); @@ -294,7 +294,7 @@ std::vector compute_submap_ghost_indices( auto it = std::lower_bound(submap_src.begin(), submap_src.end(), submap_ghost_owners[i]); assert(it != submap_src.end() and *it == submap_ghost_owners[i]); - int r = std::distance(submap_src.begin(), it); + std::size_t r = std::distance(submap_src.begin(), it); send_data[r].push_back(submap_ghosts_global[i]); ghost_perm_dict[r].push_back(i); } From a8ec0cd925a6f01fb9382b07b339d7ff28eba774 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:35:45 +0000 Subject: [PATCH 038/101] Style change --- cpp/dolfinx/common/IndexMap.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 1b777c69439..68cf1874a85 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -144,9 +144,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, { for (int j = recv_disp[i]; j < recv_disp[i + 1]; ++j) { - std::int64_t idx = recv_indices[j]; // Check that the index is in the submap - if (idx != -1) + if (std::int64_t idx = recv_indices[j]; idx != -1) { // Compute the local index std::int32_t idx_local = idx - local_range[0]; From 5631300875475c831c9d0dedc4c3e27fd44dbb85 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:41:42 +0000 Subject: [PATCH 039/101] Fix spacing --- cpp/dolfinx/common/IndexMap.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 68cf1874a85..14f376c4033 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -205,8 +205,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Local indices (w.r.t. original map) owned by this process in the submap std::vector submap_owned; + // Local indices (w.r.t. original map) ghosted by this process in the submap std::vector submap_ghost; + // The owners of the submap ghost indices (process submap_ghost_owners[i] owns // index submap_ghost[i]) std::vector submap_ghost_owners; From 57a6539f9efeca650ff61bfaa26d871552a856e4 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:43:56 +0000 Subject: [PATCH 040/101] Move initialisation --- cpp/dolfinx/common/IndexMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 14f376c4033..d0dd63d92c1 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -228,13 +228,13 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // submap_ghost_owners respectively. for (std::size_t i = 0; i < send_indices.size(); ++i) { - std::int32_t local_idx = send_indices_local[i]; std::int64_t global_idx = send_indices[i]; std::int32_t owner = recv_owners[i]; // Check if index is in the submap if (global_idx >= 0) { + std::int32_t local_idx = send_indices_local[i]; if (owner == rank) { submap_owned.push_back(local_idx); From 0de942b77032a27d426d2eea94cfe05320c9fb07 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 14:49:13 +0000 Subject: [PATCH 041/101] Initialise in if statement --- cpp/dolfinx/common/IndexMap.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index d0dd63d92c1..8232dcba6de 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -228,14 +228,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // submap_ghost_owners respectively. for (std::size_t i = 0; i < send_indices.size(); ++i) { - std::int64_t global_idx = send_indices[i]; - std::int32_t owner = recv_owners[i]; - // Check if index is in the submap - if (global_idx >= 0) + if (std::int64_t global_idx = send_indices[i]; global_idx >= 0) { std::int32_t local_idx = send_indices_local[i]; - if (owner == rank) + if (std::int32_t owner = recv_owners[i]; owner == rank) { submap_owned.push_back(local_idx); } From 8df44a18c1246be25df4a51a97ae42d9369b9bab Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 15:02:34 +0000 Subject: [PATCH 042/101] Use span --- cpp/dolfinx/common/IndexMap.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 8232dcba6de..697cd475d9a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -261,12 +261,13 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, /// @param[in] imap The original index map /// @pre submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( - const std::vector& submap_src, const std::vector& submap_dest, - const std::vector& submap_owned, - const std::vector& submap_ghosts_global, - const std::vector& submap_ghost_owners, + std::span submap_src, + std::span submap_dest, + std::span submap_owned, + std::span submap_ghosts_global, + std::span submap_ghost_owners, const int submap_offset, - const dolfinx::common::IndexMap& imap) // NOTE: This is the original imap! + const dolfinx::common::IndexMap& imap) { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning // rank @@ -383,7 +384,7 @@ std::vector compute_submap_ghost_indices( // --- Step 4---: Unpack received data - std::vector ghost_submap_gidx(submap_ghosts_global); + std::vector ghost_submap_gidx(submap_ghosts_global.size()); for (std::size_t i = 0; i < recv_gidx.size(); ++i) ghost_submap_gidx[ghost_perm[i]] = recv_gidx[i]; From c67d22ad0d416dbe1aa2e9b1038f148919a0d165 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 15:38:18 +0000 Subject: [PATCH 043/101] Improve clarity --- cpp/dolfinx/common/IndexMap.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 4f95500d419..6a12d582de7 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -190,8 +190,9 @@ class IndexMap create_submap(std::span indices) const; /// @brief Create a new index map from a subset of indices in this index - /// map, changing the owner of indices that are included on other processes - /// but not on the owning process. + /// map. Any indices that are not included by their owning process, but + /// are included on sharing processes, will be owned by one of the sharing + /// processes in the submap. /// /// This can be used when, for instance, creating a submesh to ensure /// that all vertices have exactly one owner and are connected to at least From b4c2569b069ea56070f4747aef5988cefce750e0 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 16:05:11 +0000 Subject: [PATCH 044/101] Start working on scoping --- cpp/dolfinx/common/IndexMap.cpp | 131 ++++++++++++++++---------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 697cd475d9a..d317739802a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -46,84 +46,85 @@ std::tuple, std::vector, compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices) { - // --- Step 1 ---: Send ghost indices in `indices` to their owners - // and receive indices owned by this process that are in `indices` - // on other processes - const MPI_Comm comm = imap.comm(); std::span src = imap.src(); std::span dest = imap.dest(); std::span ghosts = imap.ghosts(); std::span ghost_owners = imap.owners(); - // Create neighbourhood comm (ghost -> owner) - MPI_Comm comm0; - int ierr = MPI_Dist_graph_create_adjacent( - comm, dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), src.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); - dolfinx::MPI::check_error(comm, ierr); - - // Create lookup array to determine if an index is in the sub-map + // Lookup array to determine if an index is in the sub-map std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); std::for_each(indices.begin(), indices.end(), [&is_in_submap](int i) { is_in_submap[i] = 1; }); - // Pack ghost indices that this process wants to include in the - // sub-map - std::vector> send_data(src.size()); - for (std::size_t i = 0; i < ghosts.size(); ++i) + // --- Step 1 ---: Send ghost indices in `indices` to their owners + // and receive indices owned by this process that are in `indices` + // on other processes + + std::vector send_disp, recv_disp; + std::vector send_indices, recv_indices; + std::vector send_sizes, recv_sizes; { - auto it = std::lower_bound(src.begin(), src.end(), ghost_owners[i]); - assert(it != src.end() and *it == ghost_owners[i]); - std::size_t r = std::distance(src.begin(), it); - // Send ghost index if it is in the submap, else send -1 - if (is_in_submap[imap.size_local() + i]) - send_data[r].push_back(ghosts[i]); - else - send_data[r].push_back(-1); - } + // Create neighbourhood comm (ghost -> owner) + MPI_Comm comm0; + int ierr = MPI_Dist_graph_create_adjacent( + comm, dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), src.data(), + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); + dolfinx::MPI::check_error(comm, ierr); - // Count number of indices to send per dest - std::vector send_sizes; - std::transform(send_data.begin(), send_data.end(), - std::back_inserter(send_sizes), - [](auto& d) { return d.size(); }); + // Pack ghost indices that this process wants to include in the + // sub-map + std::vector> send_data(src.size()); + for (std::size_t i = 0; i < ghosts.size(); ++i) + { + auto it = std::lower_bound(src.begin(), src.end(), ghost_owners[i]); + assert(it != src.end() and *it == ghost_owners[i]); + std::size_t r = std::distance(src.begin(), it); + // Send ghost index if it is in the submap, else send -1 + if (is_in_submap[imap.size_local() + i]) + send_data[r].push_back(ghosts[i]); + else + send_data[r].push_back(-1); + } - // Send how many indices I ghost to each owner, and receive how many - // of my indices other ranks ghost - std::vector recv_sizes; - recv_sizes.resize(dest.size(), 0); - send_sizes.reserve(1); - recv_sizes.reserve(1); - ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); - dolfinx::MPI::check_error(comm, ierr); + // Count number of indices to send per dest + std::transform(send_data.begin(), send_data.end(), + std::back_inserter(send_sizes), + [](auto& d) { return d.size(); }); - // Prepare displacement vectors - std::vector send_disp, recv_disp; - send_disp.resize(src.size() + 1, 0); - recv_disp.resize(dest.size() + 1, 0); - std::partial_sum(send_sizes.begin(), send_sizes.end(), - std::next(send_disp.begin())); - std::partial_sum(recv_sizes.begin(), recv_sizes.end(), - std::next(recv_disp.begin())); + // Send how many indices I ghost to each owner, and receive how many + // of my indices other ranks ghost + recv_sizes.resize(dest.size(), 0); + send_sizes.reserve(1); + recv_sizes.reserve(1); + ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, + recv_sizes.data(), 1, MPI_INT32_T, comm0); + dolfinx::MPI::check_error(comm, ierr); - // Send ghost indices to owner, and receive indices owned by this process - // that are ghosts on others - std::vector send_indices, recv_indices; - for (auto& d : send_data) - send_indices.insert(send_indices.end(), d.begin(), d.end()); + // Prepare displacement vectors + send_disp.resize(src.size() + 1, 0); + recv_disp.resize(dest.size() + 1, 0); + std::partial_sum(send_sizes.begin(), send_sizes.end(), + std::next(send_disp.begin())); + std::partial_sum(recv_sizes.begin(), recv_sizes.end(), + std::next(recv_disp.begin())); - recv_indices.resize(recv_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), - send_disp.data(), MPI_INT64_T, - recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, comm0); - dolfinx::MPI::check_error(comm, ierr); + // Send ghost indices to owner, and receive indices owned by this process + // that are ghosts on others + for (auto& d : send_data) + send_indices.insert(send_indices.end(), d.begin(), d.end()); - // Free the communicator - ierr = MPI_Comm_free(&comm0); - dolfinx::MPI::check_error(comm, ierr); + recv_indices.resize(recv_disp.back()); + ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), + send_disp.data(), MPI_INT64_T, + recv_indices.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT64_T, comm0); + dolfinx::MPI::check_error(comm, ierr); + + // Free the communicator + ierr = MPI_Comm_free(&comm0); + dolfinx::MPI::check_error(comm, ierr); + } // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. // indices owned by this process that are in `indices` on other processes) to @@ -186,7 +187,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; - ierr = MPI_Dist_graph_create_adjacent( + int ierr = MPI_Dist_graph_create_adjacent( comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); dolfinx::MPI::check_error(comm, ierr); @@ -261,12 +262,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, /// @param[in] imap The original index map /// @pre submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( - std::span submap_src, - std::span submap_dest, + std::span submap_src, std::span submap_dest, std::span submap_owned, std::span submap_ghosts_global, - std::span submap_ghost_owners, - const int submap_offset, + std::span submap_ghost_owners, const int submap_offset, const dolfinx::common::IndexMap& imap) { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning From aa0b113812df93c167e61ab7d55877e9c1422a8f Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 16:12:10 +0000 Subject: [PATCH 045/101] More scoping --- cpp/dolfinx/common/IndexMap.cpp | 123 +++++++++++++++++--------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index d317739802a..d184b1e56a4 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -132,77 +132,82 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // `recv_indices` will necessarily be in `indices` on this process, and thus // other processes must own them in the submap. - // Create a map from (global) owned shared indices owned to processes that can - // own them. - // FIXME Avoid map + std::vector recv_owners(send_disp.back()); const int rank = dolfinx::MPI::rank(comm); - // A map from (global) indices in `recv_indices` to a list of processes that - // can own the index in the submap. - std::map> - global_idx_to_possible_owner; - const std::array local_range = imap.local_range(); - for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { - for (int j = recv_disp[i]; j < recv_disp[i + 1]; ++j) + // Create a map from (global) owned shared indices owned to processes that + // can own them. + // FIXME Avoid map + // A map from (global) indices in `recv_indices` to a list of processes that + // can own the index in the submap. + std::map> + global_idx_to_possible_owner; + const std::array local_range = imap.local_range(); + for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { - // Check that the index is in the submap - if (std::int64_t idx = recv_indices[j]; idx != -1) + for (int j = recv_disp[i]; j < recv_disp[i + 1]; ++j) { - // Compute the local index - std::int32_t idx_local = idx - local_range[0]; - assert(idx_local >= 0); - assert(idx_local < local_range[1]); - - // Check if index is in the submap on this process. If so, this process - // remains its owner in the submap. Otherwise, add the process that sent - // it to the list of possible owners. - if (is_in_submap[idx_local]) - global_idx_to_possible_owner[idx].push_back(rank); - else - global_idx_to_possible_owner[idx].push_back(dest[i]); + // Check that the index is in the submap + if (std::int64_t idx = recv_indices[j]; idx != -1) + { + // Compute the local index + std::int32_t idx_local = idx - local_range[0]; + assert(idx_local >= 0); + assert(idx_local < local_range[1]); + + // Check if index is in the submap on this process. If so, this + // process remains its owner in the submap. Otherwise, add the process + // that sent it to the list of possible owners. + if (is_in_submap[idx_local]) + global_idx_to_possible_owner[idx].push_back(rank); + else + global_idx_to_possible_owner[idx].push_back(dest[i]); + } } } - } - // Choose the submap owner for each index in `recv_indices` - std::vector send_owners; - send_owners.reserve(recv_indices.size()); - for (auto idx : recv_indices) - { - // Check the index is in the submap, otherwise send -1 - if (idx != -1) + // Choose the submap owner for each index in `recv_indices` + std::vector send_owners; + send_owners.reserve(recv_indices.size()); + for (auto idx : recv_indices) { - // TODO Choose new owner randomly for load balancing - const std::vector& possible_owners - = global_idx_to_possible_owner[idx]; - // const int random_index = std::rand() % possible_owners.size(); - // send_owners.push_back(possible_owners[random_index]); - - // FIXME TEMPORARILY CHOOSE THE FIRST PROCESS, MAKE RANDOM - send_owners.push_back(possible_owners[0]); + // Check the index is in the submap, otherwise send -1 + if (idx != -1) + { + // TODO Choose new owner randomly for load balancing + const std::vector& possible_owners + = global_idx_to_possible_owner[idx]; + // const int random_index = std::rand() % possible_owners.size(); + // send_owners.push_back(possible_owners[random_index]); + + // FIXME TEMPORARILY CHOOSE THE FIRST PROCESS, MAKE RANDOM + send_owners.push_back(possible_owners[0]); + } + else + send_owners.push_back(-1); } - else - send_owners.push_back(-1); - } - // Create neighbourhood comm (owner -> ghost) - MPI_Comm comm1; - int ierr = MPI_Dist_graph_create_adjacent( - comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); - dolfinx::MPI::check_error(comm, ierr); + // Create neighbourhood comm (owner -> ghost) + MPI_Comm comm1; + int ierr = MPI_Dist_graph_create_adjacent( + comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); + dolfinx::MPI::check_error(comm, ierr); - // Send the data - std::vector recv_owners(send_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT32_T, - recv_owners.data(), send_sizes.data(), - send_disp.data(), MPI_INT32_T, comm1); - dolfinx::MPI::check_error(comm, ierr); + // Send the data + ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT32_T, + recv_owners.data(), send_sizes.data(), + send_disp.data(), MPI_INT32_T, comm1); + dolfinx::MPI::check_error(comm, ierr); - // Free the communicator - ierr = MPI_Comm_free(&comm1); - dolfinx::MPI::check_error(comm, ierr); + // Free the communicator + ierr = MPI_Comm_free(&comm1); + dolfinx::MPI::check_error(comm, ierr); + } + + // --- Step 3 --- : Determine the owned indices, ghost indices, and ghost + // owners in the submap // Local indices (w.r.t. original map) owned by this process in the submap std::vector submap_owned; From 8c298d23d006ecf79b59ab751508491db1b0de03 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 16:15:29 +0000 Subject: [PATCH 046/101] More scoping --- cpp/dolfinx/common/IndexMap.cpp | 52 +++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index d184b1e56a4..65a1160defc 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -219,37 +219,39 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // index submap_ghost[i]) std::vector submap_ghost_owners; - // Add owned indices to submap_owned - for (std::int32_t v : indices) - if (v < imap.size_local()) - submap_owned.push_back(v); - - // FIXME Could just create when making send_indices - std::vector send_indices_local(send_indices.size()); - imap.global_to_local(send_indices, send_indices_local); - - // Loop over ghost indices (in the original map) and add to submap_owned - // if the owning process has decided this process to be the submap owner. - // Else, add the index and its (possibly new) owner to submap_ghost and - // submap_ghost_owners respectively. - for (std::size_t i = 0; i < send_indices.size(); ++i) { - // Check if index is in the submap - if (std::int64_t global_idx = send_indices[i]; global_idx >= 0) + // Add owned indices to submap_owned + for (std::int32_t v : indices) + if (v < imap.size_local()) + submap_owned.push_back(v); + + // FIXME Could just create when making send_indices + std::vector send_indices_local(send_indices.size()); + imap.global_to_local(send_indices, send_indices_local); + + // Loop over ghost indices (in the original map) and add to submap_owned + // if the owning process has decided this process to be the submap owner. + // Else, add the index and its (possibly new) owner to submap_ghost and + // submap_ghost_owners respectively. + for (std::size_t i = 0; i < send_indices.size(); ++i) { - std::int32_t local_idx = send_indices_local[i]; - if (std::int32_t owner = recv_owners[i]; owner == rank) + // Check if index is in the submap + if (std::int64_t global_idx = send_indices[i]; global_idx >= 0) { - submap_owned.push_back(local_idx); - } - else - { - submap_ghost.push_back(local_idx); - submap_ghost_owners.push_back(owner); + std::int32_t local_idx = send_indices_local[i]; + if (std::int32_t owner = recv_owners[i]; owner == rank) + { + submap_owned.push_back(local_idx); + } + else + { + submap_ghost.push_back(local_idx); + submap_ghost_owners.push_back(owner); + } } } + std::sort(submap_owned.begin(), submap_owned.end()); } - std::sort(submap_owned.begin(), submap_owned.end()); return {std::move(submap_owned), std::move(submap_ghost), std::move(submap_ghost_owners)}; From 37a3ae042a16d921fbd9711073f8b3f2d6e9a77a Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 16:23:09 +0000 Subject: [PATCH 047/101] More scoping --- cpp/dolfinx/common/IndexMap.cpp | 66 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 65a1160defc..c8c9aa9a733 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -349,44 +349,48 @@ std::vector compute_submap_ghost_indices( // --- Step 2 ---: For each received index, compute the submap global index std::vector send_gidx; - send_gidx.reserve(recv_indices.size()); - // NOTE: Received indices are owned by this process in the submap, but not - // necessarily in the original imap, so we must use global_to_local to convert - // rather than subtracting local_range[0] - // TODO Convert recv_indices or submap_owned? - std::vector recv_indices_local(recv_indices.size()); - imap.global_to_local(recv_indices, recv_indices_local); - - // Compute submap global index - for (auto idx : recv_indices_local) { - // Could avoid search by creating look-up array - auto it = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); - assert(it != submap_owned.end() and *it == idx); - std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); - send_gidx.push_back(idx_local_submap + submap_offset); + send_gidx.reserve(recv_indices.size()); + // NOTE: Received indices are owned by this process in the submap, but not + // necessarily in the original imap, so we must use global_to_local to + // convert rather than subtracting local_range[0] + // TODO Convert recv_indices or submap_owned? + std::vector recv_indices_local(recv_indices.size()); + imap.global_to_local(recv_indices, recv_indices_local); + + // Compute submap global index + for (auto idx : recv_indices_local) + { + // Could avoid search by creating look-up array + auto it = std::lower_bound(submap_owned.begin(), submap_owned.end(), idx); + assert(it != submap_owned.end() and *it == idx); + std::size_t idx_local_submap = std::distance(submap_owned.begin(), it); + send_gidx.push_back(idx_local_submap + submap_offset); + } } // --- Step 3 ---: Send submap global indices to process that ghost them - // Create neighbourhood comm (owner -> ghost) - MPI_Comm comm1; - int ierr = MPI_Dist_graph_create_adjacent( - comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, - submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &comm1); - dolfinx::MPI::check_error(comm, ierr); - - // Send indices to ghosting ranks std::vector recv_gidx(send_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, recv_gidx.data(), - send_sizes.data(), send_disp.data(), - MPI_INT64_T, comm1); - dolfinx::MPI::check_error(comm, ierr); + { + // Create neighbourhood comm (owner -> ghost) + MPI_Comm comm1; + int ierr = MPI_Dist_graph_create_adjacent( + comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, + submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, + false, &comm1); + dolfinx::MPI::check_error(comm, ierr); - ierr = MPI_Comm_free(&comm1); - dolfinx::MPI::check_error(comm, ierr); + // Send indices to ghosting ranks + ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), + recv_disp.data(), MPI_INT64_T, + recv_gidx.data(), send_sizes.data(), + send_disp.data(), MPI_INT64_T, comm1); + dolfinx::MPI::check_error(comm, ierr); + + ierr = MPI_Comm_free(&comm1); + dolfinx::MPI::check_error(comm, ierr); + } // --- Step 4---: Unpack received data From d3f52d49b810ea6e27aaa1eb2cf808ba8b3ef160 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Wed, 15 Nov 2023 16:29:50 +0000 Subject: [PATCH 048/101] Rename comms --- cpp/dolfinx/common/IndexMap.cpp | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c8c9aa9a733..143b814c8c4 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -66,10 +66,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector send_sizes, recv_sizes; { // Create neighbourhood comm (ghost -> owner) - MPI_Comm comm0; + MPI_Comm reverse_comm; int ierr = MPI_Dist_graph_create_adjacent( comm, dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), src.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &reverse_comm); dolfinx::MPI::check_error(comm, ierr); // Pack ghost indices that this process wants to include in the @@ -98,7 +98,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_sizes.reserve(1); recv_sizes.reserve(1); ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); + recv_sizes.data(), 1, MPI_INT32_T, reverse_comm); dolfinx::MPI::check_error(comm, ierr); // Prepare displacement vectors @@ -118,11 +118,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, comm0); + recv_disp.data(), MPI_INT64_T, reverse_comm); dolfinx::MPI::check_error(comm, ierr); // Free the communicator - ierr = MPI_Comm_free(&comm0); + ierr = MPI_Comm_free(&reverse_comm); dolfinx::MPI::check_error(comm, ierr); } @@ -188,21 +188,21 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } // Create neighbourhood comm (owner -> ghost) - MPI_Comm comm1; + MPI_Comm forward_comm; int ierr = MPI_Dist_graph_create_adjacent( comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &forward_comm); dolfinx::MPI::check_error(comm, ierr); // Send the data ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), recv_disp.data(), MPI_INT32_T, recv_owners.data(), send_sizes.data(), - send_disp.data(), MPI_INT32_T, comm1); + send_disp.data(), MPI_INT32_T, forward_comm); dolfinx::MPI::check_error(comm, ierr); // Free the communicator - ierr = MPI_Comm_free(&comm1); + ierr = MPI_Comm_free(&forward_comm); dolfinx::MPI::check_error(comm, ierr); } @@ -284,11 +284,11 @@ std::vector compute_submap_ghost_indices( std::vector send_sizes, recv_sizes; { // Create neighbourhood comm (ghost -> owner) - MPI_Comm comm0; + MPI_Comm reverse_comm; int ierr = MPI_Dist_graph_create_adjacent( comm, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &comm0); + false, &reverse_comm); dolfinx::MPI::check_error(comm, ierr); // Pack ghosts indices @@ -322,7 +322,7 @@ std::vector compute_submap_ghost_indices( send_sizes.reserve(1); recv_sizes.reserve(1); ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); + recv_sizes.data(), 1, MPI_INT32_T, reverse_comm); dolfinx::MPI::check_error(comm, ierr); // Prepare displacement vectors @@ -339,10 +339,10 @@ std::vector compute_submap_ghost_indices( ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, comm0); + recv_disp.data(), MPI_INT64_T, reverse_comm); dolfinx::MPI::check_error(comm, ierr); - ierr = MPI_Comm_free(&comm0); + ierr = MPI_Comm_free(&reverse_comm); dolfinx::MPI::check_error(comm, ierr); } @@ -374,21 +374,21 @@ std::vector compute_submap_ghost_indices( std::vector recv_gidx(send_disp.back()); { // Create neighbourhood comm (owner -> ghost) - MPI_Comm comm1; + MPI_Comm forward_comm; int ierr = MPI_Dist_graph_create_adjacent( comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &comm1); + false, &forward_comm); dolfinx::MPI::check_error(comm, ierr); // Send indices to ghosting ranks ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), recv_disp.data(), MPI_INT64_T, recv_gidx.data(), send_sizes.data(), - send_disp.data(), MPI_INT64_T, comm1); + send_disp.data(), MPI_INT64_T, forward_comm); dolfinx::MPI::check_error(comm, ierr); - ierr = MPI_Comm_free(&comm1); + ierr = MPI_Comm_free(&forward_comm); dolfinx::MPI::check_error(comm, ierr); } From 532ee95eb770d1977a88cbb792e7ce78c89abef7 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 12:05:42 +0000 Subject: [PATCH 049/101] comm names --- cpp/dolfinx/common/IndexMap.cpp | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 143b814c8c4..c8c9aa9a733 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -66,10 +66,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector send_sizes, recv_sizes; { // Create neighbourhood comm (ghost -> owner) - MPI_Comm reverse_comm; + MPI_Comm comm0; int ierr = MPI_Dist_graph_create_adjacent( comm, dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), src.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &reverse_comm); + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); dolfinx::MPI::check_error(comm, ierr); // Pack ghost indices that this process wants to include in the @@ -98,7 +98,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_sizes.reserve(1); recv_sizes.reserve(1); ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, reverse_comm); + recv_sizes.data(), 1, MPI_INT32_T, comm0); dolfinx::MPI::check_error(comm, ierr); // Prepare displacement vectors @@ -118,11 +118,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, reverse_comm); + recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); // Free the communicator - ierr = MPI_Comm_free(&reverse_comm); + ierr = MPI_Comm_free(&comm0); dolfinx::MPI::check_error(comm, ierr); } @@ -188,21 +188,21 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } // Create neighbourhood comm (owner -> ghost) - MPI_Comm forward_comm; + MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &forward_comm); + MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); dolfinx::MPI::check_error(comm, ierr); // Send the data ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), recv_disp.data(), MPI_INT32_T, recv_owners.data(), send_sizes.data(), - send_disp.data(), MPI_INT32_T, forward_comm); + send_disp.data(), MPI_INT32_T, comm1); dolfinx::MPI::check_error(comm, ierr); // Free the communicator - ierr = MPI_Comm_free(&forward_comm); + ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); } @@ -284,11 +284,11 @@ std::vector compute_submap_ghost_indices( std::vector send_sizes, recv_sizes; { // Create neighbourhood comm (ghost -> owner) - MPI_Comm reverse_comm; + MPI_Comm comm0; int ierr = MPI_Dist_graph_create_adjacent( comm, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &reverse_comm); + false, &comm0); dolfinx::MPI::check_error(comm, ierr); // Pack ghosts indices @@ -322,7 +322,7 @@ std::vector compute_submap_ghost_indices( send_sizes.reserve(1); recv_sizes.reserve(1); ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, reverse_comm); + recv_sizes.data(), 1, MPI_INT32_T, comm0); dolfinx::MPI::check_error(comm, ierr); // Prepare displacement vectors @@ -339,10 +339,10 @@ std::vector compute_submap_ghost_indices( ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, reverse_comm); + recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); - ierr = MPI_Comm_free(&reverse_comm); + ierr = MPI_Comm_free(&comm0); dolfinx::MPI::check_error(comm, ierr); } @@ -374,21 +374,21 @@ std::vector compute_submap_ghost_indices( std::vector recv_gidx(send_disp.back()); { // Create neighbourhood comm (owner -> ghost) - MPI_Comm forward_comm; + MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &forward_comm); + false, &comm1); dolfinx::MPI::check_error(comm, ierr); // Send indices to ghosting ranks ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), recv_disp.data(), MPI_INT64_T, recv_gidx.data(), send_sizes.data(), - send_disp.data(), MPI_INT64_T, forward_comm); + send_disp.data(), MPI_INT64_T, comm1); dolfinx::MPI::check_error(comm, ierr); - ierr = MPI_Comm_free(&forward_comm); + ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); } From 3dbcb97178dcfa073dd978bc15129f5a98cd553f Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 14:13:32 +0000 Subject: [PATCH 050/101] Make free function --- cpp/dolfinx/common/IndexMap.cpp | 107 +++++++++++----------- cpp/dolfinx/common/IndexMap.h | 32 +++---- cpp/dolfinx/mesh/Geometry.h | 2 +- cpp/dolfinx/mesh/Topology.cpp | 4 +- python/dolfinx/wrappers/common.cpp | 30 +++--- python/test/unit/common/test_index_map.py | 4 +- 6 files changed, 92 insertions(+), 87 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c8c9aa9a733..35e1760565a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -681,6 +681,61 @@ common::stack_index_maps( return {process_offset, std::move(local_offset), std::move(ghosts_new), std::move(ghost_owners_new)}; } +//----------------------------------------------------------------------------- +std::pair> +common::create_submap_conn(const dolfinx::common::IndexMap& imap, + std::span indices) +{ + const MPI_Comm comm = imap.comm(); + + // Compute the owned, ghost, and ghost owners of submap indices. + // NOTE: All indices are local and numbered w.r.t. the original (imap) index + // map + auto [submap_owned, submap_ghost, submap_ghost_owners] + = compute_submap_indices(imap, indices); + + // Compute submap offset for this rank + std::int64_t submap_local_size = submap_owned.size(); + std::int64_t submap_offset = 0; + int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, + MPI_SUM, comm); + dolfinx::MPI::check_error(comm, ierr); + + // Get submap source ranks + std::vector submap_src(submap_ghost_owners.begin(), + submap_ghost_owners.end()); + std::sort(submap_src.begin(), submap_src.end()); + submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), + submap_src.end()); + submap_src.shrink_to_fit(); + + // Compute submap destination ranks + // NOTE: The call to NBX can be avoided by a two-step communication process + // TODO: Can NBX call be avoided by using O^r_p and O^g_p? + std::vector submap_dest + = dolfinx::MPI::compute_graph_edges_nbx(comm, submap_src); + std::sort(submap_dest.begin(), submap_dest.end()); + + // Compute the global indices (w.r.t. the submap) of the submap ghosts + std::vector submap_ghost_global(submap_ghost.size()); + imap.local_to_global(submap_ghost, submap_ghost_global); + auto submap_ghost_gidxs = compute_submap_ghost_indices( + submap_src, submap_dest, submap_owned, submap_ghost_global, + submap_ghost_owners, submap_offset, imap); + + // Create a map from (local) indices in the submap to the corresponding + // (local) index in the original map + std::vector sub_imap_to_imap; + sub_imap_to_imap.reserve(submap_owned.size() + submap_ghost.size()); + sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_owned.begin(), + submap_owned.end()); + sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), + submap_ghost.end()); + + return {IndexMap(comm, submap_local_size, submap_ghost_gidxs, + submap_ghost_owners), + std::move(sub_imap_to_imap)}; +} //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -1004,58 +1059,6 @@ IndexMap::create_submap(std::span indices) const } } //----------------------------------------------------------------------------- -std::pair> -IndexMap::create_submap_conn(std::span indices) const -{ - // Compute the owned, ghost, and ghost owners of submap indices. - // NOTE: All indices are local and numbered w.r.t. the original (this) index - // map - auto [submap_owned, submap_ghost, submap_ghost_owners] - = compute_submap_indices(*this, indices); - - // Compute submap offset for this rank - std::int64_t submap_local_size = submap_owned.size(); - std::int64_t submap_offset = 0; - int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, - MPI_SUM, _comm.comm()); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // Get submap source ranks - std::vector submap_src(submap_ghost_owners.begin(), - submap_ghost_owners.end()); - std::sort(submap_src.begin(), submap_src.end()); - submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), - submap_src.end()); - submap_src.shrink_to_fit(); - - // Compute submap destination ranks - // NOTE: The call to NBX can be avoided by a two-step communication process - // TODO: Can NBX call be avoided by using O^r_p and O^g_p? - std::vector submap_dest - = dolfinx::MPI::compute_graph_edges_nbx(_comm.comm(), submap_src); - std::sort(submap_dest.begin(), submap_dest.end()); - - // Compute the global indices (w.r.t. the submap) of the submap ghosts - std::vector submap_ghost_global(submap_ghost.size()); - this->local_to_global(submap_ghost, submap_ghost_global); - auto submap_ghost_gidxs = compute_submap_ghost_indices( - submap_src, submap_dest, submap_owned, submap_ghost_global, - submap_ghost_owners, submap_offset, *this); - - // Create a map from (local) indices in the submap to the corresponding - // (local) index in the original map - std::vector sub_imap_to_imap; - sub_imap_to_imap.reserve(submap_owned.size() + submap_ghost.size()); - sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_owned.begin(), - submap_owned.end()); - sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), - submap_ghost.end()); - - return {IndexMap(_comm.comm(), submap_local_size, submap_ghost_gidxs, - submap_ghost_owners), - std::move(sub_imap_to_imap)}; -} -//----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const { const std::int64_t offset = _local_range[0]; diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 6a12d582de7..502ff8f95b1 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -54,6 +54,22 @@ stack_index_maps( const std::vector< std::pair, int>>& maps); +/// @brief Create a new index map from a subset of indices in this index +/// map. Any indices that are not included by their owning process, but +/// are included on sharing processes, will be owned by one of the sharing +/// processes in the submap. +/// +/// This can be used when, for instance, creating a submesh to ensure +/// that all vertices have exactly one owner and are connected to at least +/// one cell on the owning process. +/// @param[in] indices Local indices to include in the new index map (owned +/// and ghost) +/// @pre `indices` must be sorted and contain no duplicates. +/// @return The (i) new index map and (ii) a map from local indices in the +/// submap to local indices in the original (this) map +std::pair> +create_submap_conn(const IndexMap& imap, std::span indices); + /// This class represents the distribution index arrays across /// processes. An index array is a contiguous collection of N+1 indices /// [0, 1, . . ., N] that are distributed across M processes. On a given @@ -189,22 +205,6 @@ class IndexMap std::pair> create_submap(std::span indices) const; - /// @brief Create a new index map from a subset of indices in this index - /// map. Any indices that are not included by their owning process, but - /// are included on sharing processes, will be owned by one of the sharing - /// processes in the submap. - /// - /// This can be used when, for instance, creating a submesh to ensure - /// that all vertices have exactly one owner and are connected to at least - /// one cell on the owning process. - /// @param[in] indices Local indices to include in the new index map (owned - /// and ghost) - /// @pre `indices` must be sorted and contain no duplicates. - /// @return The (i) new index map and (ii) a map from local indices in the - /// submap to local indices in the original (this) map - std::pair> - create_submap_conn(std::span indices) const; - /// @todo Aim to remove this function? /// /// @brief Compute map from each local (owned) index to the set of diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index 43bb7878d8c..6d09217aedf 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -333,7 +333,7 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, std::vector subx_to_x_dofmap; { std::pair> map_data - = x_index_map->create_submap_conn(sub_x_dofs); + = dolfinx::common::create_submap_conn(*x_index_map, sub_x_dofs); sub_x_dof_index_map = std::make_shared(std::move(map_data.first)); subx_to_x_dofmap = std::move(map_data.second); diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index 0ef376a2eb8..d2840b1190e 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1202,8 +1202,8 @@ mesh::create_subtopology(const Topology& topology, int dim, std::vector subvertices0; { std::pair> map_data - = map0->create_submap_conn( - compute_incident_entities(topology, subentities, dim, 0)); + = dolfinx::common::create_submap_conn( + *map0, compute_incident_entities(topology, subentities, dim, 0)); submap0 = std::make_shared(std::move(map_data.first)); subvertices0 = std::move(map_data.second); } diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 43d88a94968..021084adaec 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -137,25 +137,13 @@ void common(nb::module_& m) .def( "create_submap", [](const dolfinx::common::IndexMap& self, - nb::ndarray, nb::c_contig> - indices) + nb::ndarray, nb::c_contig> indices) { - auto [map, ghosts] = self.create_submap( - std::span(indices.data(), indices.size())); + auto [map, ghosts] + = self.create_submap(std::span(indices.data(), indices.size())); return std::pair(std::move(map), dolfinx_wrappers::as_nbarray(std::move(ghosts))); }, - nb::arg("indices")) - .def( - "create_submap_conn", - [](const dolfinx::common::IndexMap& self, - nb::ndarray, nb::c_contig> indices) - { - auto [map, submap_to_map] = self.create_submap_conn( - std::span(indices.data(), indices.size())); - return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( - std::move(submap_to_map))); - }, nb::arg("indices")); // dolfinx::common::Timer @@ -195,5 +183,17 @@ void common(nb::module_& m) dolfinx::init_logging(args.size(), argv.data()); }, nb::arg("args")); + + m.def( + "create_submap_conn", + [](const dolfinx::common::IndexMap& imap, + nb::ndarray, nb::c_contig> indices) + { + auto [map, submap_to_map] = dolfinx::common::create_submap_conn( + imap, std::span(indices.data(), indices.size())); + return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( + std::move(submap_to_map))); + }, + nb::arg("index_map"), nb::arg("indices")); } } // namespace dolfinx_wrappers diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 392c2897b93..367ff9c5396 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -12,6 +12,7 @@ import dolfinx from dolfinx.mesh import GhostMode, create_unit_square +from dolfinx import cpp as _cpp def test_sub_index_map(): @@ -161,7 +162,8 @@ def test_create_submap_connected(): submap_indices = np.array([0, 2, 3], dtype=np.int32) imap = dolfinx.common.IndexMap(comm, local_size, ghosts, owners) - sub_imap, sub_imap_to_imap = imap.create_submap_conn(submap_indices) + sub_imap, sub_imap_to_imap = _cpp.common.create_submap_conn( + imap, submap_indices) if comm.rank == 0: assert sub_imap.size_local == 2 From ed4bb6e5124c66bb1a3060b91590b09aaa758bb4 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 15:07:27 +0000 Subject: [PATCH 051/101] Update docs --- cpp/dolfinx/common/IndexMap.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 502ff8f95b1..b01aff8a4f6 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -54,7 +54,7 @@ stack_index_maps( const std::vector< std::pair, int>>& maps); -/// @brief Create a new index map from a subset of indices in this index +/// @brief Create a new index map from a subset of indices in an index /// map. Any indices that are not included by their owning process, but /// are included on sharing processes, will be owned by one of the sharing /// processes in the submap. @@ -62,6 +62,7 @@ stack_index_maps( /// This can be used when, for instance, creating a submesh to ensure /// that all vertices have exactly one owner and are connected to at least /// one cell on the owning process. +/// @param[in] imap The index map /// @param[in] indices Local indices to include in the new index map (owned /// and ghost) /// @pre `indices` must be sorted and contain no duplicates. From 3c9df33051c2ded9ecb20b04f623d0bf61be5c89 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 16:38:06 +0000 Subject: [PATCH 052/101] Extract function --- cpp/dolfinx/common/IndexMap.cpp | 106 +++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 35e1760565a..ca7db9f9d9f 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -33,37 +33,26 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } -/// Given an index map and a subset of local indices (can be owned or ghost but -/// must be unique and sorted), computes the owned, ghost and ghost owners in -/// the submap. -/// @param[in] imap An index map. -/// @param[in] indices List of entity indices (indices local to the process). -/// @pre `indices` must be sorted and unique. -/// @return The owned, ghost, and ghost owners in the submap. All indices are -/// local and with respect to the original index map. -std::tuple, std::vector, - std::vector> -compute_submap_indices(const dolfinx::common::IndexMap& imap, - std::span indices) +std::tuple, std::vector, + std::vector, std::vector, + std::vector, std::vector, std::vector> +communicate_ghosts(const dolfinx::common::IndexMap& imap, + std::span include_ghost) { + // Send ghost indices to owning rank + + // Get source ranks (ranks that own ghosts) and destination ranks + // (ranks that ghost my indices) const MPI_Comm comm = imap.comm(); std::span src = imap.src(); std::span dest = imap.dest(); std::span ghosts = imap.ghosts(); - std::span ghost_owners = imap.owners(); - - // Lookup array to determine if an index is in the sub-map - std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); - std::for_each(indices.begin(), indices.end(), - [&is_in_submap](int i) { is_in_submap[i] = 1; }); - - // --- Step 1 ---: Send ghost indices in `indices` to their owners - // and receive indices owned by this process that are in `indices` - // on other processes + std::span owners = imap.owners(); - std::vector send_disp, recv_disp; std::vector send_indices, recv_indices; + std::vector ghost_buffer_pos; std::vector send_sizes, recv_sizes; + std::vector send_disp, recv_disp; { // Create neighbourhood comm (ghost -> owner) MPI_Comm comm0; @@ -72,26 +61,37 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); dolfinx::MPI::check_error(comm, ierr); - // Pack ghost indices that this process wants to include in the - // sub-map + // Pack ghosts indices std::vector> send_data(src.size()); + std::vector> pos_to_ghost(src.size()); for (std::size_t i = 0; i < ghosts.size(); ++i) { - auto it = std::lower_bound(src.begin(), src.end(), ghost_owners[i]); - assert(it != src.end() and *it == ghost_owners[i]); - std::size_t r = std::distance(src.begin(), it); - // Send ghost index if it is in the submap, else send -1 - if (is_in_submap[imap.size_local() + i]) + auto it = std::lower_bound(src.begin(), src.end(), owners[i]); + assert(it != src.end() and *it == owners[i]); + int r = std::distance(src.begin(), it); + if (include_ghost[i]) + { send_data[r].push_back(ghosts[i]); + pos_to_ghost[r].push_back(i); + } else + { send_data[r].push_back(-1); + pos_to_ghost[r].push_back(-1); + } } - // Count number of indices to send per dest + // Count number of ghosts per dest std::transform(send_data.begin(), send_data.end(), std::back_inserter(send_sizes), [](auto& d) { return d.size(); }); + // Build send buffer and ghost position to send buffer position + for (auto& d : send_data) + send_indices.insert(send_indices.end(), d.begin(), d.end()); + for (auto& p : pos_to_ghost) + ghost_buffer_pos.insert(ghost_buffer_pos.end(), p.begin(), p.end()); + // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost recv_sizes.resize(dest.size(), 0); @@ -109,11 +109,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); - // Send ghost indices to owner, and receive indices owned by this process - // that are ghosts on others - for (auto& d : send_data) - send_indices.insert(send_indices.end(), d.begin(), d.end()); - + // Send ghost indices to owner, and receive indices recv_indices.resize(recv_disp.back()); ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, @@ -121,11 +117,47 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, recv_disp.data(), MPI_INT64_T, comm0); dolfinx::MPI::check_error(comm, ierr); - // Free the communicator ierr = MPI_Comm_free(&comm0); dolfinx::MPI::check_error(comm, ierr); } + return {std::move(send_indices), std::move(recv_indices), + std::move(ghost_buffer_pos), std::move(send_sizes), + std::move(recv_sizes), std::move(send_disp), + std::move(recv_disp)}; +} + +/// Given an index map and a subset of local indices (can be owned or ghost but +/// must be unique and sorted), computes the owned, ghost and ghost owners in +/// the submap. +/// @param[in] imap An index map. +/// @param[in] indices List of entity indices (indices local to the process). +/// @pre `indices` must be sorted and unique. +/// @return The owned, ghost, and ghost owners in the submap. All indices are +/// local and with respect to the original index map. +std::tuple, std::vector, + std::vector> +compute_submap_indices(const dolfinx::common::IndexMap& imap, + std::span indices) +{ + const MPI_Comm comm = imap.comm(); + std::span src = imap.src(); + std::span dest = imap.dest(); + + // Lookup array to determine if an index is in the sub-map + std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); + std::for_each(indices.begin(), indices.end(), + [&is_in_submap](int i) { is_in_submap[i] = 1; }); + + // --- Step 1 ---: Send ghost indices in `indices` to their owners + // and receive indices owned by this process that are in `indices` + // on other processes + auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, recv_sizes, + send_disp, recv_disp] + = communicate_ghosts( + imap, std::span(is_in_submap.cbegin() + imap.size_local(), + is_in_submap.cend())); + // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. // indices owned by this process that are in `indices` on other processes) to // their owner in the submap. This is required since not all indices in From 287e7357132ea029333bb72f8bd32d9203686f1f Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 17:34:45 +0000 Subject: [PATCH 053/101] Use function --- cpp/dolfinx/common/IndexMap.cpp | 96 ++++++--------------------------- 1 file changed, 16 insertions(+), 80 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index ca7db9f9d9f..005eab896df 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -36,19 +36,14 @@ std::array, 2> build_src_dest(MPI_Comm comm, std::tuple, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector> -communicate_ghosts(const dolfinx::common::IndexMap& imap, - std::span include_ghost) +communicate_ghosts_to_owners(const MPI_Comm comm, + std::span src, + std::span dest, + std::span ghosts, + std::span owners, + std::span include_ghost) { // Send ghost indices to owning rank - - // Get source ranks (ranks that own ghosts) and destination ranks - // (ranks that ghost my indices) - const MPI_Comm comm = imap.comm(); - std::span src = imap.src(); - std::span dest = imap.dest(); - std::span ghosts = imap.ghosts(); - std::span owners = imap.owners(); - std::vector send_indices, recv_indices; std::vector ghost_buffer_pos; std::vector send_sizes, recv_sizes; @@ -154,9 +149,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // on other processes auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, recv_sizes, send_disp, recv_disp] - = communicate_ghosts( - imap, std::span(is_in_submap.cbegin() + imap.size_local(), - is_in_submap.cend())); + = communicate_ghosts_to_owners( + imap.comm(), imap.src(), imap.dest(), imap.ghosts(), imap.owners(), + std::span(is_in_submap.cbegin() + imap.size_local(), + is_in_submap.cend())); // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. // indices owned by this process that are in `indices` on other processes) to @@ -310,73 +306,13 @@ std::vector compute_submap_ghost_indices( // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning // rank const MPI_Comm comm = imap.comm(); - std::vector recv_indices; - std::vector ghost_perm; - std::vector send_disp, recv_disp; - std::vector send_sizes, recv_sizes; - { - // Create neighbourhood comm (ghost -> owner) - MPI_Comm comm0; - int ierr = MPI_Dist_graph_create_adjacent( - comm, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, - submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, - false, &comm0); - dolfinx::MPI::check_error(comm, ierr); - - // Pack ghosts indices - std::vector> send_data(submap_src.size()); - std::vector> ghost_perm_dict(submap_src.size()); - for (std::size_t i = 0; i < submap_ghosts_global.size(); ++i) - { - auto it = std::lower_bound(submap_src.begin(), submap_src.end(), - submap_ghost_owners[i]); - assert(it != submap_src.end() and *it == submap_ghost_owners[i]); - std::size_t r = std::distance(submap_src.begin(), it); - send_data[r].push_back(submap_ghosts_global[i]); - ghost_perm_dict[r].push_back(i); - } - - // Count number of ghosts per dest - std::transform(send_data.begin(), send_data.end(), - std::back_inserter(send_sizes), - [](auto& d) { return d.size(); }); - - // Build send buffer and ghost position to send buffer position - std::vector send_indices; - for (auto& d : send_data) - send_indices.insert(send_indices.end(), d.begin(), d.end()); - for (auto& p : ghost_perm_dict) - ghost_perm.insert(ghost_perm.end(), p.begin(), p.end()); - - // Send how many indices I ghost to each owner, and receive how many - // of my indices other ranks ghost - recv_sizes.resize(submap_dest.size(), 0); - send_sizes.reserve(1); - recv_sizes.reserve(1); - ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); - dolfinx::MPI::check_error(comm, ierr); - // Prepare displacement vectors - send_disp.resize(submap_src.size() + 1, 0); - recv_disp.resize(submap_dest.size() + 1, 0); - std::partial_sum(send_sizes.begin(), send_sizes.end(), - std::next(send_disp.begin())); - std::partial_sum(recv_sizes.begin(), recv_sizes.end(), - std::next(recv_disp.begin())); - - // Send ghost indices to owner, and receive indices owned by this process - // that are shared with others in the submap - recv_indices.resize(recv_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), - send_disp.data(), MPI_INT64_T, - recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, comm0); - dolfinx::MPI::check_error(comm, ierr); - - ierr = MPI_Comm_free(&comm0); - dolfinx::MPI::check_error(comm, ierr); - } + auto [send_indices, recv_indices, ghost_perm, send_sizes, recv_sizes, + send_disp, recv_disp] + = communicate_ghosts_to_owners( + imap.comm(), submap_src, submap_dest, submap_ghosts_global, + submap_ghost_owners, + std::vector(submap_ghosts_global.size(), 1)); // --- Step 2 ---: For each received index, compute the submap global index From 104bf2279c31d09929ff5f65346cfa6c58b80350 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 17 Nov 2023 17:52:45 +0000 Subject: [PATCH 054/101] Use function --- cpp/dolfinx/common/IndexMap.cpp | 76 +++------------------------------ 1 file changed, 5 insertions(+), 71 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 005eab896df..c3ae785bccc 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -875,79 +875,13 @@ IndexMap::create_submap(std::span indices) const dolfinx::MPI::check_error(_comm.comm(), ierr); // --- Step 2: Send ghost indices to owning rank - - // Get source ranks (ranks that own ghosts) and destination ranks - // (ranks that ghost my indices) const std::vector& src = this->src(); const std::vector& dest = this->dest(); - - std::vector recv_indices; - std::vector ghost_buffer_pos; - std::vector send_disp, recv_disp; - std::vector send_sizes, recv_sizes; - { - // Create neighbourhood comm (ghost -> owner) - MPI_Comm comm0; - int ierr = MPI_Dist_graph_create_adjacent( - _comm.comm(), dest.size(), dest.data(), MPI_UNWEIGHTED, src.size(), - src.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm0); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // Pack ghosts indices - std::vector> send_data(src.size()); - std::vector> pos_to_ghost(src.size()); - for (std::size_t i = 0; i < _ghosts.size(); ++i) - { - auto it = std::lower_bound(src.begin(), src.end(), _owners[i]); - assert(it != src.end() and *it == _owners[i]); - int r = std::distance(src.begin(), it); - send_data[r].push_back(_ghosts[i]); - pos_to_ghost[r].push_back(i); - } - - // Count number of ghosts per dest - std::transform(send_data.begin(), send_data.end(), - std::back_inserter(send_sizes), - [](auto& d) { return d.size(); }); - - // Build send buffer and ghost position to send buffer position - std::vector send_indices; - for (auto& d : send_data) - send_indices.insert(send_indices.end(), d.begin(), d.end()); - for (auto& p : pos_to_ghost) - ghost_buffer_pos.insert(ghost_buffer_pos.end(), p.begin(), p.end()); - - // Send how many indices I ghost to each owner, and receive how many - // of my indices other ranks ghost - recv_sizes.resize(dest.size(), 0); - send_sizes.reserve(1); - recv_sizes.reserve(1); - ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // Prepare displacement vectors - send_disp.resize(src.size() + 1, 0); - recv_disp.resize(dest.size() + 1, 0); - std::partial_sum(send_sizes.begin(), send_sizes.end(), - std::next(send_disp.begin())); - std::partial_sum(recv_sizes.begin(), recv_sizes.end(), - std::next(recv_disp.begin())); - - // Send ghost indices to owner, and receive indices - recv_indices.resize(recv_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_indices.data(), send_sizes.data(), - send_disp.data(), MPI_INT64_T, - recv_indices.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, comm0); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - ierr = MPI_Comm_free(&comm0); - dolfinx::MPI::check_error(_comm.comm(), ierr); - } - - ierr = MPI_Wait(&request_offset, MPI_STATUS_IGNORE); - dolfinx::MPI::check_error(_comm.comm(), ierr); + auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, recv_sizes, + send_disp, recv_disp] + = communicate_ghosts_to_owners( + this->comm(), src, dest, this->ghosts(), this->owners(), + std::vector(this->ghosts().size(), 1)); // --- Step 3: Check which received indexes (all of which I should // own) are in the submap From 76a80775e8d141423fb13c7458c0eb74e836074d Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Mon, 20 Nov 2023 15:41:51 +0000 Subject: [PATCH 055/101] Start work on replacing old version --- cpp/dolfinx/mesh/Topology.cpp | 26 +++++++++-------------- python/test/unit/common/test_index_map.py | 23 ++++++++++---------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index d2840b1190e..2d558e9f446 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1166,26 +1166,20 @@ mesh::create_subtopology(const Topology& topology, int dim, // Create a map from an entity in the sub-topology to the // corresponding entity in the topology, and create an index map - std::vector subentities; std::shared_ptr submap; + std::vector subentities; { - // Entities in the sub-topology that are owned by this process - auto entity_map = topology.index_map(dim); - assert(entity_map); - std::copy_if( - entities.begin(), entities.end(), std::back_inserter(subentities), - [size = entity_map->size_local()](std::int32_t e) { return e < size; }); - + // FIXME Make this an input requirement? + std::vector unique_entities(entities.begin(), entities.end()); + std::sort(unique_entities.begin(), unique_entities.end()); + unique_entities.erase( + std::unique(unique_entities.begin(), unique_entities.end()), + unique_entities.end()); std::pair> map_data - = entity_map->create_submap(subentities); + = dolfinx::common::create_submap_conn(*topology.index_map(dim), + unique_entities); submap = std::make_shared(std::move(map_data.first)); - - // Add ghost entities to subentities - subentities.reserve(submap->size_local() + submap->num_ghosts()); - std::transform(map_data.second.begin(), map_data.second.end(), - std::back_inserter(subentities), - [offset = entity_map->size_local()](auto entity_index) - { return offset + entity_index; }); + subentities = std::move(map_data.second); } // Get the vertices in the sub-topology. Use subentities diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 367ff9c5396..6e13092b48d 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -45,7 +45,8 @@ def test_sub_index_map(): # Create sub index map and a map from the ghost position in new map # to the position in old map - submap, ghosts_pos_sub = map.create_submap(local_indices[my_rank]) + submap, submap_to_map = _cpp.common.create_submap_conn(map, local_indices[my_rank]) + ghosts_pos_sub = submap_to_map[map_local_size:] - map_local_size # Check local and global sizes assert submap.size_local == submap_local_size[my_rank] @@ -65,15 +66,15 @@ def test_sub_index_map(): # Check that ghost indices are correct in submap # NOTE This assumes size_local is the same for all ranks # TODO Consider renaming to something shorter - submap_global_to_map_global_map = np.concatenate([local_indices[rank] + map_local_size * rank - for rank in range(comm.size)]) - # FIXME Do this more elegantly - submap_ghosts = [] - for map_ghost in map.ghosts: - submap_ghost = np.where(submap_global_to_map_global_map == map_ghost)[0] - if submap_ghost.size != 0: - submap_ghosts.append(submap_ghost[0]) - assert np.allclose(submap.ghosts, submap_ghosts) + # submap_global_to_map_global_map = np.concatenate([local_indices[rank] + map_local_size * rank + # for rank in range(comm.size)]) + # # FIXME Do this more elegantly + # submap_ghosts = [] + # for map_ghost in map.ghosts: + # submap_ghost = np.where(submap_global_to_map_global_map == map_ghost)[0] + # if submap_ghost.size != 0: + # submap_ghosts.append(submap_ghost[0]) + # assert np.allclose(submap.ghosts, submap_ghosts) def test_sub_index_map_ghost_mode_none(): @@ -82,7 +83,7 @@ def test_sub_index_map_ghost_mode_none(): tdim = mesh.topology.dim map = mesh.topology.index_map(tdim) submap_indices = np.arange(0, min(2, map.size_local), dtype=np.int32) - map.create_submap(submap_indices) + _cpp.common.create_submap_conn(map, submap_indices) def test_index_map_ghost_lifetime(): From eec7b96d0728d9dc55d9198724cf90a323492d26 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 23 Nov 2023 11:07:51 +0000 Subject: [PATCH 056/101] Start work on removing old version --- cpp/dolfinx/fem/DofMap.cpp | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index fda6ee4bd11..698979d50c2 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -20,6 +20,20 @@ using namespace dolfinx; using namespace dolfinx::fem; +// A function for printing vectors +template +std::ostream &operator<<(std::ostream &os, + const std::vector &vector) +{ + os << "{ "; + for (auto v : vector) + { + os << v << " "; + } + os << "}"; + return os; +} + namespace { //----------------------------------------------------------------------------- @@ -28,6 +42,12 @@ namespace fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, const mesh::Topology& topology) { + std::stringstream ss; + + const int rank = dolfinx::MPI::rank(topology.comm()); + + ss << "Rank " << rank << ":\n"; + if (dofmap_view.element_dof_layout().block_size() > 1) { throw std::runtime_error( @@ -50,6 +70,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, dofs_view.erase(std::unique(dofs_view.begin(), dofs_view.end()), dofs_view.end()); + ss << "dofs_view = " << dofs_view << "\n"; + // Compute sizes const std::int32_t num_owned_view = dofmap_view.index_map->size_local(); @@ -62,6 +84,9 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Create sub-index map std::shared_ptr index_map; std::vector ghost_new_to_old; + std::shared_ptr index_map_conn; + std::vector new_to_old_conn; + std::vector indices_conn; if (bs_view == 1) { std::span indices(dofs_view.data(), @@ -69,6 +94,16 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); index_map = std::make_shared(std::move(_index_map)); ghost_new_to_old = std::move(gmap); + + // ss << "indices = " << indices << "\n"; + + indices_conn = dofs_view; + auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( + *dofmap_view.index_map, indices_conn); + index_map_conn = std::make_shared(std::move(_index_map_conn)); + new_to_old_conn = std::move(gmap_conn); + + ss << "indices_conn = " << dofs_view << "\n"; } else { @@ -80,8 +115,22 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); index_map = std::make_shared(std::move(_index_map)); ghost_new_to_old = std::move(gmap); + + // ss << "indices = " << indices << "\n"; + indices_conn.reserve(dofs_view.size()); + std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices_conn), + [bs_view](auto idx) { return idx / bs_view; }); + indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); + auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( + *dofmap_view.index_map, indices_conn); + index_map_conn = std::make_shared(std::move(_index_map_conn)); + new_to_old_conn = std::move(gmap_conn); + + ss << "indices_conn = " << indices_conn << "\n"; } + ss << "new_to_old_conn = " << new_to_old_conn << "\n"; + // Create map from dof in view to new dof index std::vector old_to_new(dofs_view.back() + bs_view, -1); { @@ -106,6 +155,16 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } } + ss << "old_to_new = " << old_to_new << "\n"; + + std::vector old_to_new_conn(dofs_view.back() + 1, -1); + for (std::size_t i = 0; i < dofs_view.size(); ++i) + { + old_to_new_conn[dofs_view[i]] = new_to_old_conn[indices_conn[i]]; + } + + ss << "old_to_new_conn = " << old_to_new_conn << "\n"; + // Map dofs to new collapsed indices for new dofmap auto dof_array_view = dofmap_view.map(); std::vector dofmap; @@ -123,6 +182,15 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Copy dof layout, discarding parent data ElementDofLayout element_dof_layout = dofmap_view.element_dof_layout().copy(); + for (int r = 0; r < dolfinx::MPI::size(topology.comm()); ++r) + { + if (r == rank) + { + std::cout << ss.str() << "\n"; + } + MPI_Barrier(topology.comm()); + } + // Create new dofmap and return return DofMap(std::move(element_dof_layout), index_map, 1, std::move(dofmap), 1); From 9fd6f039ab2449cde815f05e4345331d1dff8628 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 23 Nov 2023 13:43:03 +0000 Subject: [PATCH 057/101] More work --- cpp/dolfinx/fem/DofMap.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 698979d50c2..b523d677d6c 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -75,6 +75,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Compute sizes const std::int32_t num_owned_view = dofmap_view.index_map->size_local(); + ss << "num_owned_view = " << num_owned_view << "\n"; + // Get block size int bs_view = dofmap_view.index_map_bs(); @@ -155,12 +157,17 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } } - ss << "old_to_new = " << old_to_new << "\n"; + ss << "old_to_new = " << old_to_new << "\n"; std::vector old_to_new_conn(dofs_view.back() + 1, -1); - for (std::size_t i = 0; i < dofs_view.size(); ++i) + // for (std::size_t i = 0; i < dofs_view.size(); ++i) + // { + // old_to_new_conn[dofs_view[i]] = new_to_old_conn[indices_conn[i]]; + // } + for (std::size_t i = 0; i < new_to_old_conn.size(); ++i) { - old_to_new_conn[dofs_view[i]] = new_to_old_conn[indices_conn[i]]; + // FIXME This needs to take block size into account + old_to_new_conn[new_to_old_conn[i]] = i; } ss << "old_to_new_conn = " << old_to_new_conn << "\n"; @@ -172,7 +179,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::transform(dof_array_view.data_handle(), dof_array_view.data_handle() + dof_array_view.size(), std::back_inserter(dofmap), - [&old_to_new](auto idx_old) { return old_to_new[idx_old]; }); + [&old_to_new_conn](auto idx_old) { return old_to_new_conn[idx_old]; }); // Dimension sanity checks assert((int)dofmap.size() @@ -192,7 +199,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } // Create new dofmap and return - return DofMap(std::move(element_dof_layout), index_map, 1, std::move(dofmap), + return DofMap(std::move(element_dof_layout), index_map_conn, 1, std::move(dofmap), 1); } From c25a93117c67eaa669de68ee4f7fba798d960934 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 23 Nov 2023 13:50:17 +0000 Subject: [PATCH 058/101] Add todo --- cpp/dolfinx/fem/DofMap.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index b523d677d6c..89fb425b8df 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -87,8 +87,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::shared_ptr index_map; std::vector ghost_new_to_old; std::shared_ptr index_map_conn; - std::vector new_to_old_conn; - std::vector indices_conn; + std::vector new_to_old_conn; // TODO Rename as this is non-block index + std::vector indices_conn; if (bs_view == 1) { std::span indices(dofs_view.data(), From a284e701dad928c994d3c48a8d2c3a308e7d9963 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Mon, 4 Dec 2023 12:27:23 +0000 Subject: [PATCH 059/101] Comment some things --- cpp/dolfinx/common/IndexMap.cpp | 3 +++ cpp/dolfinx/fem/DofMap.cpp | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c3ae785bccc..c42e4a74a40 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -189,7 +189,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, if (is_in_submap[idx_local]) global_idx_to_possible_owner[idx].push_back(rank); else + { + // throw std::runtime_error("Index owner change detected!"); global_idx_to_possible_owner[idx].push_back(dest[i]); + } } } } diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 89fb425b8df..59f3eb553a9 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -166,8 +166,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // } for (std::size_t i = 0; i < new_to_old_conn.size(); ++i) { - // FIXME This needs to take block size into account - old_to_new_conn[new_to_old_conn[i]] = i; + // FIXME I'm not sure mapping with dofs_view is correct here + old_to_new_conn[dofs_view[new_to_old_conn[i]]] = i; } ss << "old_to_new_conn = " << old_to_new_conn << "\n"; From da0884a3fd1df5d0b94a025a67f9507709f6b261 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 12:28:17 +0000 Subject: [PATCH 060/101] Fix for bs > 1 --- cpp/dolfinx/fem/DofMap.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 59f3eb553a9..63fe0bbaafa 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -122,7 +122,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, indices_conn.reserve(dofs_view.size()); std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices_conn), [bs_view](auto idx) { return idx / bs_view; }); - indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); + // indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, indices_conn); index_map_conn = std::make_shared(std::move(_index_map_conn)); @@ -164,10 +164,21 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // { // old_to_new_conn[dofs_view[i]] = new_to_old_conn[indices_conn[i]]; // } - for (std::size_t i = 0; i < new_to_old_conn.size(); ++i) + // for (std::size_t new_unblocked_idx = 0; new_unblocked_idx < new_to_old_conn.size(); ++new_unblocked_idx) + // { + // // FIXME I'm not sure mapping with dofs_view is correct here + // std::int32_t old_unblocked_idx = new_to_old_conn[new_unblocked_idx]; + + // for (int k = 0; k < bs_view; ++k) + // { + // std::int32_t old_blocked_idx = old_unblocked_idx * bs_view + k; + // old_to_new_conn[old_blocked_idx] = new_unblocked_idx; + // } + // } + + for (std::size_t i = 0; i < indices_conn.size(); ++i) { - // FIXME I'm not sure mapping with dofs_view is correct here - old_to_new_conn[dofs_view[new_to_old_conn[i]]] = i; + old_to_new_conn[dofs_view[i]] = indices_conn[i]; } ss << "old_to_new_conn = " << old_to_new_conn << "\n"; From ff5887132657ac9c5a550d189ab1041046093f7d Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 12:28:50 +0000 Subject: [PATCH 061/101] Tidy --- cpp/dolfinx/fem/DofMap.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 63fe0bbaafa..e1a78612f69 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -160,22 +160,6 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "old_to_new = " << old_to_new << "\n"; std::vector old_to_new_conn(dofs_view.back() + 1, -1); - // for (std::size_t i = 0; i < dofs_view.size(); ++i) - // { - // old_to_new_conn[dofs_view[i]] = new_to_old_conn[indices_conn[i]]; - // } - // for (std::size_t new_unblocked_idx = 0; new_unblocked_idx < new_to_old_conn.size(); ++new_unblocked_idx) - // { - // // FIXME I'm not sure mapping with dofs_view is correct here - // std::int32_t old_unblocked_idx = new_to_old_conn[new_unblocked_idx]; - - // for (int k = 0; k < bs_view; ++k) - // { - // std::int32_t old_blocked_idx = old_unblocked_idx * bs_view + k; - // old_to_new_conn[old_blocked_idx] = new_unblocked_idx; - // } - // } - for (std::size_t i = 0; i < indices_conn.size(); ++i) { old_to_new_conn[dofs_view[i]] = indices_conn[i]; From b0ac7742755adcbf92ca156448fa5003476eb206 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 12:35:55 +0000 Subject: [PATCH 062/101] Tidy --- cpp/dolfinx/fem/DofMap.cpp | 50 -------------------------------------- 1 file changed, 50 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index e1a78612f69..c5486d14504 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -80,25 +80,12 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Get block size int bs_view = dofmap_view.index_map_bs(); - const auto it = std::lower_bound(dofs_view.begin(), dofs_view.end(), - bs_view * num_owned_view); - // Create sub-index map - std::shared_ptr index_map; - std::vector ghost_new_to_old; std::shared_ptr index_map_conn; std::vector new_to_old_conn; // TODO Rename as this is non-block index std::vector indices_conn; if (bs_view == 1) { - std::span indices(dofs_view.data(), - std::distance(dofs_view.begin(), it)); - auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); - index_map = std::make_shared(std::move(_index_map)); - ghost_new_to_old = std::move(gmap); - - // ss << "indices = " << indices << "\n"; - indices_conn = dofs_view; auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, indices_conn); @@ -109,20 +96,9 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } else { - std::vector indices; - indices.reserve(dofs_view.size()); - std::transform(dofs_view.begin(), it, std::back_inserter(indices), - [bs_view](auto idx) { return idx / bs_view; }); - indices.erase(std::unique(indices.begin(), indices.end()), indices.end()); - auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); - index_map = std::make_shared(std::move(_index_map)); - ghost_new_to_old = std::move(gmap); - - // ss << "indices = " << indices << "\n"; indices_conn.reserve(dofs_view.size()); std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices_conn), [bs_view](auto idx) { return idx / bs_view; }); - // indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, indices_conn); index_map_conn = std::make_shared(std::move(_index_map_conn)); @@ -133,32 +109,6 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; - // Create map from dof in view to new dof index - std::vector old_to_new(dofs_view.back() + bs_view, -1); - { - // old-to-new map for owned dofs - std::int32_t count = 0; - for (auto dof_old = dofs_view.begin(); dof_old != it; ++dof_old) - old_to_new[*dof_old] = count++; - - // old-to-new map for ghost dofs - const std::int32_t local_size_new = index_map->size_local(); - for (auto itp_old = ghost_new_to_old.begin(); - itp_old != ghost_new_to_old.end(); ++itp_old) - { - std::int32_t map_pos_new - = local_size_new + std::distance(ghost_new_to_old.begin(), itp_old); - std::int32_t idx = bs_view * (num_owned_view + *itp_old); - for (int k = 0; k < bs_view; ++k) - { - assert(idx + k < (int)old_to_new.size()); - old_to_new[idx + k] = map_pos_new; - } - } - } - - ss << "old_to_new = " << old_to_new << "\n"; - std::vector old_to_new_conn(dofs_view.back() + 1, -1); for (std::size_t i = 0; i < indices_conn.size(); ++i) { From c0f4d7059d00be8767f3ec51999923509a050c1d Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 15:37:47 +0000 Subject: [PATCH 063/101] Use different approach --- cpp/dolfinx/fem/DofMap.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index c5486d14504..28462dee21d 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -110,11 +110,22 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; std::vector old_to_new_conn(dofs_view.back() + 1, -1); - for (std::size_t i = 0; i < indices_conn.size(); ++i) + for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { - old_to_new_conn[dofs_view[i]] = indices_conn[i]; + for (int k = 0; k < bs_view; ++k) + { + std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; + old_to_new_conn[old_idx] = new_idx; + } } + // FIXME + // std::vector old_to_new_conn(dofs_view.back() + 1, -1); + // for (std::size_t i = 0; i < indices_conn.size(); ++i) + // { + // old_to_new_conn[dofs_view[i]] = indices_conn[i]; + // } + ss << "old_to_new_conn = " << old_to_new_conn << "\n"; // Map dofs to new collapsed indices for new dofmap From bac7d036f6f16e71d34e967d8d37adeebb5a69d5 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 16:44:14 +0000 Subject: [PATCH 064/101] Revert "Tidy" This reverts commit b0ac7742755adcbf92ca156448fa5003476eb206. --- cpp/dolfinx/fem/DofMap.cpp | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 28462dee21d..29731198028 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -80,12 +80,25 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Get block size int bs_view = dofmap_view.index_map_bs(); + const auto it = std::lower_bound(dofs_view.begin(), dofs_view.end(), + bs_view * num_owned_view); + // Create sub-index map + std::shared_ptr index_map; + std::vector ghost_new_to_old; std::shared_ptr index_map_conn; std::vector new_to_old_conn; // TODO Rename as this is non-block index std::vector indices_conn; if (bs_view == 1) { + std::span indices(dofs_view.data(), + std::distance(dofs_view.begin(), it)); + auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); + index_map = std::make_shared(std::move(_index_map)); + ghost_new_to_old = std::move(gmap); + + // ss << "indices = " << indices << "\n"; + indices_conn = dofs_view; auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, indices_conn); @@ -96,9 +109,20 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } else { + std::vector indices; + indices.reserve(dofs_view.size()); + std::transform(dofs_view.begin(), it, std::back_inserter(indices), + [bs_view](auto idx) { return idx / bs_view; }); + indices.erase(std::unique(indices.begin(), indices.end()), indices.end()); + auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); + index_map = std::make_shared(std::move(_index_map)); + ghost_new_to_old = std::move(gmap); + + // ss << "indices = " << indices << "\n"; indices_conn.reserve(dofs_view.size()); std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices_conn), [bs_view](auto idx) { return idx / bs_view; }); + // indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, indices_conn); index_map_conn = std::make_shared(std::move(_index_map_conn)); @@ -109,6 +133,32 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; + // Create map from dof in view to new dof index + std::vector old_to_new(dofs_view.back() + bs_view, -1); + { + // old-to-new map for owned dofs + std::int32_t count = 0; + for (auto dof_old = dofs_view.begin(); dof_old != it; ++dof_old) + old_to_new[*dof_old] = count++; + + // old-to-new map for ghost dofs + const std::int32_t local_size_new = index_map->size_local(); + for (auto itp_old = ghost_new_to_old.begin(); + itp_old != ghost_new_to_old.end(); ++itp_old) + { + std::int32_t map_pos_new + = local_size_new + std::distance(ghost_new_to_old.begin(), itp_old); + std::int32_t idx = bs_view * (num_owned_view + *itp_old); + for (int k = 0; k < bs_view; ++k) + { + assert(idx + k < (int)old_to_new.size()); + old_to_new[idx + k] = map_pos_new; + } + } + } + + ss << "old_to_new = " << old_to_new << "\n"; + std::vector old_to_new_conn(dofs_view.back() + 1, -1); for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { From de5c7b32f1ac1d8ca857de42a19a9584a87c3e86 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 5 Dec 2023 18:04:04 +0000 Subject: [PATCH 065/101] Fix array size --- cpp/dolfinx/fem/DofMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 29731198028..993927a0532 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -159,7 +159,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "old_to_new = " << old_to_new << "\n"; - std::vector old_to_new_conn(dofs_view.back() + 1, -1); + std::vector old_to_new_conn(dofs_view.back() + bs_view, -1); for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { for (int k = 0; k < bs_view; ++k) From 4cc5dc3a7f22198e748323be436dfeb4769a74a3 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 7 Dec 2023 10:28:56 +0000 Subject: [PATCH 066/101] Try using map --- cpp/dolfinx/fem/DofMap.cpp | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 993927a0532..07a5a9aa60e 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -84,6 +84,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, bs_view * num_owned_view); // Create sub-index map + // TODO Simplify std::shared_ptr index_map; std::vector ghost_new_to_old; std::shared_ptr index_map_conn; @@ -133,6 +134,13 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; + // TODO Simplify + std::vector non_blocked_to_blocked(indices_conn.size(), -1); + for (std::size_t i = 0; i < indices_conn.size(); ++i) + { + non_blocked_to_blocked[indices_conn[i]] = dofs_view[i]; + } + // Create map from dof in view to new dof index std::vector old_to_new(dofs_view.back() + bs_view, -1); { @@ -160,22 +168,21 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "old_to_new = " << old_to_new << "\n"; std::vector old_to_new_conn(dofs_view.back() + bs_view, -1); + // for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) + // { + // for (int k = 0; k < bs_view; ++k) + // { + // std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; + // old_to_new_conn[old_idx] = new_idx; + // } + // } + for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { - for (int k = 0; k < bs_view; ++k) - { - std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; - old_to_new_conn[old_idx] = new_idx; - } + // TODO Simplify + old_to_new_conn[non_blocked_to_blocked[new_to_old_conn[new_idx]]] = new_idx; } - // FIXME - // std::vector old_to_new_conn(dofs_view.back() + 1, -1); - // for (std::size_t i = 0; i < indices_conn.size(); ++i) - // { - // old_to_new_conn[dofs_view[i]] = indices_conn[i]; - // } - ss << "old_to_new_conn = " << old_to_new_conn << "\n"; // Map dofs to new collapsed indices for new dofmap From c0b72e70f6f61b2c2639c9b280ad4f455c32c7c4 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 7 Dec 2023 11:39:05 +0000 Subject: [PATCH 067/101] Fix array size --- cpp/dolfinx/fem/DofMap.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 07a5a9aa60e..d78b1209b96 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -135,7 +135,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; // TODO Simplify - std::vector non_blocked_to_blocked(indices_conn.size(), -1); + std::vector non_blocked_to_blocked( + indices_conn.back() + 1, -1); for (std::size_t i = 0; i < indices_conn.size(); ++i) { non_blocked_to_blocked[indices_conn[i]] = dofs_view[i]; From 0072d06f70e97d4d9d338e575005a52d0b229d69 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 7 Dec 2023 12:50:00 +0000 Subject: [PATCH 068/101] Revert "Fix array size" This reverts commit c0b72e70f6f61b2c2639c9b280ad4f455c32c7c4. --- cpp/dolfinx/fem/DofMap.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index d78b1209b96..07a5a9aa60e 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -135,8 +135,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; // TODO Simplify - std::vector non_blocked_to_blocked( - indices_conn.back() + 1, -1); + std::vector non_blocked_to_blocked(indices_conn.size(), -1); for (std::size_t i = 0; i < indices_conn.size(); ++i) { non_blocked_to_blocked[indices_conn[i]] = dofs_view[i]; From 712f44172a3f41090a85ff1aa06a5cbf214034da Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Thu, 7 Dec 2023 12:50:14 +0000 Subject: [PATCH 069/101] Revert "Try using map" This reverts commit 4cc5dc3a7f22198e748323be436dfeb4769a74a3. --- cpp/dolfinx/fem/DofMap.cpp | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 07a5a9aa60e..993927a0532 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -84,7 +84,6 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, bs_view * num_owned_view); // Create sub-index map - // TODO Simplify std::shared_ptr index_map; std::vector ghost_new_to_old; std::shared_ptr index_map_conn; @@ -134,13 +133,6 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "new_to_old_conn = " << new_to_old_conn << "\n"; - // TODO Simplify - std::vector non_blocked_to_blocked(indices_conn.size(), -1); - for (std::size_t i = 0; i < indices_conn.size(); ++i) - { - non_blocked_to_blocked[indices_conn[i]] = dofs_view[i]; - } - // Create map from dof in view to new dof index std::vector old_to_new(dofs_view.back() + bs_view, -1); { @@ -168,21 +160,22 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, ss << "old_to_new = " << old_to_new << "\n"; std::vector old_to_new_conn(dofs_view.back() + bs_view, -1); - // for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) - // { - // for (int k = 0; k < bs_view; ++k) - // { - // std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; - // old_to_new_conn[old_idx] = new_idx; - // } - // } - for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { - // TODO Simplify - old_to_new_conn[non_blocked_to_blocked[new_to_old_conn[new_idx]]] = new_idx; + for (int k = 0; k < bs_view; ++k) + { + std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; + old_to_new_conn[old_idx] = new_idx; + } } + // FIXME + // std::vector old_to_new_conn(dofs_view.back() + 1, -1); + // for (std::size_t i = 0; i < indices_conn.size(); ++i) + // { + // old_to_new_conn[dofs_view[i]] = indices_conn[i]; + // } + ss << "old_to_new_conn = " << old_to_new_conn << "\n"; // Map dofs to new collapsed indices for new dofmap From fdf26f09f05a217a9b4dc0a1bc3a0f93c7e1d1b7 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 09:43:17 +0000 Subject: [PATCH 070/101] Tidy --- cpp/dolfinx/fem/DofMap.cpp | 91 +++++++------------------------------- 1 file changed, 15 insertions(+), 76 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index 993927a0532..e8f4df1e0f0 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -22,8 +22,7 @@ using namespace dolfinx::fem; // A function for printing vectors template -std::ostream &operator<<(std::ostream &os, - const std::vector &vector) +std::ostream& operator<<(std::ostream& os, const std::vector& vector) { os << "{ "; for (auto v : vector) @@ -80,85 +79,31 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Get block size int bs_view = dofmap_view.index_map_bs(); - const auto it = std::lower_bound(dofs_view.begin(), dofs_view.end(), - bs_view * num_owned_view); - // Create sub-index map - std::shared_ptr index_map; - std::vector ghost_new_to_old; std::shared_ptr index_map_conn; - std::vector new_to_old_conn; // TODO Rename as this is non-block index - std::vector indices_conn; + std::vector new_to_old_conn; // TODO Rename if (bs_view == 1) { - std::span indices(dofs_view.data(), - std::distance(dofs_view.begin(), it)); - auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); - index_map = std::make_shared(std::move(_index_map)); - ghost_new_to_old = std::move(gmap); - - // ss << "indices = " << indices << "\n"; - - indices_conn = dofs_view; auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( - *dofmap_view.index_map, indices_conn); - index_map_conn = std::make_shared(std::move(_index_map_conn)); + *dofmap_view.index_map, dofs_view); + index_map_conn + = std::make_shared(std::move(_index_map_conn)); new_to_old_conn = std::move(gmap_conn); - - ss << "indices_conn = " << dofs_view << "\n"; } else { std::vector indices; indices.reserve(dofs_view.size()); - std::transform(dofs_view.begin(), it, std::back_inserter(indices), - [bs_view](auto idx) { return idx / bs_view; }); - indices.erase(std::unique(indices.begin(), indices.end()), indices.end()); - auto [_index_map, gmap] = dofmap_view.index_map->create_submap(indices); - index_map = std::make_shared(std::move(_index_map)); - ghost_new_to_old = std::move(gmap); - - // ss << "indices = " << indices << "\n"; - indices_conn.reserve(dofs_view.size()); - std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices_conn), + std::transform(dofs_view.begin(), dofs_view.end(), + std::back_inserter(indices), [bs_view](auto idx) { return idx / bs_view; }); - // indices_conn.erase(std::unique(indices_conn.begin(), indices_conn.end()), indices_conn.end()); - auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( - *dofmap_view.index_map, indices_conn); - index_map_conn = std::make_shared(std::move(_index_map_conn)); + auto [_index_map_conn, gmap_conn] + = dolfinx::common::create_submap_conn(*dofmap_view.index_map, indices); + index_map_conn + = std::make_shared(std::move(_index_map_conn)); new_to_old_conn = std::move(gmap_conn); - - ss << "indices_conn = " << indices_conn << "\n"; } - ss << "new_to_old_conn = " << new_to_old_conn << "\n"; - - // Create map from dof in view to new dof index - std::vector old_to_new(dofs_view.back() + bs_view, -1); - { - // old-to-new map for owned dofs - std::int32_t count = 0; - for (auto dof_old = dofs_view.begin(); dof_old != it; ++dof_old) - old_to_new[*dof_old] = count++; - - // old-to-new map for ghost dofs - const std::int32_t local_size_new = index_map->size_local(); - for (auto itp_old = ghost_new_to_old.begin(); - itp_old != ghost_new_to_old.end(); ++itp_old) - { - std::int32_t map_pos_new - = local_size_new + std::distance(ghost_new_to_old.begin(), itp_old); - std::int32_t idx = bs_view * (num_owned_view + *itp_old); - for (int k = 0; k < bs_view; ++k) - { - assert(idx + k < (int)old_to_new.size()); - old_to_new[idx + k] = map_pos_new; - } - } - } - - ss << "old_to_new = " << old_to_new << "\n"; - std::vector old_to_new_conn(dofs_view.back() + bs_view, -1); for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) { @@ -169,13 +114,6 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } } - // FIXME - // std::vector old_to_new_conn(dofs_view.back() + 1, -1); - // for (std::size_t i = 0; i < indices_conn.size(); ++i) - // { - // old_to_new_conn[dofs_view[i]] = indices_conn[i]; - // } - ss << "old_to_new_conn = " << old_to_new_conn << "\n"; // Map dofs to new collapsed indices for new dofmap @@ -185,7 +123,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::transform(dof_array_view.data_handle(), dof_array_view.data_handle() + dof_array_view.size(), std::back_inserter(dofmap), - [&old_to_new_conn](auto idx_old) { return old_to_new_conn[idx_old]; }); + [&old_to_new_conn](auto idx_old) + { return old_to_new_conn[idx_old]; }); // Dimension sanity checks assert((int)dofmap.size() @@ -205,8 +144,8 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, } // Create new dofmap and return - return DofMap(std::move(element_dof_layout), index_map_conn, 1, std::move(dofmap), - 1); + return DofMap(std::move(element_dof_layout), index_map_conn, 1, + std::move(dofmap), 1); } } // namespace From 12607cb02563cb07baa017d572b0ca57e129ce8c Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 09:53:36 +0000 Subject: [PATCH 071/101] Tidy --- cpp/dolfinx/fem/DofMap.cpp | 71 +++++++++----------------------------- 1 file changed, 16 insertions(+), 55 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index e8f4df1e0f0..cdd6ce29846 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -20,19 +20,6 @@ using namespace dolfinx; using namespace dolfinx::fem; -// A function for printing vectors -template -std::ostream& operator<<(std::ostream& os, const std::vector& vector) -{ - os << "{ "; - for (auto v : vector) - { - os << v << " "; - } - os << "}"; - return os; -} - namespace { //----------------------------------------------------------------------------- @@ -41,12 +28,6 @@ namespace fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, const mesh::Topology& topology) { - std::stringstream ss; - - const int rank = dolfinx::MPI::rank(topology.comm()); - - ss << "Rank " << rank << ":\n"; - if (dofmap_view.element_dof_layout().block_size() > 1) { throw std::runtime_error( @@ -69,26 +50,18 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, dofs_view.erase(std::unique(dofs_view.begin(), dofs_view.end()), dofs_view.end()); - ss << "dofs_view = " << dofs_view << "\n"; - - // Compute sizes - const std::int32_t num_owned_view = dofmap_view.index_map->size_local(); - - ss << "num_owned_view = " << num_owned_view << "\n"; - // Get block size int bs_view = dofmap_view.index_map_bs(); // Create sub-index map - std::shared_ptr index_map_conn; - std::vector new_to_old_conn; // TODO Rename + std::shared_ptr index_map; + std::vector new_to_old; // TODO Rename if (bs_view == 1) { - auto [_index_map_conn, gmap_conn] = dolfinx::common::create_submap_conn( + auto [_index_map, _new_to_old] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, dofs_view); - index_map_conn - = std::make_shared(std::move(_index_map_conn)); - new_to_old_conn = std::move(gmap_conn); + index_map = std::make_shared(std::move(_index_map)); + new_to_old = std::move(_new_to_old); } else { @@ -97,25 +70,23 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices), [bs_view](auto idx) { return idx / bs_view; }); - auto [_index_map_conn, gmap_conn] + auto [_index_map, _new_to_old] = dolfinx::common::create_submap_conn(*dofmap_view.index_map, indices); - index_map_conn - = std::make_shared(std::move(_index_map_conn)); - new_to_old_conn = std::move(gmap_conn); + index_map = std::make_shared(std::move(_index_map)); + new_to_old = std::move(_new_to_old); } - std::vector old_to_new_conn(dofs_view.back() + bs_view, -1); - for (std::size_t new_idx = 0; new_idx < new_to_old_conn.size(); ++new_idx) + std::vector old_to_new(dofs_view.back() + bs_view, -1); + for (std::size_t new_idx = 0; new_idx < new_to_old.size(); ++new_idx) { for (int k = 0; k < bs_view; ++k) { - std::int32_t old_idx = new_to_old_conn[new_idx] * bs_view + k; - old_to_new_conn[old_idx] = new_idx; + std::int32_t old_idx = new_to_old[new_idx] * bs_view + k; + assert(old_idx < (int)old_to_new.size()); + old_to_new[old_idx] = new_idx; } } - ss << "old_to_new_conn = " << old_to_new_conn << "\n"; - // Map dofs to new collapsed indices for new dofmap auto dof_array_view = dofmap_view.map(); std::vector dofmap; @@ -123,8 +94,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::transform(dof_array_view.data_handle(), dof_array_view.data_handle() + dof_array_view.size(), std::back_inserter(dofmap), - [&old_to_new_conn](auto idx_old) - { return old_to_new_conn[idx_old]; }); + [&old_to_new](auto idx_old) { return old_to_new[idx_old]; }); // Dimension sanity checks assert((int)dofmap.size() @@ -134,18 +104,9 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Copy dof layout, discarding parent data ElementDofLayout element_dof_layout = dofmap_view.element_dof_layout().copy(); - for (int r = 0; r < dolfinx::MPI::size(topology.comm()); ++r) - { - if (r == rank) - { - std::cout << ss.str() << "\n"; - } - MPI_Barrier(topology.comm()); - } - // Create new dofmap and return - return DofMap(std::move(element_dof_layout), index_map_conn, 1, - std::move(dofmap), 1); + return DofMap(std::move(element_dof_layout), index_map, 1, std::move(dofmap), + 1); } } // namespace From d18a2e68cdd7499ebec2ebc72446ed3202dd7c48 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 10:00:54 +0000 Subject: [PATCH 072/101] Tidy --- cpp/dolfinx/fem/DofMap.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index cdd6ce29846..dbd1f3d95ed 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -55,13 +55,15 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, // Create sub-index map std::shared_ptr index_map; - std::vector new_to_old; // TODO Rename + // Map from indices in the sub-index map to indices in the original + // index map + std::vector sub_imap_to_imap; if (bs_view == 1) { - auto [_index_map, _new_to_old] = dolfinx::common::create_submap_conn( + auto [_index_map, _sub_imap_to_imap] = dolfinx::common::create_submap_conn( *dofmap_view.index_map, dofs_view); index_map = std::make_shared(std::move(_index_map)); - new_to_old = std::move(_new_to_old); + sub_imap_to_imap = std::move(_sub_imap_to_imap); } else { @@ -70,18 +72,19 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::transform(dofs_view.begin(), dofs_view.end(), std::back_inserter(indices), [bs_view](auto idx) { return idx / bs_view; }); - auto [_index_map, _new_to_old] + auto [_index_map, _sub_imap_to_imap] = dolfinx::common::create_submap_conn(*dofmap_view.index_map, indices); index_map = std::make_shared(std::move(_index_map)); - new_to_old = std::move(_new_to_old); + sub_imap_to_imap = std::move(_sub_imap_to_imap); } + // Create a map from old dofs to new dofs std::vector old_to_new(dofs_view.back() + bs_view, -1); - for (std::size_t new_idx = 0; new_idx < new_to_old.size(); ++new_idx) + for (std::size_t new_idx = 0; new_idx < sub_imap_to_imap.size(); ++new_idx) { for (int k = 0; k < bs_view; ++k) { - std::int32_t old_idx = new_to_old[new_idx] * bs_view + k; + std::int32_t old_idx = sub_imap_to_imap[new_idx] * bs_view + k; assert(old_idx < (int)old_to_new.size()); old_to_new[old_idx] = new_idx; } From c589ffe1d77720c48101ae2ac50fc7efcf5a0ed9 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 10:42:42 +0000 Subject: [PATCH 073/101] Add option to allow owner to change --- cpp/dolfinx/common/IndexMap.cpp | 19 +++++++++++++++---- cpp/dolfinx/common/IndexMap.h | 15 +++++++-------- cpp/dolfinx/mesh/Geometry.h | 2 +- cpp/dolfinx/mesh/Topology.cpp | 2 +- python/dolfinx/wrappers/common.cpp | 7 ++++--- python/test/unit/common/test_index_map.py | 4 +++- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c42e4a74a40..2718a427f22 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -127,13 +127,18 @@ communicate_ghosts_to_owners(const MPI_Comm comm, /// the submap. /// @param[in] imap An index map. /// @param[in] indices List of entity indices (indices local to the process). +/// @param[in] allow_owner_change Allows indices that are not included by +/// their owning process but included on sharing processes to be +/// included in the submap. These indices will be owned by one of the sharing +/// processes in the submap. /// @pre `indices` must be sorted and unique. /// @return The owned, ghost, and ghost owners in the submap. All indices are /// local and with respect to the original index map. std::tuple, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, - std::span indices) + std::span indices, + bool allow_owner_change) { const MPI_Comm comm = imap.comm(); std::span src = imap.src(); @@ -163,6 +168,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::vector recv_owners(send_disp.back()); const int rank = dolfinx::MPI::rank(comm); { + bool owners_changed = false; // Create a map from (global) owned shared indices owned to processes that // can own them. // FIXME Avoid map @@ -190,11 +196,15 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, global_idx_to_possible_owner[idx].push_back(rank); else { - // throw std::runtime_error("Index owner change detected!"); + owners_changed = true; + global_idx_to_possible_owner[idx].push_back(dest[i]); } } } + + if (owners_changed && !allow_owner_change) + throw std::runtime_error("Index owner change detected!"); } // Choose the submap owner for each index in `recv_indices` @@ -655,7 +665,8 @@ common::stack_index_maps( //----------------------------------------------------------------------------- std::pair> common::create_submap_conn(const dolfinx::common::IndexMap& imap, - std::span indices) + std::span indices, + bool allow_owner_change) { const MPI_Comm comm = imap.comm(); @@ -663,7 +674,7 @@ common::create_submap_conn(const dolfinx::common::IndexMap& imap, // NOTE: All indices are local and numbered w.r.t. the original (imap) index // map auto [submap_owned, submap_ghost, submap_ghost_owners] - = compute_submap_indices(imap, indices); + = compute_submap_indices(imap, indices, allow_owner_change); // Compute submap offset for this rank std::int64_t submap_local_size = submap_owned.size(); diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index b01aff8a4f6..1f786f9fd97 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -55,21 +55,20 @@ stack_index_maps( std::pair, int>>& maps); /// @brief Create a new index map from a subset of indices in an index -/// map. Any indices that are not included by their owning process, but -/// are included on sharing processes, will be owned by one of the sharing -/// processes in the submap. -/// -/// This can be used when, for instance, creating a submesh to ensure -/// that all vertices have exactly one owner and are connected to at least -/// one cell on the owning process. +/// map. /// @param[in] imap The index map /// @param[in] indices Local indices to include in the new index map (owned /// and ghost) +/// @param[in] allow_owner_change Allows indices that are not included by +/// their owning process but included on sharing processes to be +/// included in the submap. These indices will be owned by one of the sharing +/// processes in the submap. /// @pre `indices` must be sorted and contain no duplicates. /// @return The (i) new index map and (ii) a map from local indices in the /// submap to local indices in the original (this) map std::pair> -create_submap_conn(const IndexMap& imap, std::span indices); +create_submap_conn(const IndexMap& imap, std::span indices, + bool allow_owner_change = false); /// This class represents the distribution index arrays across /// processes. An index array is a contiguous collection of N+1 indices diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index 6d09217aedf..fcebe88e199 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -333,7 +333,7 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, std::vector subx_to_x_dofmap; { std::pair> map_data - = dolfinx::common::create_submap_conn(*x_index_map, sub_x_dofs); + = dolfinx::common::create_submap_conn(*x_index_map, sub_x_dofs, true); sub_x_dof_index_map = std::make_shared(std::move(map_data.first)); subx_to_x_dofmap = std::move(map_data.second); diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index 2d558e9f446..f98a1846a67 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1197,7 +1197,7 @@ mesh::create_subtopology(const Topology& topology, int dim, { std::pair> map_data = dolfinx::common::create_submap_conn( - *map0, compute_incident_entities(topology, subentities, dim, 0)); + *map0, compute_incident_entities(topology, subentities, dim, 0), true); submap0 = std::make_shared(std::move(map_data.first)); subvertices0 = std::move(map_data.second); } diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 021084adaec..4f9aa41fbed 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -187,13 +187,14 @@ void common(nb::module_& m) m.def( "create_submap_conn", [](const dolfinx::common::IndexMap& imap, - nb::ndarray, nb::c_contig> indices) + nb::ndarray, nb::c_contig> indices, + bool allow_owner_change = false) { auto [map, submap_to_map] = dolfinx::common::create_submap_conn( - imap, std::span(indices.data(), indices.size())); + imap, std::span(indices.data(), indices.size()), allow_owner_change); return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( std::move(submap_to_map))); }, - nb::arg("index_map"), nb::arg("indices")); + nb::arg("index_map"), nb::arg("indices"), nb::arg("allow_owner_change")); } } // namespace dolfinx_wrappers diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 6e13092b48d..dd501ab805b 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -14,6 +14,8 @@ from dolfinx.mesh import GhostMode, create_unit_square from dolfinx import cpp as _cpp +import pytest + def test_sub_index_map(): comm = MPI.COMM_WORLD @@ -164,7 +166,7 @@ def test_create_submap_connected(): imap = dolfinx.common.IndexMap(comm, local_size, ghosts, owners) sub_imap, sub_imap_to_imap = _cpp.common.create_submap_conn( - imap, submap_indices) + imap, submap_indices, True) if comm.rank == 0: assert sub_imap.size_local == 2 From 93f131d4a0337486d60c098f4ed43239183d923a Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 10:47:31 +0000 Subject: [PATCH 074/101] Remove create submap --- cpp/dolfinx/common/IndexMap.cpp | 105 ----------------------------- cpp/dolfinx/common/IndexMap.h | 16 ----- cpp/dolfinx/mesh/Topology.cpp | 3 - python/dolfinx/wrappers/common.cpp | 13 +--- 4 files changed, 1 insertion(+), 136 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2718a427f22..33a0bd9596c 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -870,111 +870,6 @@ std::vector IndexMap::global_indices() const //----------------------------------------------------------------------------- MPI_Comm IndexMap::comm() const { return _comm.comm(); } //---------------------------------------------------------------------------- -std::pair> -IndexMap::create_submap(std::span indices) const -{ - if (!indices.empty() and indices.back() >= this->size_local()) - { - throw std::runtime_error( - "Unowned index detected when creating sub-IndexMap"); - } - - // --- Step 1: Compute new offset for this rank - - std::int64_t local_size_new = indices.size(); - std::int64_t offset_new = 0; - MPI_Request request_offset; - int ierr = MPI_Iexscan(&local_size_new, &offset_new, 1, MPI_INT64_T, MPI_SUM, - _comm.comm(), &request_offset); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // --- Step 2: Send ghost indices to owning rank - const std::vector& src = this->src(); - const std::vector& dest = this->dest(); - auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, recv_sizes, - send_disp, recv_disp] - = communicate_ghosts_to_owners( - this->comm(), src, dest, this->ghosts(), this->owners(), - std::vector(this->ghosts().size(), 1)); - - // --- Step 3: Check which received indexes (all of which I should - // own) are in the submap - - // Build array for each received ghost that (i) contains the new - // submap index if it is retained, or (ii) set to -1 if it is not - // retained. - std::vector send_gidx; - send_gidx.reserve(recv_indices.size()); - for (auto idx : recv_indices) - { - assert(idx - _local_range[0] >= 0); - assert(idx - _local_range[0] < _local_range[1]); - std::int32_t idx_local = idx - _local_range[0]; - - // Could avoid search by creating look-up array - auto it = std::lower_bound(indices.begin(), indices.end(), idx_local); - if (it != indices.end() and *it == idx_local) - { - std::size_t idx_local_new = std::distance(indices.begin(), it); - send_gidx.push_back(idx_local_new + offset_new); - } - else - send_gidx.push_back(-1); - } - - // --- Step 4: Send new global indices from owner back to ranks that - // ghost the index - - // Create neighbourhood comm (owner -> ghost) - MPI_Comm comm1; - ierr = MPI_Dist_graph_create_adjacent( - _comm.comm(), src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), - dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // Send index markers to ghosting ranks - std::vector recv_gidx(send_disp.back()); - ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT64_T, recv_gidx.data(), - send_sizes.data(), send_disp.data(), - MPI_INT64_T, comm1); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - ierr = MPI_Comm_free(&comm1); - dolfinx::MPI::check_error(_comm.comm(), ierr); - - // --- Step 5: Unpack received data - - std::vector ghosts; - std::vector src_ranks; - std::vector new_to_old_ghost; - for (std::size_t i = 0; i < send_disp.size() - 1; ++i) - { - for (int j = send_disp[i]; j < send_disp[i + 1]; ++j) - { - if (std::int64_t idx = recv_gidx[j]; idx >= 0) - { - std::size_t p = ghost_buffer_pos[j]; - ghosts.push_back(idx); - src_ranks.push_back(src[i]); - new_to_old_ghost.push_back(p); - } - } - } - - if (_overlapping) - { - return {IndexMap(_comm.comm(), local_size_new, ghosts, src_ranks), - std::move(new_to_old_ghost)}; - } - else - { - assert(new_to_old_ghost.empty()); - return {IndexMap(_comm.comm(), local_size_new), - std::vector()}; - } -} -//----------------------------------------------------------------------------- graph::AdjacencyList IndexMap::index_to_dest_ranks() const { const std::int64_t offset = _local_range[0]; diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 1f786f9fd97..bd52da08524 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -189,22 +189,6 @@ class IndexMap /// index is `owners()[i]`. const std::vector& owners() const { return _owners; } - /// @brief Create new index map from a subset of indices in this index - /// map. - /// - /// The order of the owned indices is preserved, with new map - /// effectively a 'compressed' map. - /// - /// @param[in] indices Local indices in the map that should appear in - /// the new index map. All indices must be owned, i.e. indices must be - /// less than `this->size_local()`. - /// @pre `indices` must be sorted and contain no duplicates. - /// @return The (i) new index map and (ii) a map from the ghost - /// position in the new map to the ghost position in the original - /// (this) map - std::pair> - create_submap(std::span indices) const; - /// @todo Aim to remove this function? /// /// @brief Compute map from each local (owned) index to the set of diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index f98a1846a67..b3e25343f5e 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1160,9 +1160,6 @@ mesh::create_subtopology(const Topology& topology, int dim, // TODO Call common::get_owned_indices here? Do we want to // support `entities` possibly having a ghost on one process that is // not in `entities` on the owning process? - // TODO: Should entities still be ghosted in the sub-topology even if - // they are not in the `entities` list? If this is not desirable, - // create_submap needs to be changed // Create a map from an entity in the sub-topology to the // corresponding entity in the topology, and create an index map diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 4f9aa41fbed..59c2cbe109b 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -133,18 +133,7 @@ void common(nb::module_& m) std::span(global.data(), global.size())); return global; }, - nb::arg("local")) - .def( - "create_submap", - [](const dolfinx::common::IndexMap& self, - nb::ndarray, nb::c_contig> indices) - { - auto [map, ghosts] - = self.create_submap(std::span(indices.data(), indices.size())); - return std::pair(std::move(map), - dolfinx_wrappers::as_nbarray(std::move(ghosts))); - }, - nb::arg("indices")); + nb::arg("local")); // dolfinx::common::Timer nb::class_(m, "Timer", "Timer class") From d4fbd43b15d228facb93a9f32d7723943ab60345 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 10:48:31 +0000 Subject: [PATCH 075/101] Remove unused import --- python/test/unit/common/test_index_map.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index dd501ab805b..409411379b3 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -14,8 +14,6 @@ from dolfinx.mesh import GhostMode, create_unit_square from dolfinx import cpp as _cpp -import pytest - def test_sub_index_map(): comm = MPI.COMM_WORLD From 0c037cf44787671432143eb1bdf795ba9d08d668 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 10:55:50 +0000 Subject: [PATCH 076/101] Remove shrink_to_fit --- cpp/dolfinx/common/IndexMap.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 33a0bd9596c..2989ca901e3 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -689,7 +689,6 @@ common::create_submap_conn(const dolfinx::common::IndexMap& imap, std::sort(submap_src.begin(), submap_src.end()); submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), submap_src.end()); - submap_src.shrink_to_fit(); // Compute submap destination ranks // NOTE: The call to NBX can be avoided by a two-step communication process From 6b4b1ed92ca7b74ea7998741952120df871fb1e5 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 11:48:20 +0000 Subject: [PATCH 077/101] Add param --- python/dolfinx/wrappers/common.cpp | 2 +- python/test/unit/common/test_index_map.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 59c2cbe109b..344663d6c58 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -177,7 +177,7 @@ void common(nb::module_& m) "create_submap_conn", [](const dolfinx::common::IndexMap& imap, nb::ndarray, nb::c_contig> indices, - bool allow_owner_change = false) + bool allow_owner_change) { auto [map, submap_to_map] = dolfinx::common::create_submap_conn( imap, std::span(indices.data(), indices.size()), allow_owner_change); diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 409411379b3..3ea9e30957d 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -45,7 +45,7 @@ def test_sub_index_map(): # Create sub index map and a map from the ghost position in new map # to the position in old map - submap, submap_to_map = _cpp.common.create_submap_conn(map, local_indices[my_rank]) + submap, submap_to_map = _cpp.common.create_submap_conn(map, local_indices[my_rank], False) ghosts_pos_sub = submap_to_map[map_local_size:] - map_local_size # Check local and global sizes @@ -83,7 +83,7 @@ def test_sub_index_map_ghost_mode_none(): tdim = mesh.topology.dim map = mesh.topology.index_map(tdim) submap_indices = np.arange(0, min(2, map.size_local), dtype=np.int32) - _cpp.common.create_submap_conn(map, submap_indices) + _cpp.common.create_submap_conn(map, submap_indices, False) def test_index_map_ghost_lifetime(): From d49884fea6f1504a744af087937cddf8127ac737 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 8 Dec 2023 11:54:27 +0000 Subject: [PATCH 078/101] Rename --- cpp/dolfinx/common/IndexMap.cpp | 2 +- cpp/dolfinx/common/IndexMap.h | 2 +- cpp/dolfinx/fem/DofMap.cpp | 4 ++-- cpp/dolfinx/mesh/Geometry.h | 2 +- cpp/dolfinx/mesh/Topology.cpp | 4 ++-- python/dolfinx/wrappers/common.cpp | 4 ++-- python/test/unit/common/test_index_map.py | 8 ++++---- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2989ca901e3..39aa504ff4f 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -664,7 +664,7 @@ common::stack_index_maps( } //----------------------------------------------------------------------------- std::pair> -common::create_submap_conn(const dolfinx::common::IndexMap& imap, +common::create_sub_index_map(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) { diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index bd52da08524..a95925022f2 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -67,7 +67,7 @@ stack_index_maps( /// @return The (i) new index map and (ii) a map from local indices in the /// submap to local indices in the original (this) map std::pair> -create_submap_conn(const IndexMap& imap, std::span indices, +create_sub_index_map(const IndexMap& imap, std::span indices, bool allow_owner_change = false); /// This class represents the distribution index arrays across diff --git a/cpp/dolfinx/fem/DofMap.cpp b/cpp/dolfinx/fem/DofMap.cpp index dbd1f3d95ed..dc6c32fa1cb 100644 --- a/cpp/dolfinx/fem/DofMap.cpp +++ b/cpp/dolfinx/fem/DofMap.cpp @@ -60,7 +60,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::vector sub_imap_to_imap; if (bs_view == 1) { - auto [_index_map, _sub_imap_to_imap] = dolfinx::common::create_submap_conn( + auto [_index_map, _sub_imap_to_imap] = dolfinx::common::create_sub_index_map( *dofmap_view.index_map, dofs_view); index_map = std::make_shared(std::move(_index_map)); sub_imap_to_imap = std::move(_sub_imap_to_imap); @@ -73,7 +73,7 @@ fem::DofMap build_collapsed_dofmap(const DofMap& dofmap_view, std::back_inserter(indices), [bs_view](auto idx) { return idx / bs_view; }); auto [_index_map, _sub_imap_to_imap] - = dolfinx::common::create_submap_conn(*dofmap_view.index_map, indices); + = dolfinx::common::create_sub_index_map(*dofmap_view.index_map, indices); index_map = std::make_shared(std::move(_index_map)); sub_imap_to_imap = std::move(_sub_imap_to_imap); } diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index fcebe88e199..4d3126ed4ee 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -333,7 +333,7 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, std::vector subx_to_x_dofmap; { std::pair> map_data - = dolfinx::common::create_submap_conn(*x_index_map, sub_x_dofs, true); + = dolfinx::common::create_sub_index_map(*x_index_map, sub_x_dofs, true); sub_x_dof_index_map = std::make_shared(std::move(map_data.first)); subx_to_x_dofmap = std::move(map_data.second); diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index b3e25343f5e..3f80de04f7b 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1173,7 +1173,7 @@ mesh::create_subtopology(const Topology& topology, int dim, std::unique(unique_entities.begin(), unique_entities.end()), unique_entities.end()); std::pair> map_data - = dolfinx::common::create_submap_conn(*topology.index_map(dim), + = dolfinx::common::create_sub_index_map(*topology.index_map(dim), unique_entities); submap = std::make_shared(std::move(map_data.first)); subentities = std::move(map_data.second); @@ -1193,7 +1193,7 @@ mesh::create_subtopology(const Topology& topology, int dim, std::vector subvertices0; { std::pair> map_data - = dolfinx::common::create_submap_conn( + = dolfinx::common::create_sub_index_map( *map0, compute_incident_entities(topology, subentities, dim, 0), true); submap0 = std::make_shared(std::move(map_data.first)); subvertices0 = std::move(map_data.second); diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 344663d6c58..3a967de098e 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -174,12 +174,12 @@ void common(nb::module_& m) nb::arg("args")); m.def( - "create_submap_conn", + "create_sub_index_map", [](const dolfinx::common::IndexMap& imap, nb::ndarray, nb::c_contig> indices, bool allow_owner_change) { - auto [map, submap_to_map] = dolfinx::common::create_submap_conn( + auto [map, submap_to_map] = dolfinx::common::create_sub_index_map( imap, std::span(indices.data(), indices.size()), allow_owner_change); return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( std::move(submap_to_map))); diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 3ea9e30957d..9c01d3ad41d 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -45,7 +45,7 @@ def test_sub_index_map(): # Create sub index map and a map from the ghost position in new map # to the position in old map - submap, submap_to_map = _cpp.common.create_submap_conn(map, local_indices[my_rank], False) + submap, submap_to_map = _cpp.common.create_sub_index_map(map, local_indices[my_rank], False) ghosts_pos_sub = submap_to_map[map_local_size:] - map_local_size # Check local and global sizes @@ -83,7 +83,7 @@ def test_sub_index_map_ghost_mode_none(): tdim = mesh.topology.dim map = mesh.topology.index_map(tdim) submap_indices = np.arange(0, min(2, map.size_local), dtype=np.int32) - _cpp.common.create_submap_conn(map, submap_indices, False) + _cpp.common.create_sub_index_map(map, submap_indices, False) def test_index_map_ghost_lifetime(): @@ -115,7 +115,7 @@ def test_index_map_ghost_lifetime(): # whose owner changes in the submap def test_create_submap_connected(): """ - Test create_submap_conn. The diagram illustrates the case with four + Test create_sub_index_map. The diagram illustrates the case with four processes. Original map numbering and connectivity (G indicates a ghost index): Global Rank 0 Rank 1 Rank 2 Rank 3 @@ -163,7 +163,7 @@ def test_create_submap_connected(): submap_indices = np.array([0, 2, 3], dtype=np.int32) imap = dolfinx.common.IndexMap(comm, local_size, ghosts, owners) - sub_imap, sub_imap_to_imap = _cpp.common.create_submap_conn( + sub_imap, sub_imap_to_imap = _cpp.common.create_sub_index_map( imap, submap_indices, True) if comm.rank == 0: From 0f75497fb937a0cadc942afe105e2dd9982c091a Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 11:40:44 +0000 Subject: [PATCH 079/101] Use non-blocking comm --- cpp/dolfinx/common/IndexMap.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 39aa504ff4f..13c6b8ad7e0 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -81,26 +81,27 @@ communicate_ghosts_to_owners(const MPI_Comm comm, std::back_inserter(send_sizes), [](auto& d) { return d.size(); }); - // Build send buffer and ghost position to send buffer position - for (auto& d : send_data) - send_indices.insert(send_indices.end(), d.begin(), d.end()); - for (auto& p : pos_to_ghost) - ghost_buffer_pos.insert(ghost_buffer_pos.end(), p.begin(), p.end()); - // Send how many indices I ghost to each owner, and receive how many // of my indices other ranks ghost recv_sizes.resize(dest.size(), 0); send_sizes.reserve(1); recv_sizes.reserve(1); - ierr = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, - recv_sizes.data(), 1, MPI_INT32_T, comm0); - dolfinx::MPI::check_error(comm, ierr); + MPI_Request sizes_request; + MPI_Ineighbor_alltoall(send_sizes.data(), 1, MPI_INT32_T, recv_sizes.data(), + 1, MPI_INT32_T, comm0, &sizes_request); + + // Build send buffer and ghost position to send buffer position + for (auto& d : send_data) + send_indices.insert(send_indices.end(), d.begin(), d.end()); + for (auto& p : pos_to_ghost) + ghost_buffer_pos.insert(ghost_buffer_pos.end(), p.begin(), p.end()); // Prepare displacement vectors send_disp.resize(src.size() + 1, 0); recv_disp.resize(dest.size() + 1, 0); std::partial_sum(send_sizes.begin(), send_sizes.end(), std::next(send_disp.begin())); + MPI_Wait(&sizes_request, MPI_STATUS_IGNORE); std::partial_sum(recv_sizes.begin(), recv_sizes.end(), std::next(recv_disp.begin())); @@ -665,8 +666,8 @@ common::stack_index_maps( //----------------------------------------------------------------------------- std::pair> common::create_sub_index_map(const dolfinx::common::IndexMap& imap, - std::span indices, - bool allow_owner_change) + std::span indices, + bool allow_owner_change) { const MPI_Comm comm = imap.comm(); From f3967c3bf44ebf6c0e511efd706f1133e084bb41 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 13:57:29 +0000 Subject: [PATCH 080/101] Remove NBX --- cpp/dolfinx/common/IndexMap.cpp | 102 ++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 13c6b8ad7e0..fb624340905 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,9 +14,34 @@ #include #include +#include + using namespace dolfinx; using namespace dolfinx::common; +template +std::ostream& operator<<(std::ostream& os, const std::vector& vector) +{ + os << "{ "; + for (auto v : vector) + { + os << v << " "; + } + os << "}"; + return os; +} + +// A function for printing maps +template +std::ostream& operator<<(std::ostream& os, const std::map& map) +{ + os << "{ "; + for (const auto& [k, v] : map) + os << k << ": " << v << " "; + os << "}"; + return os; +} + namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -135,16 +160,22 @@ communicate_ghosts_to_owners(const MPI_Comm comm, /// @pre `indices` must be sorted and unique. /// @return The owned, ghost, and ghost owners in the submap. All indices are /// local and with respect to the original index map. +// TODO Update docs std::tuple, std::vector, + std::vector, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) { + std::stringstream ss; + const MPI_Comm comm = imap.comm(); std::span src = imap.src(); std::span dest = imap.dest(); + ss << "rank " << dolfinx::MPI::rank(comm) << ":\n"; + // Lookup array to determine if an index is in the sub-map std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); std::for_each(indices.begin(), indices.end(), @@ -167,6 +198,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // other processes must own them in the submap. std::vector recv_owners(send_disp.back()); + std::vector submap_dest; const int rank = dolfinx::MPI::rank(comm); { bool owners_changed = false; @@ -194,11 +226,13 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // process remains its owner in the submap. Otherwise, add the process // that sent it to the list of possible owners. if (is_in_submap[idx_local]) + { global_idx_to_possible_owner[idx].push_back(rank); + submap_dest.push_back(dest[i]); + } else { owners_changed = true; - global_idx_to_possible_owner[idx].push_back(dest[i]); } } @@ -208,8 +242,14 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, throw std::runtime_error("Index owner change detected!"); } - // Choose the submap owner for each index in `recv_indices` - std::vector send_owners; + std::sort(submap_dest.begin(), submap_dest.end()); + submap_dest.erase(std::unique(submap_dest.begin(), submap_dest.end()), + submap_dest.end()); + submap_dest.shrink_to_fit(); + + // Choose the submap owner for each index in `recv_indices` + std::vector + send_owners; send_owners.reserve(recv_indices.size()); for (auto idx : recv_indices) { @@ -229,6 +269,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_owners.push_back(-1); } + ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner + << "\n"; + + ss << "send_owners = " << send_owners << "\n"; + // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( @@ -243,6 +288,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_disp.data(), MPI_INT32_T, comm1); dolfinx::MPI::check_error(comm, ierr); + ss << "recv_owners = " << recv_owners << "\n"; + // Free the communicator ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); @@ -295,8 +342,37 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::sort(submap_owned.begin(), submap_owned.end()); } + // Get submap source ranks + std::vector submap_src(submap_ghost_owners.begin(), + submap_ghost_owners.end()); + std::sort(submap_src.begin(), submap_src.end()); + submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), + submap_src.end()); + submap_src.shrink_to_fit(); + + // Compute submap destination ranks + // NOTE: The call to NBX can be avoided by a two-step communication + // process + // TODO: Can NBX call be avoided by using O^r_p and O^g_p? + // std::vector submap_dest + // = dolfinx::MPI::compute_graph_edges_nbx(comm, submap_src); + // std::sort(submap_dest.begin(), submap_dest.end()); + + ss + << "submap_dest = " << submap_dest << "\n"; + + for (int i = 0; i < dolfinx::MPI::size(comm); ++i) + { + if (i == dolfinx::MPI::rank(comm)) + { + std::cout << ss.str() << "\n"; + } + MPI_Barrier(comm); + } + return {std::move(submap_owned), std::move(submap_ghost), - std::move(submap_ghost_owners)}; + std::move(submap_ghost_owners), std::move(submap_src), + std::move(submap_dest)}; } /// Computes the global indices of ghosts in a submap. @@ -674,7 +750,8 @@ common::create_sub_index_map(const dolfinx::common::IndexMap& imap, // Compute the owned, ghost, and ghost owners of submap indices. // NOTE: All indices are local and numbered w.r.t. the original (imap) index // map - auto [submap_owned, submap_ghost, submap_ghost_owners] + auto [submap_owned, submap_ghost, submap_ghost_owners, submap_src, + submap_dest] = compute_submap_indices(imap, indices, allow_owner_change); // Compute submap offset for this rank @@ -684,20 +761,6 @@ common::create_sub_index_map(const dolfinx::common::IndexMap& imap, MPI_SUM, comm); dolfinx::MPI::check_error(comm, ierr); - // Get submap source ranks - std::vector submap_src(submap_ghost_owners.begin(), - submap_ghost_owners.end()); - std::sort(submap_src.begin(), submap_src.end()); - submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), - submap_src.end()); - - // Compute submap destination ranks - // NOTE: The call to NBX can be avoided by a two-step communication process - // TODO: Can NBX call be avoided by using O^r_p and O^g_p? - std::vector submap_dest - = dolfinx::MPI::compute_graph_edges_nbx(comm, submap_src); - std::sort(submap_dest.begin(), submap_dest.end()); - // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); imap.local_to_global(submap_ghost, submap_ghost_global); @@ -714,6 +777,7 @@ common::create_sub_index_map(const dolfinx::common::IndexMap& imap, sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), submap_ghost.end()); + // TODO Change constructor return {IndexMap(comm, submap_local_size, submap_ghost_gidxs, submap_ghost_owners), std::move(sub_imap_to_imap)}; From 49ff4ffe311e0473847fcd2d6c141d853960e6a8 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 14:03:24 +0000 Subject: [PATCH 081/101] Tidy --- cpp/dolfinx/common/IndexMap.cpp | 45 +++++---------------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index fb624340905..9868023602b 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,34 +14,9 @@ #include #include -#include - using namespace dolfinx; using namespace dolfinx::common; -template -std::ostream& operator<<(std::ostream& os, const std::vector& vector) -{ - os << "{ "; - for (auto v : vector) - { - os << v << " "; - } - os << "}"; - return os; -} - -// A function for printing maps -template -std::ostream& operator<<(std::ostream& os, const std::map& map) -{ - os << "{ "; - for (const auto& [k, v] : map) - os << k << ": " << v << " "; - os << "}"; - return os; -} - namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -157,13 +132,16 @@ communicate_ghosts_to_owners(const MPI_Comm comm, /// their owning process but included on sharing processes to be /// included in the submap. These indices will be owned by one of the sharing /// processes in the submap. +/// @param[in] submap_src Submap source ranks (processes that own the ghost indices +/// of this rank in the submap) +/// @param[in] submap_dest Submap dest ranks (processes that ghost the indices owned +/// by this rank in the submap) /// @pre `indices` must be sorted and unique. /// @return The owned, ghost, and ghost owners in the submap. All indices are /// local and with respect to the original index map. -// TODO Update docs std::tuple, std::vector, - std::vector, std::vector, - std::vector> + std::vector, std::vector, + std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) @@ -350,17 +328,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, submap_src.end()); submap_src.shrink_to_fit(); - // Compute submap destination ranks - // NOTE: The call to NBX can be avoided by a two-step communication - // process - // TODO: Can NBX call be avoided by using O^r_p and O^g_p? - // std::vector submap_dest - // = dolfinx::MPI::compute_graph_edges_nbx(comm, submap_src); - // std::sort(submap_dest.begin(), submap_dest.end()); - - ss - << "submap_dest = " << submap_dest << "\n"; - for (int i = 0; i < dolfinx::MPI::size(comm); ++i) { if (i == dolfinx::MPI::rank(comm)) From 3aad60813adc25f59df3d935f67a46f0c37262e1 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 14:20:14 +0000 Subject: [PATCH 082/101] Tidy --- cpp/dolfinx/common/IndexMap.cpp | 51 +++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 9868023602b..57c1e684af3 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,9 +14,37 @@ #include #include +#include + using namespace dolfinx; using namespace dolfinx::common; +// A function for printing vectors +template +std::ostream &operator<<(std::ostream &os, + const std::vector &vector) +{ + os << "{ "; + for (auto v : vector) + { + os << v << " "; + } + os << "}"; + return os; +} + +// A function for printing maps +template +std::ostream &operator<<(std::ostream &os, + const std::map &map) +{ + os << "{ "; + for (const auto &[k, v] : map) + os << k << ": " << v << " "; + os << "}"; + return os; +} + namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -132,16 +160,16 @@ communicate_ghosts_to_owners(const MPI_Comm comm, /// their owning process but included on sharing processes to be /// included in the submap. These indices will be owned by one of the sharing /// processes in the submap. -/// @param[in] submap_src Submap source ranks (processes that own the ghost indices -/// of this rank in the submap) -/// @param[in] submap_dest Submap dest ranks (processes that ghost the indices owned -/// by this rank in the submap) +/// @param[in] submap_src Submap source ranks (processes that own the ghost +/// indices of this rank in the submap) +/// @param[in] submap_dest Submap dest ranks (processes that ghost the indices +/// owned by this rank in the submap) /// @pre `indices` must be sorted and unique. /// @return The owned, ghost, and ghost owners in the submap. All indices are /// local and with respect to the original index map. std::tuple, std::vector, - std::vector, std::vector, - std::vector> + std::vector, std::vector, + std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) @@ -176,7 +204,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // other processes must own them in the submap. std::vector recv_owners(send_disp.back()); - std::vector submap_dest; + std::vector submap_dest; const int rank = dolfinx::MPI::rank(comm); { bool owners_changed = false; @@ -225,9 +253,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, submap_dest.end()); submap_dest.shrink_to_fit(); - // Choose the submap owner for each index in `recv_indices` - std::vector - send_owners; + // Choose the submap owner for each index in `recv_indices` + std::vector send_owners; send_owners.reserve(recv_indices.size()); for (auto idx : recv_indices) { @@ -321,8 +348,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } // Get submap source ranks - std::vector submap_src(submap_ghost_owners.begin(), - submap_ghost_owners.end()); + std::vector submap_src(submap_ghost_owners.begin(), + submap_ghost_owners.end()); std::sort(submap_src.begin(), submap_src.end()); submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), submap_src.end()); From 7384371c76579233123c1ba2f70f116e00ac2512 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 15:13:06 +0000 Subject: [PATCH 083/101] Tidy --- cpp/dolfinx/common/IndexMap.cpp | 48 --------------------------------- 1 file changed, 48 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 57c1e684af3..848ab512662 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -14,37 +14,9 @@ #include #include -#include - using namespace dolfinx; using namespace dolfinx::common; -// A function for printing vectors -template -std::ostream &operator<<(std::ostream &os, - const std::vector &vector) -{ - os << "{ "; - for (auto v : vector) - { - os << v << " "; - } - os << "}"; - return os; -} - -// A function for printing maps -template -std::ostream &operator<<(std::ostream &os, - const std::map &map) -{ - os << "{ "; - for (const auto &[k, v] : map) - os << k << ": " << v << " "; - os << "}"; - return os; -} - namespace { std::array, 2> build_src_dest(MPI_Comm comm, @@ -174,14 +146,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) { - std::stringstream ss; - const MPI_Comm comm = imap.comm(); std::span src = imap.src(); std::span dest = imap.dest(); - ss << "rank " << dolfinx::MPI::rank(comm) << ":\n"; - // Lookup array to determine if an index is in the sub-map std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); std::for_each(indices.begin(), indices.end(), @@ -274,11 +242,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_owners.push_back(-1); } - ss << "global_idx_to_possible_owner = " << global_idx_to_possible_owner - << "\n"; - - ss << "send_owners = " << send_owners << "\n"; - // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( @@ -293,8 +256,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_disp.data(), MPI_INT32_T, comm1); dolfinx::MPI::check_error(comm, ierr); - ss << "recv_owners = " << recv_owners << "\n"; - // Free the communicator ierr = MPI_Comm_free(&comm1); dolfinx::MPI::check_error(comm, ierr); @@ -355,15 +316,6 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, submap_src.end()); submap_src.shrink_to_fit(); - for (int i = 0; i < dolfinx::MPI::size(comm); ++i) - { - if (i == dolfinx::MPI::rank(comm)) - { - std::cout << ss.str() << "\n"; - } - MPI_Barrier(comm); - } - return {std::move(submap_owned), std::move(submap_ghost), std::move(submap_ghost_owners), std::move(submap_src), std::move(submap_dest)}; From 1c0d032710c8c5a9d537b6bd1798c6b2e6c9bc23 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 15:22:15 +0000 Subject: [PATCH 084/101] Tidy --- cpp/dolfinx/common/IndexMap.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 848ab512662..2a67aa4194b 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -229,14 +229,9 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Check the index is in the submap, otherwise send -1 if (idx != -1) { - // TODO Choose new owner randomly for load balancing - const std::vector& possible_owners - = global_idx_to_possible_owner[idx]; - // const int random_index = std::rand() % possible_owners.size(); - // send_owners.push_back(possible_owners[random_index]); - - // FIXME TEMPORARILY CHOOSE THE FIRST PROCESS, MAKE RANDOM - send_owners.push_back(possible_owners[0]); + // NOTE Could choose new owner in a way that is is better for + // load balancing, though the impact is probably only very small + send_owners.push_back(global_idx_to_possible_owner[idx][0]); } else send_owners.push_back(-1); From 10a97f7c65702f2a4c3e8bf80dcb6d8d3fb6b312 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 15:25:23 +0000 Subject: [PATCH 085/101] Use more efficient constructor for index map --- cpp/dolfinx/common/IndexMap.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2a67aa4194b..2b7e2d45aa2 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -718,9 +718,8 @@ common::create_sub_index_map(const dolfinx::common::IndexMap& imap, sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), submap_ghost.end()); - // TODO Change constructor - return {IndexMap(comm, submap_local_size, submap_ghost_gidxs, - submap_ghost_owners), + return {IndexMap(comm, submap_local_size, {submap_src, submap_dest}, + submap_ghost_gidxs, submap_ghost_owners), std::move(sub_imap_to_imap)}; } From 79f5f8b7a54b9e44fade5a624ed9fbbe8faa89aa Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 16:07:53 +0000 Subject: [PATCH 086/101] Add docs --- cpp/dolfinx/common/IndexMap.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2b7e2d45aa2..6ed36645866 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -33,6 +33,26 @@ std::array, 2> build_src_dest(MPI_Comm comm, return {std::move(src), std::move(dest)}; } +/// This helper function sends ghost indices on a given process to their +/// owning rank, and receives indices owned by this process that are ghosts +/// on other processes. The function returns the data structures used in this +/// common communication pattern. +/// @param[in] comm The communicator +/// @param[in] src Source ranks +/// @param[in] dest Destination ranks +/// @param[in] ghosts Ghost indices +/// @param[in] owners Ghost owners +/// @param[in] include_ghost A list of the same length as `ghosts` whose ith +/// entry must be non-zero to include ghost[i], otherwise the ghost will be +/// excluded +/// @return 1) The ghost indices packed for communication +/// 2) The received indices +/// 3) A map relating the position of a ghost in the packed data to +/// to its position in `ghosts` +/// 4) The number of indices to send to each process +/// 5) The number of indices received by each process +/// 6) The send displacements +/// 7) The received displacements std::tuple, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector> From 82fa3a87d0a17936973a9b97171d2ea25a210444 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 16:12:29 +0000 Subject: [PATCH 087/101] Change name --- python/test/unit/common/test_index_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 9c01d3ad41d..16254a9ba59 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -113,7 +113,7 @@ def test_index_map_ghost_lifetime(): # TODO: Add test for case where more than one two process shares an index # whose owner changes in the submap -def test_create_submap_connected(): +def test_create_submap_owner_change(): """ Test create_sub_index_map. The diagram illustrates the case with four processes. Original map numbering and connectivity (G indicates a ghost From 3683712d1ddcd7d469ca18ad25e78b2973773b23 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Fri, 15 Dec 2023 16:15:09 +0000 Subject: [PATCH 088/101] Improve description --- python/test/unit/common/test_index_map.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 16254a9ba59..22f7a228299 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -115,7 +115,8 @@ def test_index_map_ghost_lifetime(): # whose owner changes in the submap def test_create_submap_owner_change(): """ - Test create_sub_index_map. The diagram illustrates the case with four + Test create_sub_index_map where the ownership of indices is not + preserved in the submap. The diagram illustrates the case with four processes. Original map numbering and connectivity (G indicates a ghost index): Global Rank 0 Rank 1 Rank 2 Rank 3 From f1645a92c2d9f8d2e09ffb2b623ac7884870a924 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Mon, 18 Dec 2023 12:10:01 +0000 Subject: [PATCH 089/101] Work on docs --- cpp/dolfinx/common/IndexMap.cpp | 2 +- cpp/dolfinx/common/IndexMap.h | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 6ed36645866..3376aab10d0 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -232,7 +232,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } } - if (owners_changed && !allow_owner_change) + if (owners_changed and !allow_owner_change) throw std::runtime_error("Index owner change detected!"); } diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index a95925022f2..697dd46bb9b 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -54,21 +54,26 @@ stack_index_maps( const std::vector< std::pair, int>>& maps); -/// @brief Create a new index map from a subset of indices in an index -/// map. -/// @param[in] imap The index map -/// @param[in] indices Local indices to include in the new index map (owned -/// and ghost) -/// @param[in] allow_owner_change Allows indices that are not included by -/// their owning process but included on sharing processes to be -/// included in the submap. These indices will be owned by one of the sharing -/// processes in the submap. -/// @pre `indices` must be sorted and contain no duplicates. -/// @return The (i) new index map and (ii) a map from local indices in the -/// submap to local indices in the original (this) map +/// @brief Create a new index map from a subset of indices in an +/// existing index map. +/// +/// @param[in] imap Parent map to create a new sub-map from. +/// @param[in] indices Local indices in `imap` (owned and ghost) to +/// include in the new index map. +/// @param[in] allow_owner_change If `true`, indices that are not +/// included in `indices` by their owning process can be included in +/// `indices` by processes that ghost the indices to be included in the +/// new submap. These indices will be owned by one of the sharing +/// processes in the submap. If `false`, and exception is raised if an +/// index is included by a sharing process and not by the owning +/// process. +/// @return The (i) new index map and (ii) a map from local indices in +/// the submap to local indices in the original (this) map. +/// @pre `indices` must be sorted and must not contain duplicates. std::pair> -create_sub_index_map(const IndexMap& imap, std::span indices, - bool allow_owner_change = false); +create_sub_index_map(const IndexMap& imap, + std::span indices, + bool allow_owner_change = false); /// This class represents the distribution index arrays across /// processes. An index array is a contiguous collection of N+1 indices From 8cd900cfee487cd3dc54caa362ed90d4d1175619 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 11:13:48 +0000 Subject: [PATCH 090/101] Consistency fix --- cpp/dolfinx/common/IndexMap.cpp | 113 +++++++++++++++-------------- cpp/dolfinx/common/IndexMap.h | 8 +- cpp/dolfinx/common/Scatterer.h | 9 ++- cpp/dolfinx/fem/DirichletBC.cpp | 14 ++-- cpp/dolfinx/fem/dofmapbuilder.cpp | 7 +- cpp/dolfinx/io/VTKFile.cpp | 2 +- cpp/dolfinx/io/vtk_utils.h | 2 +- cpp/dolfinx/io/xdmf_mesh.cpp | 2 +- cpp/dolfinx/la/MatrixCSR.h | 19 +++-- cpp/dolfinx/la/SparsityPattern.cpp | 15 ++-- cpp/dolfinx/mesh/Topology.cpp | 9 ++- cpp/dolfinx/refinement/utils.cpp | 8 +- python/dolfinx/wrappers/common.cpp | 11 ++- 13 files changed, 112 insertions(+), 107 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 3376aab10d0..f751f53c492 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -7,6 +7,7 @@ #include "IndexMap.h" #include "sort.h" #include +#include #include #include #include @@ -34,17 +35,17 @@ std::array, 2> build_src_dest(MPI_Comm comm, } /// This helper function sends ghost indices on a given process to their -/// owning rank, and receives indices owned by this process that are ghosts -/// on other processes. The function returns the data structures used in this -/// common communication pattern. -/// @param[in] comm The communicator -/// @param[in] src Source ranks -/// @param[in] dest Destination ranks -/// @param[in] ghosts Ghost indices -/// @param[in] owners Ghost owners -/// @param[in] include_ghost A list of the same length as `ghosts` whose ith -/// entry must be non-zero to include ghost[i], otherwise the ghost will be -/// excluded +/// owning rank, and receives indices owned by this process that are +/// ghosts on other processes. The function returns the data structures +/// used in this common communication pattern. +/// @param[in] comm The communicator (global). +/// @param[in] src Source ranks. +/// @param[in] dest Destination ranks. +/// @param[in] ghosts Ghost indices. +/// @param[in] owners Owning rank for each entry in `ghosts`. +/// @param[in] include_ghost A list of the same length as `ghosts` whose +/// ith entry must be non-zero (true) to include `ghost[i]`, otherwise +/// the ghost will be excluded /// @return 1) The ghost indices packed for communication /// 2) The received indices /// 3) A map relating the position of a ghost in the packed data to @@ -53,15 +54,16 @@ std::array, 2> build_src_dest(MPI_Comm comm, /// 5) The number of indices received by each process /// 6) The send displacements /// 7) The received displacements +/// @pre `src` must be sorted and unique +/// @pre `dest` must be sorted and unique std::tuple, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector> -communicate_ghosts_to_owners(const MPI_Comm comm, - std::span src, - std::span dest, +communicate_ghosts_to_owners(const MPI_Comm comm, std::span src, + std::span dest, std::span ghosts, std::span owners, - std::span include_ghost) + std::span include_ghost) { // Send ghost indices to owning rank std::vector send_indices, recv_indices; @@ -83,7 +85,7 @@ communicate_ghosts_to_owners(const MPI_Comm comm, { auto it = std::lower_bound(src.begin(), src.end(), owners[i]); assert(it != src.end() and *it == owners[i]); - int r = std::distance(src.begin(), it); + std::size_t r = std::distance(src.begin(), it); if (include_ghost[i]) { send_data[r].push_back(ghosts[i]); @@ -143,22 +145,23 @@ communicate_ghosts_to_owners(const MPI_Comm comm, std::move(recv_disp)}; } -/// Given an index map and a subset of local indices (can be owned or ghost but -/// must be unique and sorted), computes the owned, ghost and ghost owners in -/// the submap. +/// Given an index map and a subset of local indices (can be owned or +/// ghost but must be unique and sorted), computes the owned, ghost and +/// ghost owners in the submap. /// @param[in] imap An index map. -/// @param[in] indices List of entity indices (indices local to the process). -/// @param[in] allow_owner_change Allows indices that are not included by -/// their owning process but included on sharing processes to be -/// included in the submap. These indices will be owned by one of the sharing -/// processes in the submap. -/// @param[in] submap_src Submap source ranks (processes that own the ghost -/// indices of this rank in the submap) -/// @param[in] submap_dest Submap dest ranks (processes that ghost the indices -/// owned by this rank in the submap) +/// @param[in] indices List of entity indices (indices local to the +/// process). +/// @param[in] allow_owner_change Allows indices that are not included +/// by their owning process but included on sharing processes to be +/// included in the submap. These indices will be owned by one of the +/// sharing processes in the submap. +/// @param[in] submap_src Submap source ranks (processes that own the +/// ghost indices of this rank in the submap) +/// @param[in] submap_dest Submap dest ranks (processes that ghost the +/// indices owned by this rank in the submap) /// @pre `indices` must be sorted and unique. -/// @return The owned, ghost, and ghost owners in the submap. All indices are -/// local and with respect to the original index map. +/// @return The owned, ghost, and ghost owners in the submap. All +/// indices are local and with respect to the original index map. std::tuple, std::vector, std::vector, std::vector, std::vector> @@ -167,13 +170,14 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, bool allow_owner_change) { const MPI_Comm comm = imap.comm(); - std::span src = imap.src(); - std::span dest = imap.dest(); + std::span src = imap.src(); + std::span dest = imap.dest(); // Lookup array to determine if an index is in the sub-map - std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); + std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), + 0); std::for_each(indices.begin(), indices.end(), - [&is_in_submap](int i) { is_in_submap[i] = 1; }); + [&is_in_submap](auto i) { is_in_submap[i] = 1; }); // --- Step 1 ---: Send ghost indices in `indices` to their owners // and receive indices owned by this process that are in `indices` @@ -182,8 +186,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, send_disp, recv_disp] = communicate_ghosts_to_owners( imap.comm(), imap.src(), imap.dest(), imap.ghosts(), imap.owners(), - std::span(is_in_submap.cbegin() + imap.size_local(), - is_in_submap.cend())); + std::span(is_in_submap.cbegin() + imap.size_local(), + is_in_submap.cend())); // --- Step 2 ---: Create a map from the indices in `recv_indices` (i.e. // indices owned by this process that are in `indices` on other processes) to @@ -339,12 +343,14 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, /// Computes the global indices of ghosts in a submap. /// @param[in] submap_src The submap source ranks /// @param[in] submap_dest The submap destination ranks -/// @param[in] submap_owned Owned submap indices (local w.r.t. original index -/// map) -/// @param[in] submap_ghosts_global Ghost submap indices (global w.r.t. original +/// @param[in] submap_owned Owned submap indices (local w.r.t. original /// index map) -/// @param[in] submap_ghost_owners The ranks that own the ghosts in the submap -/// @param[in] submap_offset The global offset for this rank in the submap +/// @param[in] submap_ghosts_global Ghost submap indices (global w.r.t. +/// original index map) +/// @param[in] submap_ghost_owners The ranks that own the ghosts in the +/// submap +/// @param[in] submap_offset The global offset for this rank in the +/// submap /// @param[in] imap The original index map /// @pre submap_owned must be sorted and contain no repeated indices std::vector compute_submap_ghost_indices( @@ -363,7 +369,7 @@ std::vector compute_submap_ghost_indices( = communicate_ghosts_to_owners( imap.comm(), submap_src, submap_dest, submap_ghosts_global, submap_ghost_owners, - std::vector(submap_ghosts_global.size(), 1)); + std::vector(submap_ghosts_global.size(), 1)); // --- Step 2 ---: For each received index, compute the submap global index @@ -429,8 +435,8 @@ common::compute_owned_indices(std::span indices, // Assume that indices are sorted and unique assert(std::is_sorted(indices.begin(), indices.end())); - std::vector ghosts = map.ghosts(); - std::vector owners = map.owners(); + std::span ghosts = map.ghosts(); + std::vector owners(map.owners().begin(), map.owners().end()); // Find first index that is not owned by this rank std::int32_t size_local = map.size_local(); @@ -454,8 +460,8 @@ common::compute_owned_indices(std::span indices, std::sort(global_indices.begin(), global_indices.end()); std::sort(ghost_owners.begin(), ghost_owners.end()); - const std::vector& dest = map.dest(); - const std::vector& src = map.src(); + std::span dest = map.dest(); + std::span src = map.src(); // Count number of ghost per destination std::vector send_sizes(src.size(), 0); @@ -557,9 +563,8 @@ common::stack_index_maps( std::set dest_set; for (auto& [map, _] : maps) { - const std::vector& _src = map.get().src(); - const std::vector& _dest = map.get().dest(); - + std::span _src = map.get().src(); + std::span _dest = map.get().dest(); src_set.insert(_src.begin(), _src.end()); dest_set.insert(_dest.begin(), _dest.end()); } @@ -592,8 +597,8 @@ common::stack_index_maps( { const int bs = maps[m].second; const common::IndexMap& map = maps[m].first.get(); - const std::vector& ghosts = map.ghosts(); - const std::vector& owners = map.owners(); + std::span ghosts = map.ghosts(); + std::span owners = map.owners(); // For each owning rank (on comm), create vector of this rank's // ghosts @@ -829,7 +834,7 @@ std::int32_t IndexMap::size_local() const noexcept //----------------------------------------------------------------------------- std::int64_t IndexMap::size_global() const noexcept { return _size_global; } //----------------------------------------------------------------------------- -const std::vector& IndexMap::ghosts() const noexcept +std::span IndexMap::ghosts() const noexcept { return _ghosts; } @@ -1190,9 +1195,9 @@ std::vector IndexMap::shared_indices() const return shared; } //----------------------------------------------------------------------------- -const std::vector& IndexMap::src() const noexcept { return _src; } +std::span IndexMap::src() const noexcept { return _src; } //----------------------------------------------------------------------------- -const std::vector& IndexMap::dest() const noexcept { return _dest; } +std::span IndexMap::dest() const noexcept { return _dest; } //----------------------------------------------------------------------------- bool IndexMap::overlapped() const noexcept { return _overlapping; } //----------------------------------------------------------------------------- diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 697dd46bb9b..dd2b2e1298f 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -164,7 +164,7 @@ class IndexMap /// Local-to-global map for ghosts (local indexing beyond end of local /// range) - const std::vector& ghosts() const noexcept; + std::span ghosts() const noexcept; /// @brief Return the MPI communicator that the map is defined on. /// @return Communicator @@ -192,7 +192,7 @@ class IndexMap /// @brief The ranks that own each ghost index. /// @return List of ghost owners. The owning rank of the ith ghost /// index is `owners()[i]`. - const std::vector& owners() const { return _owners; } + std::span owners() const { return _owners; } /// @todo Aim to remove this function? /// @@ -213,7 +213,7 @@ class IndexMap /// /// @return MPI ranks than own ghost indices. The ranks are unique /// and sorted. - const std::vector& src() const noexcept; + std::span src() const noexcept; /// @brief Ordered set of MPI ranks that ghost indices owned by /// caller. @@ -222,7 +222,7 @@ class IndexMap /// /// @return MPI ranks than own ghost indices. The ranks are unique /// and sorted. - const std::vector& dest() const noexcept; + std::span dest() const noexcept; /// @brief Check if index map has overlaps (ghosts on any rank). /// diff --git a/cpp/dolfinx/common/Scatterer.h b/cpp/dolfinx/common/Scatterer.h index 500372b1c1b..80387c0050b 100644 --- a/cpp/dolfinx/common/Scatterer.h +++ b/cpp/dolfinx/common/Scatterer.h @@ -47,8 +47,9 @@ class Scatterer /// `map` that will be scattered/gathered. /// @param[in] alloc The memory allocator for indices. Scatterer(const IndexMap& map, int bs, const Allocator& alloc = Allocator()) - : _bs(bs), _remote_inds(0, alloc), _local_inds(0, alloc), _src(map.src()), - _dest(map.dest()) + : _bs(bs), _remote_inds(0, alloc), _local_inds(0, alloc), + _src(map.src().begin(), map.src().end()), + _dest(map.dest().begin(), map.dest().end()) { if (map.overlapped()) { @@ -72,14 +73,14 @@ class Scatterer _comm1 = dolfinx::MPI::Comm(comm1, false); // Build permutation array that sorts ghost indices by owning rank - const std::vector& owners = map.owners(); + std::span owners = map.owners(); std::vector perm(owners.size()); std::iota(perm.begin(), perm.end(), 0); dolfinx::argsort_radix(owners, perm); // Sort (i) ghost indices and (ii) ghost index owners by rank // (using perm array) - const std::vector& ghosts = map.ghosts(); + std::span ghosts = map.ghosts(); std::vector owners_sorted(owners.size()); std::vector ghosts_sorted(owners.size()); std::transform(perm.begin(), perm.end(), owners_sorted.begin(), diff --git a/cpp/dolfinx/fem/DirichletBC.cpp b/cpp/dolfinx/fem/DirichletBC.cpp index 610cd41425d..e1ad135fbdb 100644 --- a/cpp/dolfinx/fem/DirichletBC.cpp +++ b/cpp/dolfinx/fem/DirichletBC.cpp @@ -147,7 +147,7 @@ get_remote_dofs(MPI_Comm comm, const common::IndexMap& map, int bs_map, // Build vector of local dof indices that have been marked by another // process const std::array range = map.local_range(); - const std::vector& ghosts = map.ghosts(); + std::span ghosts = map.ghosts(); // Build map from ghost global index to local position // NOTE: Should we use map here or just one vector with ghosts and @@ -265,14 +265,12 @@ std::vector fem::locate_dofs_topological( // Create 'symmetric' neighbourhood communicator MPI_Comm comm; { - const std::vector& src = map->src(); - const std::vector& dest = map->dest(); - + std::span src = map->src(); + std::span dest = map->dest(); std::vector ranks; std::set_union(src.begin(), src.end(), dest.begin(), dest.end(), std::back_inserter(ranks)); ranks.erase(std::unique(ranks.begin(), ranks.end()), ranks.end()); - MPI_Dist_graph_create_adjacent( map->comm(), ranks.size(), ranks.data(), MPI_UNWEIGHTED, ranks.size(), ranks.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm); @@ -389,14 +387,12 @@ std::array, 2> fem::locate_dofs_topological( // Create 'symmetric' neighbourhood communicator MPI_Comm comm; { - const std::vector& src = map0->src(); - const std::vector& dest = map0->dest(); - + std::span src = map0->src(); + std::span dest = map0->dest(); std::vector ranks; std::set_union(src.begin(), src.end(), dest.begin(), dest.end(), std::back_inserter(ranks)); ranks.erase(std::unique(ranks.begin(), ranks.end()), ranks.end()); - MPI_Dist_graph_create_adjacent(map0->comm(), ranks.size(), ranks.data(), MPI_UNWEIGHTED, ranks.size(), ranks.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, diff --git a/cpp/dolfinx/fem/dofmapbuilder.cpp b/cpp/dolfinx/fem/dofmapbuilder.cpp index 3653fd25fc6..8bcc7b63b54 100644 --- a/cpp/dolfinx/fem/dofmapbuilder.cpp +++ b/cpp/dolfinx/fem/dofmapbuilder.cpp @@ -496,9 +496,8 @@ std::pair, std::vector> get_global_indices( auto map = topology.index_map(d); if (map) { - const std::vector& src = map->src(); - const std::vector& dest = map->dest(); - + std::span src = map->src(); + std::span dest = map->dest(); MPI_Dist_graph_create_adjacent( map->comm(), src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm[d]); @@ -550,7 +549,7 @@ std::pair, std::vector> get_global_indices( MPI_Waitany(requests_dim.size(), requests.data(), &idx, MPI_STATUS_IGNORE); d = requests_dim[idx]; - const std::vector& src = topology.index_map(d)->src(); + std::span src = topology.index_map(d)->src(); // Build (global old, global new) map for dofs of dimension d std::vector>> diff --git a/cpp/dolfinx/io/VTKFile.cpp b/cpp/dolfinx/io/VTKFile.cpp index 6ef87185509..307cc350020 100644 --- a/cpp/dolfinx/io/VTKFile.cpp +++ b/cpp/dolfinx/io/VTKFile.cpp @@ -274,7 +274,7 @@ void add_mesh(std::span x, std::array /*xshape*/, max_idx -= 1; if (!cellmap.ghosts().empty()) { - auto& ghosts = cellmap.ghosts(); + std::span ghosts = cellmap.ghosts(); auto minmax = std::minmax_element(ghosts.begin(), ghosts.end()); min_idx = std::min(min_idx, *minmax.first); max_idx = std::max(max_idx, *minmax.second); diff --git a/cpp/dolfinx/io/vtk_utils.h b/cpp/dolfinx/io/vtk_utils.h index 0840853e464..bcd609d0f58 100644 --- a/cpp/dolfinx/io/vtk_utils.h +++ b/cpp/dolfinx/io/vtk_utils.h @@ -155,7 +155,7 @@ tabulate_lagrange_dof_coordinates(const fem::FunctionSpace& V) std::array range = map_dofs->local_range(); std::int32_t size_local = range[1] - range[0]; std::iota(x_id.begin(), std::next(x_id.begin(), size_local), range[0]); - const std::vector& ghosts = map_dofs->ghosts(); + std::span ghosts = map_dofs->ghosts(); std::copy(ghosts.begin(), ghosts.end(), std::next(x_id.begin(), size_local)); // Ghosts diff --git a/cpp/dolfinx/io/xdmf_mesh.cpp b/cpp/dolfinx/io/xdmf_mesh.cpp index 9e9caa5f3f9..33378de2a2c 100644 --- a/cpp/dolfinx/io/xdmf_mesh.cpp +++ b/cpp/dolfinx/io/xdmf_mesh.cpp @@ -68,7 +68,7 @@ void xdmf_mesh::add_topology_data(MPI_Comm comm, pugi::xml_node& xml_node, assert(map_g); const std::int64_t offset_g = map_g->local_range()[0]; - const std::vector& ghosts = map_g->ghosts(); + std::span ghosts = map_g->ghosts(); const std::vector vtk_map = io::cells::transpose( io::cells::perm_vtk(entity_cell_type, num_nodes_per_entity)); auto map_e = topology.index_map(dim); diff --git a/cpp/dolfinx/la/MatrixCSR.h b/cpp/dolfinx/la/MatrixCSR.h index dc5e45666d3..e3260c2a735 100644 --- a/cpp/dolfinx/la/MatrixCSR.h +++ b/cpp/dolfinx/la/MatrixCSR.h @@ -419,9 +419,10 @@ MatrixCSR::MatrixCSR(const SparsityPattern& p, BlockMode mode) { const auto im = _index_maps[i]; const int size_local = im->size_local() * _bs[i]; - const std::vector& ghost_i = im->ghosts(); + std::span ghost_i = im->ghosts(); std::vector ghosts; - const std::vector ghost_owner_i = im->owners(); + const std::vector ghost_owner_i(im->owners().begin(), + im->owners().end()); std::vector src_rank; for (std::size_t j = 0; j < ghost_i.size(); ++j) { @@ -432,7 +433,10 @@ MatrixCSR::MatrixCSR(const SparsityPattern& p, BlockMode mode) } } const std::array, 2> src_dest0 - = {_index_maps[i]->src(), _index_maps[i]->dest()}; + = {std::vector(_index_maps[i]->src().begin(), + _index_maps[i]->src().end()), + std::vector(_index_maps[i]->dest().begin(), + _index_maps[i]->dest().end())}; _index_maps[i] = std::make_shared( _index_maps[i]->comm(), size_local, src_dest0, ghosts, src_rank); } @@ -444,7 +448,6 @@ MatrixCSR::MatrixCSR(const SparsityPattern& p, BlockMode mode) rowptr_container_type new_row_ptr = {0}; new_row_ptr.reserve(_row_ptr.size() * _bs[0]); std::span num_diag_nnz = p.off_diagonal_offsets(); - for (std::size_t i = 0; i < _row_ptr.size() - 1; ++i) { // Repeat row _bs[0] times @@ -479,11 +482,11 @@ MatrixCSR::MatrixCSR(const SparsityPattern& p, BlockMode mode) = {_index_maps[0]->size_local(), _index_maps[1]->size_local()}; const std::array local_range = {_index_maps[0]->local_range(), _index_maps[1]->local_range()}; - const std::vector& ghosts1 = _index_maps[1]->ghosts(); + std::span ghosts1 = _index_maps[1]->ghosts(); - const std::vector& ghosts0 = _index_maps[0]->ghosts(); - const std::vector& src_ranks = _index_maps[0]->src(); - const std::vector& dest_ranks = _index_maps[0]->dest(); + std::span ghosts0 = _index_maps[0]->ghosts(); + std::span src_ranks = _index_maps[0]->src(); + std::span dest_ranks = _index_maps[0]->dest(); // Create neighbourhood communicator (owner <- ghost) MPI_Comm comm; diff --git a/cpp/dolfinx/la/SparsityPattern.cpp b/cpp/dolfinx/la/SparsityPattern.cpp index e9c81839b60..ff9f4cf303f 100644 --- a/cpp/dolfinx/la/SparsityPattern.cpp +++ b/cpp/dolfinx/la/SparsityPattern.cpp @@ -233,17 +233,18 @@ void SparsityPattern::finalize() assert(_index_maps[0]); const std::int32_t local_size0 = _index_maps[0]->size_local(); const std::array local_range0 = _index_maps[0]->local_range(); - const std::vector& ghosts0 = _index_maps[0]->ghosts(); - - const std::vector& owners0 = _index_maps[0]->owners(); - const std::vector& src0 = _index_maps[0]->src(); + std::span ghosts0 = _index_maps[0]->ghosts(); + std::span owners0 = _index_maps[0]->owners(); + std::span src0 = _index_maps[0]->src(); assert(_index_maps[1]); const std::int32_t local_size1 = _index_maps[1]->size_local(); const std::array local_range1 = _index_maps[1]->local_range(); - _col_ghosts = _index_maps[1]->ghosts(); - _col_ghost_owners = _index_maps[1]->owners(); + _col_ghosts.assign(_index_maps[1]->ghosts().begin(), + _index_maps[1]->ghosts().end()); + _col_ghost_owners.assign(_index_maps[1]->owners().begin(), + _index_maps[1]->owners().end()); // Compute size of data to send to each process std::vector send_sizes(src0.size(), 0); @@ -297,7 +298,7 @@ void SparsityPattern::finalize() std::vector ghost_data_in; { MPI_Comm comm; - const std::vector& dest0 = _index_maps[0]->dest(); + std::span dest0 = _index_maps[0]->dest(); MPI_Dist_graph_create_adjacent( _index_maps[0]->comm(), dest0.size(), dest0.data(), MPI_UNWEIGHTED, src0.size(), src0.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm); diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index 3f80de04f7b..ab128daf59b 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -516,8 +516,8 @@ exchange_ghost_indexing(const common::IndexMap& map0, // owner. MPI_Comm comm; - const std::vector& src = map0.src(); - const std::vector& dest = map0.dest(); + std::span src = map0.src(); + std::span dest = map0.dest(); MPI_Dist_graph_create_adjacent(map0.comm(), src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm); @@ -1174,7 +1174,7 @@ mesh::create_subtopology(const Topology& topology, int dim, unique_entities.end()); std::pair> map_data = dolfinx::common::create_sub_index_map(*topology.index_map(dim), - unique_entities); + unique_entities); submap = std::make_shared(std::move(map_data.first)); subentities = std::move(map_data.second); } @@ -1194,7 +1194,8 @@ mesh::create_subtopology(const Topology& topology, int dim, { std::pair> map_data = dolfinx::common::create_sub_index_map( - *map0, compute_incident_entities(topology, subentities, dim, 0), true); + *map0, compute_incident_entities(topology, subentities, dim, 0), + true); submap0 = std::make_shared(std::move(map_data.first)); subvertices0 = std::move(map_data.second); } diff --git a/cpp/dolfinx/refinement/utils.cpp b/cpp/dolfinx/refinement/utils.cpp index 5b76b2b6425..29655dd31d5 100644 --- a/cpp/dolfinx/refinement/utils.cpp +++ b/cpp/dolfinx/refinement/utils.cpp @@ -37,7 +37,7 @@ std::int64_t refinement::impl::local_to_global(std::int32_t local_index, } else { - const std::vector& ghosts = map.ghosts(); + std::span ghosts = map.ghosts(); assert((local_index - local_size) < (int)ghosts.size()); return ghosts[local_index - local_size]; } @@ -111,9 +111,9 @@ refinement::adjust_indices(const common::IndexMap& map, std::int32_t n) std::int64_t global_offset = 0; MPI_Exscan(&num_local, &global_offset, 1, MPI_INT64_T, MPI_SUM, map.comm()); - const std::vector& owners = map.owners(); - const std::vector& src = map.src(); - const std::vector& dest = map.dest(); + std::span owners = map.owners(); + std::span src = map.src(); + std::span dest = map.dest(); MPI_Comm comm; MPI_Dist_graph_create_adjacent(map.comm(), src.size(), src.data(), diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 3a967de098e..76419042c9d 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -108,7 +108,7 @@ void common(nb::module_& m) "ghosts", [](const dolfinx::common::IndexMap& self) { - const std::vector& ghosts = self.ghosts(); + std::span ghosts = self.ghosts(); return nb::ndarray(ghosts.data(), {ghosts.size()}); }, @@ -117,7 +117,7 @@ void common(nb::module_& m) "owners", [](const dolfinx::common::IndexMap& self) { - const std::vector& owners = self.owners(); + std::span owners = self.owners(); return nb::ndarray>( owners.data(), {owners.size()}); }, @@ -128,9 +128,7 @@ void common(nb::module_& m) nb::ndarray, nb::c_contig> local) { std::vector global(local.size()); - self.local_to_global( - std::span(local.data(), local.size()), - std::span(global.data(), global.size())); + self.local_to_global(std::span(local.data(), local.size()), global); return global; }, nb::arg("local")); @@ -180,7 +178,8 @@ void common(nb::module_& m) bool allow_owner_change) { auto [map, submap_to_map] = dolfinx::common::create_sub_index_map( - imap, std::span(indices.data(), indices.size()), allow_owner_change); + imap, std::span(indices.data(), indices.size()), + allow_owner_change); return std::pair(std::move(map), dolfinx_wrappers::as_nbarray( std::move(submap_to_map))); }, From 16fe9a67e09d553655e89eabe8c1eec9a6dd35ad Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 11:33:44 +0000 Subject: [PATCH 091/101] Remove std::map --- cpp/dolfinx/common/IndexMap.cpp | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index f751f53c492..2e348928fe1 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -205,8 +205,10 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // FIXME Avoid map // A map from (global) indices in `recv_indices` to a list of processes that // can own the index in the submap. - std::map> - global_idx_to_possible_owner; + // std::map> + // global_idx_to_possible_owner; + std::vector> + _global_idx_to_possible_owner; const std::array local_range = imap.local_range(); for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { @@ -225,13 +227,15 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // that sent it to the list of possible owners. if (is_in_submap[idx_local]) { - global_idx_to_possible_owner[idx].push_back(rank); + // global_idx_to_possible_owner[idx].push_back(rank); + _global_idx_to_possible_owner.push_back({idx, rank}); submap_dest.push_back(dest[i]); } else { owners_changed = true; - global_idx_to_possible_owner[idx].push_back(dest[i]); + // global_idx_to_possible_owner[idx].push_back(dest[i]); + _global_idx_to_possible_owner.push_back({idx, dest[i]}); } } } @@ -240,22 +244,30 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, throw std::runtime_error("Index owner change detected!"); } + std::sort(_global_idx_to_possible_owner.begin(), + _global_idx_to_possible_owner.end()); + std::sort(submap_dest.begin(), submap_dest.end()); submap_dest.erase(std::unique(submap_dest.begin(), submap_dest.end()), submap_dest.end()); submap_dest.shrink_to_fit(); // Choose the submap owner for each index in `recv_indices` - std::vector send_owners; + std::vector send_owners; send_owners.reserve(recv_indices.size()); - for (auto idx : recv_indices) + for (std::int64_t idx : recv_indices) { // Check the index is in the submap, otherwise send -1 if (idx != -1) { - // NOTE Could choose new owner in a way that is is better for + // NOTE: Could choose new owner in a way that is is better for // load balancing, though the impact is probably only very small - send_owners.push_back(global_idx_to_possible_owner[idx][0]); + auto it = std::lower_bound(_global_idx_to_possible_owner.begin(), + _global_idx_to_possible_owner.end(), idx, + [](auto a, auto b) { return a.first < b; }); + assert(it != _global_idx_to_possible_owner.end() and it->first == idx); + send_owners.push_back(it->second); + // send_owners.push_back(global_idx_to_possible_owner[idx][0]); } else send_owners.push_back(-1); @@ -869,7 +881,6 @@ void IndexMap::global_to_local(std::span global, global_local_ghosts[i] = {_ghosts[i], i + local_size}; std::map global_to_local( global_local_ghosts.begin(), global_local_ghosts.end()); - std::transform(global.begin(), global.end(), local.begin(), [range = _local_range, &global_to_local](std::int64_t index) -> std::int32_t From e906b0cf5f4ec663aec3e16a81eba4c4394ace6a Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 13:37:33 +0000 Subject: [PATCH 092/101] Tidy up --- cpp/dolfinx/common/IndexMap.cpp | 64 ++++++++++++++++----------------- cpp/dolfinx/common/MPI.cpp | 2 -- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 2e348928fe1..e688a44a0a6 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -202,13 +202,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, bool owners_changed = false; // Create a map from (global) owned shared indices owned to processes that // can own them. - // FIXME Avoid map + // A map from (global) indices in `recv_indices` to a list of processes that // can own the index in the submap. - // std::map> - // global_idx_to_possible_owner; std::vector> - _global_idx_to_possible_owner; + global_idx_to_possible_owner; const std::array local_range = imap.local_range(); for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { @@ -222,20 +220,19 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, assert(idx_local >= 0); assert(idx_local < local_range[1]); - // Check if index is in the submap on this process. If so, this - // process remains its owner in the submap. Otherwise, add the process - // that sent it to the list of possible owners. + // Check if index is in the submap on this process. If so, + // this process remains its owner in the submap. Otherwise, + // add the process that sent it to the list of possible + // owners. if (is_in_submap[idx_local]) { - // global_idx_to_possible_owner[idx].push_back(rank); - _global_idx_to_possible_owner.push_back({idx, rank}); + global_idx_to_possible_owner.push_back({idx, rank}); submap_dest.push_back(dest[i]); } else { owners_changed = true; - // global_idx_to_possible_owner[idx].push_back(dest[i]); - _global_idx_to_possible_owner.push_back({idx, dest[i]}); + global_idx_to_possible_owner.push_back({idx, dest[i]}); } } } @@ -244,8 +241,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, throw std::runtime_error("Index owner change detected!"); } - std::sort(_global_idx_to_possible_owner.begin(), - _global_idx_to_possible_owner.end()); + std::sort(global_idx_to_possible_owner.begin(), + global_idx_to_possible_owner.end()); std::sort(submap_dest.begin(), submap_dest.end()); submap_dest.erase(std::unique(submap_dest.begin(), submap_dest.end()), @@ -262,12 +259,11 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, { // NOTE: Could choose new owner in a way that is is better for // load balancing, though the impact is probably only very small - auto it = std::lower_bound(_global_idx_to_possible_owner.begin(), - _global_idx_to_possible_owner.end(), idx, + auto it = std::lower_bound(global_idx_to_possible_owner.begin(), + global_idx_to_possible_owner.end(), idx, [](auto a, auto b) { return a.first < b; }); - assert(it != _global_idx_to_possible_owner.end() and it->first == idx); + assert(it != global_idx_to_possible_owner.end() and it->first == idx); send_owners.push_back(it->second); - // send_owners.push_back(global_idx_to_possible_owner[idx][0]); } else send_owners.push_back(-1); @@ -292,17 +288,19 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, dolfinx::MPI::check_error(comm, ierr); } - // --- Step 3 --- : Determine the owned indices, ghost indices, and ghost - // owners in the submap + // --- Step 3 --- : Determine the owned indices, ghost indices, and + // ghost owners in the submap - // Local indices (w.r.t. original map) owned by this process in the submap + // Local indices (w.r.t. original map) owned by this process in the + // submap std::vector submap_owned; - // Local indices (w.r.t. original map) ghosted by this process in the submap + // Local indices (w.r.t. original map) ghosted by this process in the + // submap std::vector submap_ghost; - // The owners of the submap ghost indices (process submap_ghost_owners[i] owns - // index submap_ghost[i]) + // The owners of the submap ghost indices (process + // submap_ghost_owners[i] owns index submap_ghost[i]) std::vector submap_ghost_owners; { @@ -311,14 +309,14 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, if (v < imap.size_local()) submap_owned.push_back(v); - // FIXME Could just create when making send_indices + // FIXME: Could just create when making send_indices std::vector send_indices_local(send_indices.size()); imap.global_to_local(send_indices, send_indices_local); - // Loop over ghost indices (in the original map) and add to submap_owned - // if the owning process has decided this process to be the submap owner. - // Else, add the index and its (possibly new) owner to submap_ghost and - // submap_ghost_owners respectively. + // Loop over ghost indices (in the original map) and add to + // submap_owned if the owning process has decided this process to be + // the submap owner. Else, add the index and its (possibly new) + // owner to submap_ghost and submap_ghost_owners respectively. for (std::size_t i = 0; i < send_indices.size(); ++i) { // Check if index is in the submap @@ -326,9 +324,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, { std::int32_t local_idx = send_indices_local[i]; if (std::int32_t owner = recv_owners[i]; owner == rank) - { submap_owned.push_back(local_idx); - } else { submap_ghost.push_back(local_idx); @@ -372,9 +368,9 @@ std::vector compute_submap_ghost_indices( std::span submap_ghost_owners, const int submap_offset, const dolfinx::common::IndexMap& imap) { - // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to owning - // rank - const MPI_Comm comm = imap.comm(); + // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to + // owning rank + MPI_Comm comm = imap.comm(); auto [send_indices, recv_indices, ghost_perm, send_sizes, recv_sizes, send_disp, recv_disp] @@ -522,7 +518,7 @@ common::compute_owned_indices(std::span indices, recv_disp.data(), MPI_INT64_T, comm); dolfinx::MPI::check_error(comm, ierr); ierr = MPI_Comm_free(&comm); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(map.comm(), ierr); // Remove duplicates from received indices std::sort(recv_buffer.begin(), recv_buffer.end()); diff --git a/cpp/dolfinx/common/MPI.cpp b/cpp/dolfinx/common/MPI.cpp index cc3e7c9d617..784629e0cf5 100644 --- a/cpp/dolfinx/common/MPI.cpp +++ b/cpp/dolfinx/common/MPI.cpp @@ -85,10 +85,8 @@ void dolfinx::MPI::check_error(MPI_Comm comm, int code) std::string error_string(MPI_MAX_ERROR_STRING, ' '); MPI_Error_string(code, error_string.data(), &len); error_string.resize(len); - std::cerr << error_string << std::endl; MPI_Abort(comm, code); - std::abort(); } } From 6aae46a4b3541cc338190fd6fe8023950c88640e Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 13:48:33 +0000 Subject: [PATCH 093/101] Remove a map --- cpp/dolfinx/common/IndexMap.cpp | 14 +++++++------- cpp/dolfinx/common/IndexMap.h | 33 ++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index e688a44a0a6..c01d68ec2bc 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -870,13 +869,12 @@ void IndexMap::global_to_local(std::span global, std::span local) const { const std::int32_t local_size = _local_range[1] - _local_range[0]; - - std::vector> global_local_ghosts( + std::vector> global_to_local( _ghosts.size()); for (std::size_t i = 0; i < _ghosts.size(); ++i) - global_local_ghosts[i] = {_ghosts[i], i + local_size}; - std::map global_to_local( - global_local_ghosts.begin(), global_local_ghosts.end()); + global_to_local[i] = {_ghosts[i], i + local_size}; + + std::sort(global_to_local.begin(), global_to_local.end()); std::transform(global.begin(), global.end(), local.begin(), [range = _local_range, &global_to_local](std::int64_t index) -> std::int32_t @@ -885,7 +883,9 @@ void IndexMap::global_to_local(std::span global, return index - range[0]; else { - auto it = global_to_local.find(index); + auto it = std::lower_bound( + global_to_local.begin(), global_to_local.end(), index, + [](auto a, auto b) { return a.first < b; }); return it != global_to_local.end() ? it->second : -1; } }); diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index dd2b2e1298f..dbc653a172a 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -19,10 +19,10 @@ namespace dolfinx::common // Forward declaration class IndexMap; -/// @brief Given a sorted vector of indices (local numbering, owned or ghost) -/// and an index map, this function returns the indices owned by this process, -/// including indices that might have been in the list of indices on another -/// processes. +/// @brief Given a sorted vector of indices (local numbering, owned or +/// ghost) and an index map, this function returns the indices owned by +/// this process, including indices that might have been in the list of +/// indices on another processes. /// @param[in] indices List of indices /// @param[in] map The index map /// @return Indices owned by the calling process @@ -76,11 +76,11 @@ create_sub_index_map(const IndexMap& imap, bool allow_owner_change = false); /// This class represents the distribution index arrays across -/// processes. An index array is a contiguous collection of N+1 indices -/// [0, 1, . . ., N] that are distributed across M processes. On a given -/// process, the IndexMap stores a portion of the index set using local -/// indices [0, 1, . . . , n], and a map from the local indices to a -/// unique global index. +/// processes. An index array is a contiguous collection of `N+1` +/// indices `[0, 1, . . ., N]` that are distributed across `M` +/// processes. On a given process, the IndexMap stores a portion of the +/// index set using local indices `[0, 1, . . . , n]`, and a map from +/// the local indices to a unique global index. class IndexMap { public: @@ -88,9 +88,10 @@ class IndexMap /// /// @note Collective /// - /// @param[in] comm The MPI communicator + /// @param[in] comm MPI communicator that the index map is distributed + /// across. /// @param[in] local_size Local size of the index map, i.e. the number - /// of owned entries + /// of owned entries. IndexMap(MPI_Comm comm, std::int32_t local_size); /// @brief Create an overlapping (ghosted) index map. @@ -103,7 +104,8 @@ class IndexMap /// /// @note Collective /// - /// @param[in] comm The MPI communicator + /// @param[in] comm MPI communicator that the index map is distributed + /// across. /// @param[in] local_size Local size of the index map, i.e. the number /// of owned entries /// @param[in] ghosts The global indices of ghost entries @@ -122,7 +124,8 @@ class IndexMap /// /// @note Collective /// - /// @param[in] comm The MPI communicator + /// @param[in] comm MPI communicator that the index map is distributed + /// across. /// @param[in] local_size Local size of the index map, i.e. the number /// @param[in] src_dest Lists of [0] src and [1] dest ranks. The list /// in each must be sorted and not contain duplicates. `src` ranks are @@ -176,7 +179,7 @@ class IndexMap void local_to_global(std::span local, std::span global) const; - /// @brief Compute local indices for array of global indices + /// @brief Compute local indices for array of global indices. /// @param[in] global Global indices /// @param[out] local The local of the corresponding global index in /// 'global'. Returns -1 if the local index does not exist on this @@ -185,7 +188,7 @@ class IndexMap std::span local) const; /// @brief Build list of indices with global indexing. - /// @return The global index for all local indices (0, 1, 2, ...) on + /// @return The global index for all local indices `(0, 1, 2, ...)` on /// this process, including ghosts std::vector global_indices() const; From 3640a3c5c3a0e3481c4d49e5b33c067562fc8543 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 14:32:18 +0000 Subject: [PATCH 094/101] Simplifications and fixes --- cpp/dolfinx/common/IndexMap.cpp | 30 ++++++++++++-------------- cpp/dolfinx/mesh/Geometry.h | 9 ++++---- cpp/dolfinx/mesh/Topology.cpp | 4 ++-- python/dolfinx/wrappers/common.cpp | 10 ++++----- python/test/unit/la/test_matrix_csr.py | 2 +- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index c01d68ec2bc..1a78afa3a19 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -58,7 +58,7 @@ std::array, 2> build_src_dest(MPI_Comm comm, std::tuple, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector> -communicate_ghosts_to_owners(const MPI_Comm comm, std::span src, +communicate_ghosts_to_owners(MPI_Comm comm, std::span src, std::span dest, std::span ghosts, std::span owners, @@ -159,11 +159,11 @@ communicate_ghosts_to_owners(const MPI_Comm comm, std::span src, /// @param[in] submap_dest Submap dest ranks (processes that ghost the /// indices owned by this rank in the submap) /// @pre `indices` must be sorted and unique. -/// @return The owned, ghost, and ghost owners in the submap. All +/// @return The (1) owned, (2) ghost and (3) ghost owners in the submap, +/// and (4) submap src ranks and (5) submap destination ranks. All /// indices are local and with respect to the original index map. std::tuple, std::vector, - std::vector, std::vector, - std::vector> + std::vector, std::vector, std::vector> compute_submap_indices(const dolfinx::common::IndexMap& imap, std::span indices, bool allow_owner_change) @@ -194,8 +194,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // `recv_indices` will necessarily be in `indices` on this process, and thus // other processes must own them in the submap. - std::vector recv_owners(send_disp.back()); - std::vector submap_dest; + std::vector recv_owners(send_disp.back()), submap_dest; const int rank = dolfinx::MPI::rank(comm); { bool owners_changed = false; @@ -204,8 +203,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // A map from (global) indices in `recv_indices` to a list of processes that // can own the index in the submap. - std::vector> - global_idx_to_possible_owner; + std::vector> global_idx_to_possible_owner; const std::array local_range = imap.local_range(); for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { @@ -277,9 +275,9 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // Send the data ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), - recv_disp.data(), MPI_INT32_T, - recv_owners.data(), send_sizes.data(), - send_disp.data(), MPI_INT32_T, comm1); + recv_disp.data(), MPI_INT, recv_owners.data(), + send_sizes.data(), send_disp.data(), MPI_INT, + comm1); dolfinx::MPI::check_error(comm, ierr); // Free the communicator @@ -300,7 +298,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, // The owners of the submap ghost indices (process // submap_ghost_owners[i] owns index submap_ghost[i]) - std::vector submap_ghost_owners; + std::vector submap_ghost_owners; { // Add owned indices to submap_owned @@ -322,7 +320,7 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, if (std::int64_t global_idx = send_indices[i]; global_idx >= 0) { std::int32_t local_idx = send_indices_local[i]; - if (std::int32_t owner = recv_owners[i]; owner == rank) + if (int owner = recv_owners[i]; owner == rank) submap_owned.push_back(local_idx); else { @@ -335,8 +333,8 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, } // Get submap source ranks - std::vector submap_src(submap_ghost_owners.begin(), - submap_ghost_owners.end()); + std::vector submap_src(submap_ghost_owners.begin(), + submap_ghost_owners.end()); std::sort(submap_src.begin(), submap_src.end()); submap_src.erase(std::unique(submap_src.begin(), submap_src.end()), submap_src.end()); @@ -364,7 +362,7 @@ std::vector compute_submap_ghost_indices( std::span submap_src, std::span submap_dest, std::span submap_owned, std::span submap_ghosts_global, - std::span submap_ghost_owners, const int submap_offset, + std::span submap_ghost_owners, int submap_offset, const dolfinx::common::IndexMap& imap) { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index 4d3126ed4ee..8c490c7a888 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -332,11 +332,10 @@ create_subgeometry(const Topology& topology, const Geometry& geometry, std::shared_ptr sub_x_dof_index_map; std::vector subx_to_x_dofmap; { - std::pair> map_data - = dolfinx::common::create_sub_index_map(*x_index_map, sub_x_dofs, true); - sub_x_dof_index_map - = std::make_shared(std::move(map_data.first)); - subx_to_x_dofmap = std::move(map_data.second); + auto [map, new_to_old] + = common::create_sub_index_map(*x_index_map, sub_x_dofs, true); + sub_x_dof_index_map = std::make_shared(std::move(map)); + subx_to_x_dofmap = std::move(new_to_old); } // Create sub-geometry coordinates diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index ab128daf59b..622a8c9a900 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1173,8 +1173,8 @@ mesh::create_subtopology(const Topology& topology, int dim, std::unique(unique_entities.begin(), unique_entities.end()), unique_entities.end()); std::pair> map_data - = dolfinx::common::create_sub_index_map(*topology.index_map(dim), - unique_entities); + = common::create_sub_index_map(*topology.index_map(dim), + unique_entities); submap = std::make_shared(std::move(map_data.first)); subentities = std::move(map_data.second); } diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 76419042c9d..58e9ad99b0d 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -100,10 +100,10 @@ void common(nb::module_& m) .def_prop_ro("num_ghosts", &dolfinx::common::IndexMap::num_ghosts) .def_prop_ro("local_range", &dolfinx::common::IndexMap::local_range, "Range of indices owned by this map") - .def_prop_ro("index_to_dest_ranks", - &dolfinx::common::IndexMap::index_to_dest_ranks) - .def_prop_ro("imbalance", &dolfinx::common::IndexMap::imbalance, - "Imbalance of the current IndexMap.") + .def("index_to_dest_ranks", + &dolfinx::common::IndexMap::index_to_dest_ranks) + .def("imbalance", &dolfinx::common::IndexMap::imbalance, + "Imbalance of the current IndexMap.") .def_prop_ro( "ghosts", [](const dolfinx::common::IndexMap& self) @@ -129,7 +129,7 @@ void common(nb::module_& m) { std::vector global(local.size()); self.local_to_global(std::span(local.data(), local.size()), global); - return global; + return dolfinx_wrappers::as_nbarray(std::move(global)); }, nb::arg("local")); diff --git a/python/test/unit/la/test_matrix_csr.py b/python/test/unit/la/test_matrix_csr.py index ad1e809bf9c..48cb2837962 100644 --- a/python/test/unit/la/test_matrix_csr.py +++ b/python/test/unit/la/test_matrix_csr.py @@ -190,7 +190,7 @@ def test_set_diagonal_distributed(dtype): nlocal = index_map.size_local assert (diag[nlocal:] == dtype(0.0)).all() - shared_dofs = index_map.index_to_dest_ranks + shared_dofs = index_map.index_to_dest_ranks() for dof in range(nlocal): owners = shared_dofs.links(dof) assert diag[dof] == len(owners) + 1 From 60c50b37311b7c4c56ff0514d22faaab1557cdbd Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 15:16:42 +0000 Subject: [PATCH 095/101] Simplify --- cpp/dolfinx/common/IndexMap.cpp | 23 ++++++++++++----------- cpp/dolfinx/mesh/Topology.cpp | 20 +++++++++----------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 1a78afa3a19..0261d6ea2a8 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -164,7 +164,7 @@ communicate_ghosts_to_owners(MPI_Comm comm, std::span src, /// indices are local and with respect to the original index map. std::tuple, std::vector, std::vector, std::vector, std::vector> -compute_submap_indices(const dolfinx::common::IndexMap& imap, +compute_submap_indices(const IndexMap& imap, std::span indices, bool allow_owner_change) { @@ -358,12 +358,13 @@ compute_submap_indices(const dolfinx::common::IndexMap& imap, /// submap /// @param[in] imap The original index map /// @pre submap_owned must be sorted and contain no repeated indices -std::vector compute_submap_ghost_indices( - std::span submap_src, std::span submap_dest, - std::span submap_owned, - std::span submap_ghosts_global, - std::span submap_ghost_owners, int submap_offset, - const dolfinx::common::IndexMap& imap) +std::vector +compute_submap_ghost_indices(std::span submap_src, + std::span submap_dest, + std::span submap_owned, + std::span submap_ghosts_global, + std::span submap_ghost_owners, + int submap_offset, const IndexMap& imap) { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to // owning rank @@ -545,8 +546,8 @@ std::tuple, std::vector>, std::vector>> common::stack_index_maps( - const std::vector< - std::pair, int>>& maps) + const std::vector, int>>& + maps) { // Compute process offset for stacked index map const std::int64_t process_offset = std::accumulate( @@ -601,7 +602,7 @@ common::stack_index_maps( for (std::size_t m = 0; m < maps.size(); ++m) { const int bs = maps[m].second; - const common::IndexMap& map = maps[m].first.get(); + const IndexMap& map = maps[m].first.get(); std::span ghosts = map.ghosts(); std::span owners = map.owners(); @@ -712,7 +713,7 @@ common::stack_index_maps( } //----------------------------------------------------------------------------- std::pair> -common::create_sub_index_map(const dolfinx::common::IndexMap& imap, +common::create_sub_index_map(const IndexMap& imap, std::span indices, bool allow_owner_change) { diff --git a/cpp/dolfinx/mesh/Topology.cpp b/cpp/dolfinx/mesh/Topology.cpp index 622a8c9a900..f353ef0c7e8 100644 --- a/cpp/dolfinx/mesh/Topology.cpp +++ b/cpp/dolfinx/mesh/Topology.cpp @@ -1167,16 +1167,14 @@ mesh::create_subtopology(const Topology& topology, int dim, std::vector subentities; { // FIXME Make this an input requirement? - std::vector unique_entities(entities.begin(), entities.end()); - std::sort(unique_entities.begin(), unique_entities.end()); - unique_entities.erase( - std::unique(unique_entities.begin(), unique_entities.end()), - unique_entities.end()); - std::pair> map_data - = common::create_sub_index_map(*topology.index_map(dim), - unique_entities); - submap = std::make_shared(std::move(map_data.first)); - subentities = std::move(map_data.second); + std::vector _entities(entities.begin(), entities.end()); + std::sort(_entities.begin(), _entities.end()); + _entities.erase(std::unique(_entities.begin(), _entities.end()), + _entities.end()); + auto [_submap, _subentities] + = common::create_sub_index_map(*topology.index_map(dim), _entities); + submap = std::make_shared(std::move(_submap)); + subentities = std::move(_subentities); } // Get the vertices in the sub-topology. Use subentities @@ -1193,7 +1191,7 @@ mesh::create_subtopology(const Topology& topology, int dim, std::vector subvertices0; { std::pair> map_data - = dolfinx::common::create_sub_index_map( + = common::create_sub_index_map( *map0, compute_incident_entities(topology, subentities, dim, 0), true); submap0 = std::make_shared(std::move(map_data.first)); From 3b761cdc27f44cd8917bdccc18beb08b1e2370a2 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 15:25:50 +0000 Subject: [PATCH 096/101] Improvements --- cpp/dolfinx/common/IndexMap.cpp | 8 ++++---- cpp/dolfinx/mesh/Geometry.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 0261d6ea2a8..043f877086b 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -1217,12 +1217,12 @@ std::array IndexMap::imbalance() const // Find the maximum number of owned indices and the maximum number of ghost // indices across all processes. - MPI_Allreduce(local_sizes.data(), max_count.data(), 2, - dolfinx::MPI::mpi_type(), MPI_MAX, _comm.comm()); + MPI_Allreduce(local_sizes.data(), max_count.data(), 2, MPI_INT32_T, MPI_MAX, + _comm.comm()); std::int32_t total_num_ghosts = 0; - MPI_Allreduce(&local_sizes[1], &total_num_ghosts, 1, - dolfinx::MPI::mpi_type(), MPI_SUM, _comm.comm()); + MPI_Allreduce(&local_sizes[1], &total_num_ghosts, 1, MPI_INT32_T, MPI_SUM, + _comm.comm()); // Compute the average number of owned and ghost indices per process. int comm_size = dolfinx::MPI::size(_comm.comm()); diff --git a/cpp/dolfinx/mesh/Geometry.h b/cpp/dolfinx/mesh/Geometry.h index 8c490c7a888..f0b63a58cc5 100644 --- a/cpp/dolfinx/mesh/Geometry.h +++ b/cpp/dolfinx/mesh/Geometry.h @@ -175,7 +175,7 @@ class Geometry /// @param[in] reorder_fn Function for re-ordering the degree-of-freedom /// map associated with the geometry data template -mesh::Geometry> +Geometry> create_geometry( MPI_Comm comm, const Topology& topology, const std::vector -std::pair, std::vector> +std::pair, std::vector> create_subgeometry(const Topology& topology, const Geometry& geometry, int dim, std::span subentity_to_entity) { From 66c05ad5f961c34dbf8a6bf51d995bd373d3655e Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 15:27:38 +0000 Subject: [PATCH 097/101] Small fix for docs --- cpp/dolfinx/common/IndexMap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index dbc653a172a..5ec93c329d9 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -51,8 +51,8 @@ std::tuple, std::vector>, std::vector>> stack_index_maps( - const std::vector< - std::pair, int>>& maps); + const std::vector, int>>& + maps); /// @brief Create a new index map from a subset of indices in an /// existing index map. From 5fab0e10990a990789c6b99cebb236bbdff65ff8 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 19 Dec 2023 15:37:55 +0000 Subject: [PATCH 098/101] Updates --- cpp/dolfinx/common/IndexMap.cpp | 39 ++++++++++------------- python/test/unit/common/test_index_map.py | 26 +++------------ 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 043f877086b..519e4bc0659 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -26,10 +26,8 @@ std::array, 2> build_src_dest(MPI_Comm comm, std::sort(src.begin(), src.end()); src.erase(std::unique(src.begin(), src.end()), src.end()); src.shrink_to_fit(); - std::vector dest = dolfinx::MPI::compute_graph_edges_nbx(comm, src); std::sort(dest.begin(), dest.end()); - return {std::move(src), std::move(dest)}; } @@ -168,7 +166,6 @@ compute_submap_indices(const IndexMap& imap, std::span indices, bool allow_owner_change) { - const MPI_Comm comm = imap.comm(); std::span src = imap.src(); std::span dest = imap.dest(); @@ -195,7 +192,7 @@ compute_submap_indices(const IndexMap& imap, // other processes must own them in the submap. std::vector recv_owners(send_disp.back()), submap_dest; - const int rank = dolfinx::MPI::rank(comm); + const int rank = dolfinx::MPI::rank(imap.comm()); { bool owners_changed = false; // Create a map from (global) owned shared indices owned to processes that @@ -269,20 +266,20 @@ compute_submap_indices(const IndexMap& imap, // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( - comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), dest.data(), - MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); - dolfinx::MPI::check_error(comm, ierr); + imap.comm(), src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), + dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); + dolfinx::MPI::check_error(imap.comm(), ierr); // Send the data ierr = MPI_Neighbor_alltoallv(send_owners.data(), recv_sizes.data(), recv_disp.data(), MPI_INT, recv_owners.data(), send_sizes.data(), send_disp.data(), MPI_INT, comm1); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(imap.comm(), ierr); // Free the communicator ierr = MPI_Comm_free(&comm1); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(imap.comm(), ierr); } // --- Step 3 --- : Determine the owned indices, ghost indices, and @@ -368,7 +365,6 @@ compute_submap_ghost_indices(std::span submap_src, { // --- Step 1 ---: Send global ghost indices (w.r.t. original imap) to // owning rank - MPI_Comm comm = imap.comm(); auto [send_indices, recv_indices, ghost_perm, send_sizes, recv_sizes, send_disp, recv_disp] @@ -377,7 +373,8 @@ compute_submap_ghost_indices(std::span submap_src, submap_ghost_owners, std::vector(submap_ghosts_global.size(), 1)); - // --- Step 2 ---: For each received index, compute the submap global index + // --- Step 2 ---: For each received index, compute the submap global + // index std::vector send_gidx; { @@ -407,20 +404,20 @@ compute_submap_ghost_indices(std::span submap_src, // Create neighbourhood comm (owner -> ghost) MPI_Comm comm1; int ierr = MPI_Dist_graph_create_adjacent( - comm, submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, + imap.comm(), submap_src.size(), submap_src.data(), MPI_UNWEIGHTED, submap_dest.size(), submap_dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, &comm1); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(imap.comm(), ierr); // Send indices to ghosting ranks ierr = MPI_Neighbor_alltoallv(send_gidx.data(), recv_sizes.data(), recv_disp.data(), MPI_INT64_T, recv_gidx.data(), send_sizes.data(), send_disp.data(), MPI_INT64_T, comm1); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(imap.comm(), ierr); ierr = MPI_Comm_free(&comm1); - dolfinx::MPI::check_error(comm, ierr); + dolfinx::MPI::check_error(imap.comm(), ierr); } // --- Step 4---: Unpack received data @@ -717,11 +714,9 @@ common::create_sub_index_map(const IndexMap& imap, std::span indices, bool allow_owner_change) { - const MPI_Comm comm = imap.comm(); - // Compute the owned, ghost, and ghost owners of submap indices. - // NOTE: All indices are local and numbered w.r.t. the original (imap) index - // map + // NOTE: All indices are local and numbered w.r.t. the original (imap) + // index map auto [submap_owned, submap_ghost, submap_ghost_owners, submap_src, submap_dest] = compute_submap_indices(imap, indices, allow_owner_change); @@ -730,8 +725,8 @@ common::create_sub_index_map(const IndexMap& imap, std::int64_t submap_local_size = submap_owned.size(); std::int64_t submap_offset = 0; int ierr = MPI_Exscan(&submap_local_size, &submap_offset, 1, MPI_INT64_T, - MPI_SUM, comm); - dolfinx::MPI::check_error(comm, ierr); + MPI_SUM, imap.comm()); + dolfinx::MPI::check_error(imap.comm(), ierr); // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); @@ -749,7 +744,7 @@ common::create_sub_index_map(const IndexMap& imap, sub_imap_to_imap.insert(sub_imap_to_imap.end(), submap_ghost.begin(), submap_ghost.end()); - return {IndexMap(comm, submap_local_size, {submap_src, submap_dest}, + return {IndexMap(imap.comm(), submap_local_size, {submap_src, submap_dest}, submap_ghost_gidxs, submap_ghost_owners), std::move(sub_imap_to_imap)}; } diff --git a/python/test/unit/common/test_index_map.py b/python/test/unit/common/test_index_map.py index 22f7a228299..012f9a54c11 100644 --- a/python/test/unit/common/test_index_map.py +++ b/python/test/unit/common/test_index_map.py @@ -63,19 +63,6 @@ def test_sub_index_map(): subowners = submap.owners assert (owners[ghosts_pos_sub] == subowners).all() - # Check that ghost indices are correct in submap - # NOTE This assumes size_local is the same for all ranks - # TODO Consider renaming to something shorter - # submap_global_to_map_global_map = np.concatenate([local_indices[rank] + map_local_size * rank - # for rank in range(comm.size)]) - # # FIXME Do this more elegantly - # submap_ghosts = [] - # for map_ghost in map.ghosts: - # submap_ghost = np.where(submap_global_to_map_global_map == map_ghost)[0] - # if submap_ghost.size != 0: - # submap_ghosts.append(submap_ghost[0]) - # assert np.allclose(submap.ghosts, submap_ghosts) - def test_sub_index_map_ghost_mode_none(): n = 3 @@ -114,8 +101,7 @@ def test_index_map_ghost_lifetime(): # TODO: Add test for case where more than one two process shares an index # whose owner changes in the submap def test_create_submap_owner_change(): - """ - Test create_sub_index_map where the ownership of indices is not + """Test create_sub_index_map where the ownership of indices is not preserved in the submap. The diagram illustrates the case with four processes. Original map numbering and connectivity (G indicates a ghost index): @@ -164,8 +150,7 @@ def test_create_submap_owner_change(): submap_indices = np.array([0, 2, 3], dtype=np.int32) imap = dolfinx.common.IndexMap(comm, local_size, ghosts, owners) - sub_imap, sub_imap_to_imap = _cpp.common.create_sub_index_map( - imap, submap_indices, True) + sub_imap, sub_imap_to_imap = _cpp.common.create_sub_index_map(imap, submap_indices, True) if comm.rank == 0: assert sub_imap.size_local == 2 @@ -183,8 +168,5 @@ def test_create_submap_owner_change(): assert np.array_equal(sub_imap.owners, [comm.rank + 1]) assert np.array_equal(sub_imap_to_imap, [0, 2, 3]) - global_indices = sub_imap.local_to_global( - np.arange(sub_imap.size_local + sub_imap.num_ghosts, - dtype=np.int32)) - assert np.array_equal( - global_indices, np.arange(comm.rank * 2, comm.rank * 2 + 3)) + global_indices = sub_imap.local_to_global(np.arange(sub_imap.size_local + sub_imap.num_ghosts, dtype=np.int32)) + assert np.array_equal(global_indices, np.arange(comm.rank * 2, comm.rank * 2 + 3)) From a56c51f4252b47a266140290cb734756f0ad7608 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Tue, 26 Dec 2023 12:36:07 +0000 Subject: [PATCH 099/101] Doc improvements --- cpp/dolfinx/common/IndexMap.cpp | 51 +++++++++++++++------------------ cpp/dolfinx/common/IndexMap.h | 8 ++---- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 519e4bc0659..9a450cc4b1a 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -33,24 +33,24 @@ std::array, 2> build_src_dest(MPI_Comm comm, /// This helper function sends ghost indices on a given process to their /// owning rank, and receives indices owned by this process that are -/// ghosts on other processes. The function returns the data structures -/// used in this common communication pattern. +/// ghosts on other processes. It also returns the data structures used +/// in this common communication pattern. /// @param[in] comm The communicator (global). -/// @param[in] src Source ranks. -/// @param[in] dest Destination ranks. -/// @param[in] ghosts Ghost indices. +/// @param[in] src Source ranks on `comm`. +/// @param[in] dest Destination ranks on `comm`. +/// @param[in] ghosts Ghost indices on calling process. /// @param[in] owners Owning rank for each entry in `ghosts`. -/// @param[in] include_ghost A list of the same length as `ghosts` whose +/// @param[in] include_ghost A list of the same length as `ghosts`, whose /// ith entry must be non-zero (true) to include `ghost[i]`, otherwise /// the ghost will be excluded -/// @return 1) The ghost indices packed for communication -/// 2) The received indices -/// 3) A map relating the position of a ghost in the packed data to -/// to its position in `ghosts` -/// 4) The number of indices to send to each process -/// 5) The number of indices received by each process -/// 6) The send displacements -/// 7) The received displacements +/// @return 1) The ghost indices packed in a buffer for communication +/// 2) The received indices (in receive buffer layout) +/// 3) A map relating the position of a ghost in the packed +/// data (1) to to its position in `ghosts`. +/// 4) The number of indices to send to each process. +/// 5) The number of indices received by each process. +/// 6) The send displacements. +/// 7) The received displacements. /// @pre `src` must be sorted and unique /// @pre `dest` must be sorted and unique std::tuple, std::vector, @@ -143,7 +143,7 @@ communicate_ghosts_to_owners(MPI_Comm comm, std::span src, } /// Given an index map and a subset of local indices (can be owned or -/// ghost but must be unique and sorted), computes the owned, ghost and +/// ghost but must be unique and sorted), compute the owned, ghost and /// ghost owners in the submap. /// @param[in] imap An index map. /// @param[in] indices List of entity indices (indices local to the @@ -152,10 +152,6 @@ communicate_ghosts_to_owners(MPI_Comm comm, std::span src, /// by their owning process but included on sharing processes to be /// included in the submap. These indices will be owned by one of the /// sharing processes in the submap. -/// @param[in] submap_src Submap source ranks (processes that own the -/// ghost indices of this rank in the submap) -/// @param[in] submap_dest Submap dest ranks (processes that ghost the -/// indices owned by this rank in the submap) /// @pre `indices` must be sorted and unique. /// @return The (1) owned, (2) ghost and (3) ghost owners in the submap, /// and (4) submap src ranks and (5) submap destination ranks. All @@ -169,7 +165,7 @@ compute_submap_indices(const IndexMap& imap, std::span src = imap.src(); std::span dest = imap.dest(); - // Lookup array to determine if an index is in the sub-map + // Create lookup array to determine if an index is in the sub-map std::vector is_in_submap(imap.size_local() + imap.num_ghosts(), 0); std::for_each(indices.begin(), indices.end(), @@ -178,8 +174,8 @@ compute_submap_indices(const IndexMap& imap, // --- Step 1 ---: Send ghost indices in `indices` to their owners // and receive indices owned by this process that are in `indices` // on other processes - auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, recv_sizes, - send_disp, recv_disp] + const auto [send_indices, recv_indices, ghost_buffer_pos, send_sizes, + recv_sizes, send_disp, recv_disp] = communicate_ghosts_to_owners( imap.comm(), imap.src(), imap.dest(), imap.ghosts(), imap.owners(), std::span(is_in_submap.cbegin() + imap.size_local(), @@ -237,7 +233,6 @@ compute_submap_indices(const IndexMap& imap, std::sort(global_idx_to_possible_owner.begin(), global_idx_to_possible_owner.end()); - std::sort(submap_dest.begin(), submap_dest.end()); submap_dest.erase(std::unique(submap_dest.begin(), submap_dest.end()), submap_dest.end()); @@ -246,7 +241,7 @@ compute_submap_indices(const IndexMap& imap, // Choose the submap owner for each index in `recv_indices` std::vector send_owners; send_owners.reserve(recv_indices.size()); - for (std::int64_t idx : recv_indices) + for (auto idx : recv_indices) { // Check the index is in the submap, otherwise send -1 if (idx != -1) @@ -342,7 +337,7 @@ compute_submap_indices(const IndexMap& imap, std::move(submap_dest)}; } -/// Computes the global indices of ghosts in a submap. +/// Compute the global indices of ghosts in a submap. /// @param[in] submap_src The submap source ranks /// @param[in] submap_dest The submap destination ranks /// @param[in] submap_owned Owned submap indices (local w.r.t. original @@ -717,8 +712,8 @@ common::create_sub_index_map(const IndexMap& imap, // Compute the owned, ghost, and ghost owners of submap indices. // NOTE: All indices are local and numbered w.r.t. the original (imap) // index map - auto [submap_owned, submap_ghost, submap_ghost_owners, submap_src, - submap_dest] + const auto [submap_owned, submap_ghost, submap_ghost_owners, submap_src, + submap_dest] = compute_submap_indices(imap, indices, allow_owner_change); // Compute submap offset for this rank @@ -731,7 +726,7 @@ common::create_sub_index_map(const IndexMap& imap, // Compute the global indices (w.r.t. the submap) of the submap ghosts std::vector submap_ghost_global(submap_ghost.size()); imap.local_to_global(submap_ghost, submap_ghost_global); - auto submap_ghost_gidxs = compute_submap_ghost_indices( + std::vector submap_ghost_gidxs = compute_submap_ghost_indices( submap_src, submap_dest, submap_owned, submap_ghost_global, submap_ghost_owners, submap_offset, imap); diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 5ec93c329d9..6728e3d5570 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -109,8 +109,7 @@ class IndexMap /// @param[in] local_size Local size of the index map, i.e. the number /// of owned entries /// @param[in] ghosts The global indices of ghost entries - /// @param[in] owners Owner rank (on global communicator) of each - /// entry in `ghosts` + /// @param[in] owners Owner rank (on `comm`) of each entry in `ghosts` IndexMap(MPI_Comm comm, std::int32_t local_size, std::span ghosts, std::span owners); @@ -132,8 +131,7 @@ class IndexMap /// owners of the indices in `ghosts`. `dest` ranks are the rank that /// ghost indices owned by the caller. /// @param[in] ghosts The global indices of ghost entries - /// @param[in] owners Owner rank (on global communicator) of each entry - /// in `ghosts` + /// @param[in] owners Owner rank (on `comm`) of each entry in `ghosts` IndexMap(MPI_Comm comm, std::int32_t local_size, const std::array, 2>& src_dest, std::span ghosts, std::span owners); @@ -265,7 +263,7 @@ class IndexMap // Local-to-global map for ghost indices std::vector _ghosts; - // Local-to-global map for ghost indices + // Owning rank on _comm for the ith ghost index std::vector _owners; // Set of ranks that own ghosts From 8a102aea585dcd938ad3510300868ed1dd148f9a Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 2 Jan 2024 12:44:08 +0000 Subject: [PATCH 100/101] Improve docs --- cpp/dolfinx/common/IndexMap.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 9a450cc4b1a..8a714dd8a1d 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -243,7 +243,8 @@ compute_submap_indices(const IndexMap& imap, send_owners.reserve(recv_indices.size()); for (auto idx : recv_indices) { - // Check the index is in the submap, otherwise send -1 + // Check if the index is in the submap. If so, choose its owner in the submap, + // otherwise send -1 if (idx != -1) { // NOTE: Could choose new owner in a way that is is better for From 9faa9260b2df90c2d8a5d72129c06d923711de08 Mon Sep 17 00:00:00 2001 From: Joe Dean Date: Tue, 2 Jan 2024 13:05:16 +0000 Subject: [PATCH 101/101] Improve comments --- cpp/dolfinx/common/IndexMap.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 8a714dd8a1d..7778ef56ee6 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -190,19 +190,20 @@ compute_submap_indices(const IndexMap& imap, std::vector recv_owners(send_disp.back()), submap_dest; const int rank = dolfinx::MPI::rank(imap.comm()); { + // Flag to track if the owner of any indices have changed in the submap bool owners_changed = false; - // Create a map from (global) owned shared indices owned to processes that - // can own them. - // A map from (global) indices in `recv_indices` to a list of processes that - // can own the index in the submap. + // Create a map from (global) indices in `recv_indices` to a list of + // processes that can own them in the submap. std::vector> global_idx_to_possible_owner; const std::array local_range = imap.local_range(); + // Loop through the received indices for (std::size_t i = 0; i < recv_disp.size() - 1; ++i) { for (int j = recv_disp[i]; j < recv_disp[i + 1]; ++j) { - // Check that the index is in the submap + // Check if the index is included in the submap by process dest[i]. + // If so, it must be assigned a submap owner if (std::int64_t idx = recv_indices[j]; idx != -1) { // Compute the local index @@ -210,13 +211,13 @@ compute_submap_indices(const IndexMap& imap, assert(idx_local >= 0); assert(idx_local < local_range[1]); - // Check if index is in the submap on this process. If so, + // Check if index is included in the submap on this process. If so, // this process remains its owner in the submap. Otherwise, - // add the process that sent it to the list of possible - // owners. + // add the process that sent it to the list of possible owners. if (is_in_submap[idx_local]) { global_idx_to_possible_owner.push_back({idx, rank}); + // Add the sending process as a destination rank in the submap submap_dest.push_back(dest[i]); } else @@ -243,8 +244,8 @@ compute_submap_indices(const IndexMap& imap, send_owners.reserve(recv_indices.size()); for (auto idx : recv_indices) { - // Check if the index is in the submap. If so, choose its owner in the submap, - // otherwise send -1 + // Check if the index is in the submap. If so, choose its owner in the + // submap, otherwise send -1 if (idx != -1) { // NOTE: Could choose new owner in a way that is is better for