diff --git a/config/sst_check_metis.m4 b/config/sst_check_metis.m4 new file mode 100644 index 000000000..7bc236f64 --- /dev/null +++ b/config/sst_check_metis.m4 @@ -0,0 +1,53 @@ +AC_DEFUN([SST_CHECK_METIS], +[ + sst_check_metis_happy="yes" + sst_check_metis_requested="yes" + + AC_ARG_WITH([metis], + [AS_HELP_STRING([--with-metis@<:@=DIR@:>@], + [Use Metis package installed in optionally specified DIR])], + [sst_check_metis_requested="yes"], + [sst_check_metis_requested="no"]) + + AS_IF([test "x$with_metis" = "xno"], [sst_check_metis_happy="no"]) + + CPPFLAGS_saved="$CPPFLAGS" + LDFLAGS_saved="$LDFLAGS" + CXX_saved="$CXX" + CC_saved="$CC" + + AS_IF([test "x$MPICXX" != "x"], [CXX="$MPICXX"]) + AS_IF([test "x$MPICC" != "x"], [CC="$MPICC"]) + + AS_IF([test "$sst_check_metis_happy" = "yes"], [ + AS_IF([test ! -z "$with_metis" -a "$with_metis" != "yes"], + [METIS_CPPFLAGS="-I$with_metis/include" + CPPFLAGS="$METIS_CPPFLAGS $CPPFLAGS" + METIS_LDFLAGS="-L$with_metis/lib" + LDFLAGS="$METIS_LDFLAGS $LDFLAGS"], + [METIS_CPPFLAGS= + METIS_LDFLAGS=]) + + AC_CHECK_HEADERS([metis.h], [], [sst_check_metis_happy="no"]) + AC_CHECK_LIB([metis], [METIS_PartGraphKway], + [METIS_LIB="-lmetis"], [sst_check_metis_happy="no"], [-lm])]) + + CPPFLAGS="$CPPFLAGS_saved" + LDFLAGS="$LDFLAGS_saved" + CXX="$CXX_saved" + CC="$CC_saved" + + AC_SUBST([METIS_CPPFLAGS]) + AC_SUBST([METIS_LDFLAGS]) + AC_SUBST([METIS_LIB]) + + AC_MSG_CHECKING([for Metis package]) + AC_MSG_RESULT([$sst_check_metis_happy]) + + # if user doesn't specify --with-metis then string is empty, otherwise is has a value + # if the value is not "no" then we treat as a path, and MUST find in order to be + # successful + AS_IF([test "x$sst_check_metis_requested" = "xyes" -a "x$sst_check_metis_happy" != "xyes"], + [AC_MSG_FAILURE([Metis was requested by configure (--with-metis=$with_metis) but the required libraries and header files could not be found.])]) + AS_IF([test "$sst_check_metis_happy" = "yes"], [$1], [$2]) +]) diff --git a/configure.ac b/configure.ac index 95f786c34..b8a89198e 100644 --- a/configure.ac +++ b/configure.ac @@ -78,6 +78,11 @@ SST_ENABLE_CORE_PROFILE() SST_CHECK_FPIC() +SST_CHECK_METIS([have_metis=1],[have_metis=0],[AC_MSG_ERROR([Metis requested but not found])]) +AS_IF([test "x$have_metis" = "x1"], [AC_DEFINE_UNQUOTED([HAVE_METIS], [1], + [Define if you have the metis library.])]) +AM_CONDITIONAL([HAVE_METIS], [test "$have_metis" = 1]) + AC_DEFINE_UNQUOTED([SST_CPPFLAGS], ["$CPPFLAGS"], [Defines the CPPFLAGS used to build SST]) AC_DEFINE_UNQUOTED([SST_CFLAGS], ["$CFLAGS"], [Defines the CFLAGS used to build SST]) AC_DEFINE_UNQUOTED([SST_CXXFLAGS], ["$CXXFLAGS"], [Defines the CXXFLAGS used to build SST]) @@ -231,6 +236,11 @@ if test "x$PYTHON_VERSION3" = "xyes" ; then else printf "%38s : No\n" "Python3" fi +if test "x$sst_check_metis_happy" = "xyes" ; then + printf "%38s : YES\n" "Metis Partitioner" +else + printf "%38s : No\n" "Metis Partitioner" +fi if test "x$sst_check_hdf5_happy" = "xyes" ; then printf "%38s : Yes\n" "HDF5 Support" else diff --git a/src/sst/core/Makefile.am b/src/sst/core/Makefile.am index 0a44aeacb..0c80fb6f3 100644 --- a/src/sst/core/Makefile.am +++ b/src/sst/core/Makefile.am @@ -9,6 +9,7 @@ AM_CPPFLAGS = \ -DTIXML_USE_STL \ -DSST_BUILDING_CORE=1 \ $(ZOLTAN_CPPFLAGS) \ + $(METIS_CPPFLAGS) \ $(PYTHON_CPPFLAGS) \ -I$(top_srcdir)/external \ $(LTDLINCL) @@ -278,6 +279,7 @@ sstinfo_x_SOURCES = \ sstsim_x_LDADD = \ $(PYTHON_LIBS) \ $(ZOLTAN_LIB) \ + $(METIS_LIB) \ $(PARMETIS_LIB) \ $(MPILIBS) \ $(TCMALLOC_LIB) \ @@ -286,6 +288,7 @@ sstsim_x_LDADD = \ sstsim_x_LDFLAGS = \ $(PARMETIS_LDFLAGS) \ $(ZOLTAN_LDFLAGS) \ + $(METIS_LDFLAGS) \ $(TCMALLOC_LDFLAGS) \ $(PYTHON_LDFLAGS) \ -export-dynamic \ @@ -294,6 +297,7 @@ sstsim_x_LDFLAGS = \ sstinfo_x_LDADD = \ $(ZOLTAN_LIB) \ $(PARMETIS_LIB) \ + $(METIS_LIB) \ $(MPILIBS) \ $(TCMALLOC_LIB) \ -lm @@ -301,6 +305,7 @@ sstinfo_x_LDADD = \ sstinfo_x_LDFLAGS = \ $(PARMETIS_LDFLAGS) \ $(ZOLTAN_LDFLAGS) \ + $(METIS_LDFLAGS) \ $(TCMALLOC_LDFLAGS) \ $(PYTHON_LDFLAGS) \ -export-dynamic \ diff --git a/src/sst/core/impl/partitioners/Makefile.inc b/src/sst/core/impl/partitioners/Makefile.inc index 5589e579b..dee678601 100644 --- a/src/sst/core/impl/partitioners/Makefile.inc +++ b/src/sst/core/impl/partitioners/Makefile.inc @@ -13,3 +13,11 @@ sst_core_sources += \ impl/partitioners/simplepart.h \ impl/partitioners/singlepart.cc \ impl/partitioners/singlepart.h + +if HAVE_METIS +sst_core_sources += \ + impl/partitioners/util.hpp \ + impl/partitioners/csrmat.hpp \ + impl/partitioners/metispart.h \ + impl/partitioners/metispart.cc +endif \ No newline at end of file diff --git a/src/sst/core/impl/partitioners/csrmat.hpp b/src/sst/core/impl/partitioners/csrmat.hpp new file mode 100644 index 000000000..dd8073f87 --- /dev/null +++ b/src/sst/core/impl/partitioners/csrmat.hpp @@ -0,0 +1,271 @@ +#ifndef SST_CORE_IMPL_PARTITONERS_CSRMAT_HPP +#define SST_CORE_IMPL_PARTITONERS_CSRMAT_HPP + +#include +#include +#include +#include +#include +#include + +#include "sst/core/impl/partitioners/util.h" +#include + + class CSRMat { + public: + // construct from components + CSRMat(std::unordered_map> nw, std::unordered_map, double> ew) + : _edges{}, _nodes{}, _node_wgts_map(std::move(nw)), _edge_wgts_map(std::move(ew)) { + if (_node_wgts_map.size() > 0) { + _constraint_number = _node_wgts_map.cbegin()->second.size(); + } + for (const auto& p : _node_wgts_map) { + _nodes.push_back(p.first); + + // Enforce that each vector-value in the node weight map has the same length. + if (p.second.size() != _constraint_number) { + std::string err_msg = + "Error: Two nodes with different number of constraints\n" + " Node " + + std::to_string(_node_wgts_map.cbegin()->first) + " has constraints { "; + for (const double& c : _node_wgts_map.cbegin()->second) { + err_msg += std::to_string(c) + " "; + } + err_msg += "}\n"; + err_msg += " Node " + std::to_string(p.first) + " has constraints { "; + for (const double& c : p.second) { + err_msg += std::to_string(c) + " "; + } + err_msg += "}\n"; + throw std::logic_error(err_msg); + } + } + std::sort(_nodes.begin(), _nodes.end()); + + // use of intermediate edge_sets normalizes all edges to be + // bi-directional + std::unordered_map> edge_sets; + for (const auto& p : _edge_wgts_map) { + int src = p.first.first, dst = p.first.second; + edge_sets[src].insert(dst); + edge_sets[dst].insert(src); + } + for (const auto& p : edge_sets) { + int src = p.first; + for (int dst : p.second) { + _edges[src].push_back(dst); + } + std::sort(_edges[src].begin(), _edges[src].end()); + } + } + + size_t size() const { return _nodes.size(); } + const std::vector& node_ID() const { return _nodes; } + //maps flattend vertex indices to vertex id + int get(unsigned int index) const { return _nodes[index]; } + std::vector node_weight(int id) const { return _node_wgts_map.at(id); } + + std::vector xadj() const { + std::vector result; + result.push_back(0); + for (auto id : _nodes) { + int edge_n = _edges.count(id) ? _edges.at(id).size() : 0; + int idx = result.back() + edge_n; + result.push_back(idx); + } + return result; + } + + std::vector adj() const { + std::vector result; + // mapping from node IDs to index [0, num_nodes) + std::unordered_map mapping; + for (unsigned i = 0; i < _nodes.size(); ++i) { + mapping[_nodes[i]] = i; + } + for (auto src : _nodes) { + if (_edges.count(src)) { + for (auto dst : _edges.at(src)) { + result.push_back(mapping.at(dst)); + } + } + } + return result; + } + + std::size_t constraint_number() const { return _constraint_number; } + + // vector of node weights in order + std::vector node_wgts() const { + std::vector result; + result.reserve(_constraint_number * size()); // pre-allocate + for (auto id : _nodes) { // _nodes is sorted + for (uint c = 0; c < _constraint_number; ++c) { + result.push_back(_node_wgts_map.at(id).at(c)); + } + } + return result; + } + + // returns a vector of edge weights in order + std::vector edge_wgts() const { + std::vector result; + for (auto src : _nodes) { // _nodes is sorted + if (_edges.count(src)) { // node might not be connected to anything + for (auto dst : _edges.at(src)) { // _edges.at(src) is sorted + double w{0}; + auto id = std::make_pair(src, dst); + if (_edge_wgts_map.count(id)) { + w += _edge_wgts_map.at(id); + } + id = std::make_pair(dst, src); + if (_edge_wgts_map.count(id)) { + w += _edge_wgts_map.at(id); + } + result.push_back(w); + } + } + } + return result; + } + + std::vector idxs_to_node_ID(std::vector idxs) const { + std::vector result; + for (auto idx : idxs) { + result.push_back(_nodes[idx]); + } + return result; + } + + friend std::ostream& operator<<(std::ostream& os, const CSRMat& mat) { + os << "CSRMat (" << mat.size() << ")\n"; + auto nw = mat.node_wgts(); + auto ew = mat.edge_wgts(); + int nidx = 0, eidx = 0; + for (auto nid : mat._nodes) { + os << "(" << nid << "," << nw[nidx++] << ") : "; + for (auto eid : mat._edges.at(nid)) { + os << "(" << eid << "," << ew[eidx++] << "),"; + } + os << '\n'; + } + return os; + } + + void csr_info() { + std::cout << "size() " << _nodes.size() << std::endl; + + std::cout << "First ten elements of _nodes \n"; + for (int j = 0; j < 10; ++j) + std::cout << _nodes[j] << std::endl; + + std::cout << "First ten elements of _edges \n"; + for (int j = 0; j < 10; ++j) { + for (int k : _edges[j]) + std::cout << k << " "; + std::cout << "\n"; + } + } + + std::unordered_map> get_edges() { return _edges; } + + private: + std::size_t _constraint_number; + std::unordered_map> _edges; + std::vector _nodes; // sorted node IDs + std::unordered_map> _node_wgts_map; + std::unordered_map, double> _edge_wgts_map; +}; + +template +std::vector to_int64(const std::vector& vec) { + std::vector result; + result.reserve(vec.size()); + for (auto val : vec) { + result.push_back(static_cast(val)); + } + return result; +} + +template +std::vector scale_to_int64(const std::vector& vec) { + double max_aval = 0; + for (auto val : vec) { + if (fabs(val) > max_aval) + max_aval = fabs(val); + } + double target = std::sqrt(double(std::numeric_limits::max())); + double scale_factor = target / max_aval; + std::vector result; + result.reserve(vec.size()); + for (auto val : vec) { + result.push_back(static_cast(scale_factor * val)); + } + return result; +} + +template +void summarize_vec(std::vector vec) { + std::cout << "size=" << vec.size(); + auto min_v = std::numeric_limits::max(); + auto max_v = std::numeric_limits::min(); + for (const auto& v : vec) { + min_v = std::min(v, min_v); + max_v = std::max(v, max_v); + } + std::cout << ", min=" << min_v; + std::cout << ", max=" << max_v; + std::cout << ", ("; + for (size_t i = 0; i < std::min(4ul, vec.size()); ++i) { + std::cout << vec[i] << ", "; + } + std::cout << "..., "; + for (size_t i = std::max(0ul, vec.size() - 4); i < vec.size(); ++i) { + std::cout << vec[i] << ", "; + } + std::cout << "\b\b)"; +} + +std::vector metis_part(const CSRMat& mat, int64_t nparts, const double imba_ratio) { + static_assert(sizeof(idx_t) == sizeof(int64_t), "Requires 64-bit METIS idx_t"); + static_assert(sizeof(real_t) == sizeof(double), "Requires 64-bit METIS real_t"); + + int64_t nvtxs = mat.size(); + + // set up metis parameters + auto ncon = static_cast(mat.constraint_number()); + int64_t objval; + std::vector options(METIS_NOPTIONS), part(nvtxs); + std::vector tpwgts(nparts * ncon, 1.0 / nparts), ubvec(ncon, imba_ratio); + // set up weights vectorse + + std::vector node_wgts = scale_to_int64(mat.node_wgts()), edge_wgts = scale_to_int64(mat.edge_wgts()); + // check parameters + + /*std::cout << "nparts: " << nparts; + std::cout << "nvtx: " << nvtxs; + std::cout << "xadj: " + summarize_vec(mat.xadj()); + std::cout << " adj: "; + summarize_vec(mat.adj()); + std::cout << "ncon: " << ncon;*/ + + // do partitioning + METIS_SetDefaultOptions(options.data()); + METIS_PartGraphKway(&nvtxs, + &ncon, + to_int64(mat.xadj()).data(), + to_int64(mat.adj()).data(), + node_wgts.data(), + nullptr, + edge_wgts.data(), + &nparts, + tpwgts.data(), + ubvec.data(), + options.data(), + &objval, + part.data()); + return part; +} + +#endif diff --git a/src/sst/core/impl/partitioners/metispart.cc b/src/sst/core/impl/partitioners/metispart.cc new file mode 100644 index 000000000..2c4ede094 --- /dev/null +++ b/src/sst/core/impl/partitioners/metispart.cc @@ -0,0 +1,112 @@ +//leave translation unit empty if metis isn't provided + +#include "sst/core/sst_config.h" + +#ifdef HAVE_METIS +#include "sst/core/impl/partitioners/metispart.h" + +#include "sst/core/warnmacros.h" +#include "sst/core/configGraph.h" + +#include "sst/core/impl/partitioners/csrmat.hpp" + +namespace SST { +namespace IMPL { +namespace Partition { + +SSTMetisPartition::SSTMetisPartition(RankInfo world_size, RankInfo UNUSED(my_rank), int verbosity) + : rankcount(world_size), partOutput(new Output("MetisPartition ", verbosity, 0, SST::Output::STDOUT)) +{} + +void SSTMetisPartition::performPartition(PartitionGraph* pgraph) { + assert(rankcount.rank > 0); + assert(partOutput); + + std::unordered_map> node_weights; + std::unordered_map, double> edge_weights; + std::unordered_map component2group; + + //set node weights + PartitionComponentMap_t& components = pgraph->getComponentMap(); + for ( auto iter = components.begin(); iter != components.end(); ++iter) { + node_weights[iter->key()] = std::vector(1,iter->weight); + + for ( auto& component_id : iter->group ) { + component2group[component_id] = iter->key(); + } + } + + //set edge weights + PartitionLinkMap_t& links = pgraph->getLinkMap(); + for ( auto iter = links.begin(); iter != links.end(); ++iter ) { + int group0 = component2group.at(iter->component[0]); + int group1 = component2group.at(iter->component[1]); + std::pair key = std::make_pair( + std::min(group0, group1), std::max(group0, group1)); + //give a uniform weight at the moment + edge_weights[key] = 1; + } + + try { + partOutput->verbose(CALL_INFO, 1, 0,"Partitioning graph with %" PRIu64 " vertices\n", + node_weights.size()); + partOutput->verbose(CALL_INFO, 1, 0, " and %" PRIu64 " edges\n", + edge_weights.size()); + + CSRMat csr(node_weights, edge_weights); + + //The imbalance ratio sets a goal imbalance for node weights across different + //rank partitions + constexpr double imbalance_ratio = 1.04; + std::vector rank_partition = metis_part(csr, + rankcount.thread * rankcount.rank, + imbalance_ratio); + + std::vector rank_weights(rankcount.thread * rankcount.rank, 0.); + + for ( uint i = 0; i < csr.size(); ++i ) { + int64_t flat_rank = rank_partition[i]; + + PartitionComponent& c = components[csr.get(i)]; + c.rank = RankInfo(flat_rank / rankcount.thread, flat_rank % rankcount.thread); + + rank_weights[flat_rank] += c.weight; + } + + //print out partition quality + double max_rank_weight{0.}, avg_rank_weight{0.}; + for ( auto w : rank_weights ) { + max_rank_weight = std::max(w, max_rank_weight); + avg_rank_weight += w; + } + avg_rank_weight /= (rankcount.thread * rankcount.rank); + + partOutput->verbose(CALL_INFO, 1, 0, "Partition imbalance (max/avg rank weight): %f\n", + max_rank_weight/avg_rank_weight); + + double sum_edge_weights{0.}, cut_weights{0.}; + for ( auto& ew : edge_weights ) { + auto it0 = std::lower_bound(csr.node_ID().cbegin(), csr.node_ID().cend(), + ew.first.first); + auto it1 = std::lower_bound(it0, csr.node_ID().cend(), ew.first.second); + + sum_edge_weights += ew.second; + if ( rank_partition[*it0] != rank_partition[*it1] ) { + cut_weights += ew.second; + } + } + double cut_pct = cut_weights/sum_edge_weights*100; + partOutput->verbose(CALL_INFO, 1, 0, "Percentage of edges cut: %f\n", + cut_pct); + + } catch (const std::exception& e) { + partOutput->fatal(CALL_INFO, 1, 0, e.what()); + } + + partOutput->verbose(CALL_INFO, 1, 0, "Metis partitioner finished.\n"); +} +} +} +} + +#endif diff --git a/src/sst/core/impl/partitioners/metispart.h b/src/sst/core/impl/partitioners/metispart.h new file mode 100644 index 000000000..dfebf7b07 --- /dev/null +++ b/src/sst/core/impl/partitioners/metispart.h @@ -0,0 +1,50 @@ +#ifndef SST_CORE_IMPL_PARTITIONERS_METISPART_H +#define SST_CORE_IMPL_PARTITIONERS_METISPART_H + +#include "sst/core/sst_config.h" + +#ifdef HAVE_METIS + +#include "sst/core/sstpart.h" +#include "sst/core/output.h" +#include "sst/core/eli/elementinfo.h" + + +namespace SST { +namespace IMPL { +namespace Partition { + +class SSTMetisPartition : public SST::Partition::SSTPartitioner { + +public: + + SST_ELI_REGISTER_PARTITIONER( + SSTMetisPartition, + "sst", + "metis", + SST_ELI_ELEMENT_VERSION(1,0,0), + "metis graph partitioner"); + + SSTMetisPartition(RankInfo world_size, RankInfo my_rank, int verbosity); + ~SSTMetisPartition() { + if (partOutput) delete partOutput; + } + + void performPartition(PartitionGraph* graph) override; + + bool requiresConfigGraph() override { return false; } + bool spawnOnAllRanks() override { return false; } + +private: + /** Number of ranks in the simulation */ + RankInfo rankcount; + Output* partOutput=nullptr; +}; +} +} +} + +#endif //End of HAVE_METIS + + +#endif diff --git a/src/sst/core/impl/partitioners/util.h b/src/sst/core/impl/partitioners/util.h new file mode 100644 index 000000000..b1b200849 --- /dev/null +++ b/src/sst/core/impl/partitioners/util.h @@ -0,0 +1,59 @@ +#ifndef REBALANCE_UTIL_HPP +#define REBALANCE_UTIL_HPP + +#include +#include +#include +#include + +namespace stdx { + template + struct hash_tuple { + std::size_t operator()(size_t h, const tup &x) const { + h ^= std::hash::type>()(std::get(x)); + h *= 0x9e3779b97f4a7c15u; + return hash_tuple()(h, x); + } + }; + + template + struct hash_tuple { + std::size_t operator()(size_t h, const tup &x) const { + return h; + } + }; +} + +namespace std { + template + std::size_t hash_of(const Ts ...xs) { + return stdx::hash_tuple<0, sizeof...(Ts), std::tuple>()(0, std::tuple(xs...)); + } + + /*template + struct hash { + std::size_t operator()(const T &x) const { + return hash_of(x); + } + };*/ + + template + struct hash> { + std::size_t operator()(std::tuple x) const { + return stdx::hash_tuple<0, sizeof...(Ts), std::tuple>()(0, x); + } + }; + + // grabbed from jdbachan's typeclass.hxx + template + struct hash> { + inline size_t operator()(const pair &x) const { + size_t h = hash()(x.first); + h ^= h >> 13; + h *= 41; + h += hash()(x.second); + return h; + } + }; +} +#endif \ No newline at end of file