HP/Cray MPI Support (Demote MPI_CXX types to MPI_C)

CMakeLists.txt

@@ -85,6 +85,22 @@ message(STATUS "MPI includes: ${MPI_CXX_INCLUDE_PATH}")
 message(STATUS "MPI C++ libs: ${MPI_CXX_LIBRARIES}")
 message(STATUS "MPI flags: ${MPI_CXX_COMPILE_FLAGS} ${MPI_C_COMPILE_FLAGS}")
 
+#
+# Workaround for MPI implementations that do not properly support
+# MPI_CXX_* datatypes
+#
+option(Use_MPI_C_datatypes
+    "Workaround: Use MPI_C_* datatypes instead of similar MPI_CXX_* datatypes"
+    OFF)
+mark_as_advanced(Use_MPI_C_datatypes)
+if(Use_MPI_C_datatypes)
+    add_definitions(-DPOMEROL_MPI_BOOL=MPI_C_BOOL
+                    -DPOMEROL_MPI_DOUBLE_COMPLEX=MPI_C_DOUBLE_COMPLEX)
+else(Use_MPI_C_datatypes)
+    add_definitions(-DPOMEROL_MPI_BOOL=MPI_CXX_BOOL
+                    -DPOMEROL_MPI_DOUBLE_COMPLEX=MPI_CXX_DOUBLE_COMPLEX)
+endif(Use_MPI_C_datatypes)
+
 # Boost
 find_package(Boost 1.54.0 REQUIRED)
 message(STATUS "Boost includes: ${Boost_INCLUDE_DIRS}" )

README.md

@@ -79,6 +79,12 @@ two-particle Green's functions as well as susceptibilities.
 The library, _libpomerol_ is built. It can be used for linking with executables.
 Some working executables are given in `prog` subdirectory.
 
+> :warning: It has been [reported](https://github.com/aeantipov/pomerol/pull/60)
+that some [MPICH](https://www.mpich.org/)-based MPI implementations, such as HPE
+Cray MPI may not properly support `MPI_CXX_*` datatypes, which pomerol's code
+depends on. In case you see failing MPI unit tests when linking to said MPI
+libraries, try using CMake option `-DUse_MPI_C_datatypes=ON`.
+
 ## Interfacing with your own code and other libraries
 
 Check the `tutorial` directory for an example of a pomerol-based code that is

src/pomerol/Hamiltonian.cpp

@@ -42,7 +42,7 @@ template <bool C> void Hamiltonian::prepareImpl(LOperatorTypeRC<C> const& HOp, M
     std::map<pMPI::JobId, pMPI::WorkerId> job_map = skel.run(comm, false);
     MPI_Barrier(comm);
 
-    MPI_Datatype H_dt = C ? MPI_CXX_DOUBLE_COMPLEX : MPI_DOUBLE;
+    MPI_Datatype H_dt = C ? POMEROL_MPI_DOUBLE_COMPLEX : MPI_DOUBLE;
 
     for(int p = 0; p < static_cast<int>(parts.size()); ++p) {
         auto& part = parts[p];
@@ -77,7 +77,7 @@ template <bool C> void Hamiltonian::computeImpl(MPI_Comm const& comm) {
 
     // Start distributing data
     MPI_Barrier(comm);
-    MPI_Datatype H_dt = C ? MPI_CXX_DOUBLE_COMPLEX : MPI_DOUBLE;
+    MPI_Datatype H_dt = C ? POMEROL_MPI_DOUBLE_COMPLEX : MPI_DOUBLE;
     for(int p = 0; p < static_cast<int>(parts.size()); ++p) {
         auto& part = parts[p];
         auto& H = part.getMatrix<C>();

src/pomerol/TwoParticleGF.cpp

@@ -185,7 +185,7 @@ std::vector<ComplexType> TwoParticleGF::compute(bool clear, FreqVec const& freqs
         MPI_Allreduce(MPI_IN_PLACE,
                       m_data.data(),
                       static_cast<int>(m_data.size()),
-                      MPI_CXX_DOUBLE_COMPLEX,
+                      POMEROL_MPI_DOUBLE_COMPLEX,
                       MPI_SUM,
                       comm);
 

src/pomerol/TwoParticleGFContainer.cpp

@@ -82,7 +82,7 @@ TwoParticleGFContainer::computeAll_split(bool clearTerms, FreqVec const& freqs,
     MPI_Barrier(comm);
     int comp = 0;
 
-    MPI_Comm comm_split = nullptr;
+    MPI_Comm comm_split = MPI_COMM_NULL;
     MPI_Comm_split(comm, proc_colors[comm_rank], comm_rank, &comm_split);
 
     for(auto iter = NonTrivialElements.begin(); iter != NonTrivialElements.end(); iter++, comp++) {
@@ -108,11 +108,11 @@ TwoParticleGFContainer::computeAll_split(bool clearTerms, FreqVec const& freqs,
                 freq_data = storage[iter->first];
                 freq_data_size = static_cast<int>(freq_data.size());
                 MPI_Bcast(&freq_data_size, 1, MPI_LONG, sender, comm);
-                MPI_Bcast(freq_data.data(), freq_data_size, MPI_CXX_DOUBLE_COMPLEX, sender, comm);
+                MPI_Bcast(freq_data.data(), freq_data_size, POMEROL_MPI_DOUBLE_COMPLEX, sender, comm);
             } else {
                 MPI_Bcast(&freq_data_size, 1, MPI_LONG, sender, comm);
                 freq_data.resize(freq_data_size);
-                MPI_Bcast(freq_data.data(), freq_data_size, MPI_CXX_DOUBLE_COMPLEX, sender, comm);
+                MPI_Bcast(freq_data.data(), freq_data_size, POMEROL_MPI_DOUBLE_COMPLEX, sender, comm);
             }
             out[iter->first] = freq_data;
 

src/pomerol/TwoParticleGFPart.cpp

@@ -87,10 +87,10 @@ MPI_Datatype TwoParticleGFPart::NonResonantTerm::mpi_datatype() {
                                     offsetof(NonResonantTerm, Weight)};
         // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays)
         MPI_Datatype types[] = {
-            MPI_CXX_DOUBLE_COMPLEX, // ComplexType Coeff
-            MPI_DOUBLE,             // RealType Poles[3]
-            MPI_CXX_BOOL,           // bool isz4
-            MPI_LONG                // long Weight
+            POMEROL_MPI_DOUBLE_COMPLEX, // ComplexType Coeff
+            MPI_DOUBLE,                 // RealType Poles[3]
+            POMEROL_MPI_BOOL,           // bool isz4
+            MPI_LONG                    // long Weight
         };
         MPI_Type_create_struct(4, blocklengths, displacements, types, &dt);
         MPI_Type_commit(&dt);
@@ -140,11 +140,11 @@ MPI_Datatype TwoParticleGFPart::ResonantTerm::mpi_datatype() {
                                     offsetof(ResonantTerm, Weight)};
         // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays)
         MPI_Datatype types[] = {
-            MPI_CXX_DOUBLE_COMPLEX, // ComplexType ResCoeff
-            MPI_CXX_DOUBLE_COMPLEX, // ComplexType NonResCoeff
-            MPI_DOUBLE,             // RealType Poles[3]
-            MPI_CXX_BOOL,           // bool isz1z2
-            MPI_LONG                // long Weight
+            POMEROL_MPI_DOUBLE_COMPLEX, // ComplexType ResCoeff
+            POMEROL_MPI_DOUBLE_COMPLEX, // ComplexType NonResCoeff
+            MPI_DOUBLE,                 // RealType Poles[3]
+            POMEROL_MPI_BOOL,           // bool isz1z2
+            MPI_LONG                    // long Weight
         };
         MPI_Type_create_struct(5, blocklengths, displacements, types, &dt);
         MPI_Type_commit(&dt);