Fixes for building portable wheel : nrnivmodl-core should be usable from wheel

.github/workflows/coreneuron-ci.yml

@@ -36,8 +36,8 @@ jobs:
         config:
           # Defaults: CORENRN_ENABLE_SOA=ON CORENRN_ENABLE_MPI=ON
           - {cmake_option: "-DCORENRN_ENABLE_MPI=ON", documentation: ON}
-          - {cmake_option: "-DCORENRN_ENABLE_DYNAMIC_MPI=ON"}
-          - {cmake_option: "-DCORENRN_ENABLE_DYNAMIC_MPI=ON -DCORENRN_ENABLE_SHARED=OFF"}
+          - {cmake_option: "-DCORENRN_ENABLE_MPI_DYNAMIC=ON"}
+          - {cmake_option: "-DCORENRN_ENABLE_MPI_DYNAMIC=ON -DCORENRN_ENABLE_SHARED=OFF"}
           - {cmake_option: "-DCORENRN_ENABLE_MPI=OFF"}
           - {cmake_option: "-DCORENRN_ENABLE_SOA=OFF"}
           - {use_nmodl: ON, py_version: 3.6.7}

CMake/MakefileBuildOptions.cmake

@@ -73,7 +73,7 @@ endforeach()
 string(REPLACE ";" " " CXX14_STD_FLAGS "${CMAKE_CXX14_STANDARD_COMPILE_OPTION}")
 string(TOUPPER "${CMAKE_BUILD_TYPE}" _BUILD_TYPE)
 set(CORENRN_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${_BUILD_TYPE}} ${CXX14_STD_FLAGS} ${NVHPC_CXX_INLINE_FLAGS}"
+    "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${_BUILD_TYPE}} ${CXX14_STD_FLAGS} ${NVHPC_ACC_COMP_FLAGS} ${NVHPC_CXX_INLINE_FLAGS}"
 )
 
 # =============================================================================

CMake/OpenAccHelper.cmake

@@ -56,7 +56,7 @@ if(CORENRN_ENABLE_GPU)
   # more information about this. -gpu=cudaX.Y ensures that OpenACC code is compiled with the same
   # CUDA version as is used for the explicit CUDA code.
   set(NVHPC_ACC_COMP_FLAGS "-acc -gpu=cuda${CORENRN_CUDA_VERSION_SHORT}")
-  set(NVHPC_ACC_LINK_FLAGS "-cuda")
+  set(NVHPC_ACC_LINK_FLAGS "-acc -cuda")
   # Make sure that OpenACC code is generated for the same compute capabilities as the explicit CUDA
   # code. Otherwise there may be confusing linker errors. We cannot rely on nvcc and nvc++ using the
   # same default compute capabilities as each other, particularly on GPU-less build machines.
@@ -65,7 +65,6 @@ if(CORENRN_ENABLE_GPU)
   endforeach()
   # avoid PGI adding standard compliant "-A" flags
   set(CMAKE_CXX14_STANDARD_COMPILE_OPTION --c++14)
-  string(APPEND CMAKE_CXX_FLAGS " ${NVHPC_ACC_COMP_FLAGS} ${PGI_DIAG_FLAGS}")
   string(APPEND CMAKE_EXE_LINKER_FLAGS " ${NVHPC_ACC_LINK_FLAGS}")
   # Use `-Mautoinline` option to compile .cpp files generated from .mod files only. This is
   # especially needed when we compile with -O0 or -O1 optimisation level where we get link errors.

CMakeLists.txt

@@ -88,7 +88,7 @@ option(CORENRN_ENABLE_OPENMP "Build the CORE NEURON with OpenMP implementation"
 option(CORENRN_ENABLE_TIMEOUT "Enable nrn_timeout implementation" ON)
 option(CORENRN_ENABLE_REPORTING "Enable use of ReportingLib for soma reports" OFF)
 option(CORENRN_ENABLE_MPI "Enable MPI-based execution" ON)
-option(CORENRN_ENABLE_DYNAMIC_MPI "Enable dynamic MPI support" OFF)
+option(CORENRN_ENABLE_MPI_DYNAMIC "Enable dynamic MPI support" OFF)
 option(CORENRN_ENABLE_SOA "Enable SoA Memory Layout" ON)
 option(CORENRN_ENABLE_HOC_EXP "Enable wrapping exp with hoc_exp()" OFF)
 option(CORENRN_ENABLE_SPLAYTREE_QUEUING "Enable use of Splay tree for spike queuing" ON)
@@ -332,11 +332,11 @@ set(NMODL_ENABLE_LEGACY_UNITS
     ${CORENRN_ENABLE_LEGACY_UNITS}
     CACHE BOOL "" FORCE)
 
-if(CORENRN_ENABLE_DYNAMIC_MPI)
+if(CORENRN_ENABLE_MPI_DYNAMIC)
   if(NOT CORENRN_ENABLE_MPI)
     message(FATAL_ERROR "Cannot enable dynamic mpi without mpi")
   endif()
-  add_compile_definitions(CORENRN_ENABLE_DYNAMIC_MPI)
+  add_compile_definitions(CORENRN_ENABLE_MPI_DYNAMIC)
 endif()
 
 if(CORENRN_ENABLE_PRCELLSTATE)
@@ -499,7 +499,21 @@ message(STATUS "Build Type          | ${COMPILE_LIBRARY_TYPE}")
 message(STATUS "MPI                 | ${CORENRN_ENABLE_MPI}")
 if(CORENRN_ENABLE_MPI)
   message(STATUS "  INC               | ${MPI_CXX_INCLUDE_PATH}")
-  message(STATUS "  DYNAMIC           | ${CORENRN_ENABLE_DYNAMIC_MPI}")
+  message(STATUS "  DYNAMIC           | ${CORENRN_ENABLE_MPI_DYNAMIC}")
+  if(CORENRN_ENABLE_MPI_DYNAMIC AND NRN_MPI_LIBNAME_LIST)
+    # ~~~
+    # for dynamic mpi, rely on neuron for list of libraries to build
+    # this is to avoid cmake code duplication on the coreneuron side
+    # ~~~
+    list(LENGTH NRN_MPI_LIBNAME_LIST _num_mpi)
+    math(EXPR num_mpi "${_num_mpi} - 1")
+    foreach(val RANGE ${num_mpi})
+      list(GET NRN_MPI_LIBNAME_LIST ${val} libname)
+      list(GET NRN_MPI_INCLUDE_LIST ${val} include)
+      message(STATUS "    LIBNAME         | core${libname}")
+      message(STATUS "    INC             | ${include}")
+    endforeach(val)
+  endif()
 endif()
 message(STATUS "OpenMP              | ${CORENRN_ENABLE_OPENMP}")
 message(STATUS "Use legacy units    | ${CORENRN_ENABLE_LEGACY_UNITS}")

coreneuron/CMakeLists.txt

@@ -168,16 +168,17 @@ add_custom_target(kin_deriv_header DEPENDS "${KINDERIV_HEADER_FILE}")
 # create libraries
 # =============================================================================
 
-# mpi related target, this is a separate library for dynamic MPI
-if(CORENRN_ENABLE_MPI)
-  if(CORENRN_ENABLE_DYNAMIC_MPI)
-    add_library(corenrn_mpi SHARED ${MPI_LIB_FILES})
-  else()
-    add_library(corenrn_mpi OBJECT ${MPI_LIB_FILES})
-    set(OBJ_MPI $<TARGET_OBJECTS:corenrn_mpi>)
-  endif()
-  target_include_directories(corenrn_mpi PRIVATE ${MPI_INCLUDE_PATH})
-  set_property(TARGET corenrn_mpi PROPERTY POSITION_INDEPENDENT_CODE ON)
+# name of coreneuron mpi objects or dynamic library
+set(CORENRN_MPI_LIB_NAME
+    "corenrn_mpi"
+    CACHE INTERNAL "")
+
+# for non-dynamic mpi mode just build object files
+if(CORENRN_ENABLE_MPI AND NOT CORENRN_ENABLE_MPI_DYNAMIC)
+  add_library(${CORENRN_MPI_LIB_NAME} OBJECT ${MPI_LIB_FILES})
+  target_include_directories(${CORENRN_MPI_LIB_NAME} PRIVATE ${MPI_INCLUDE_PATH})
+  set_property(TARGET ${CORENRN_MPI_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON)
+  set(CORENRN_MPI_OBJ $<TARGET_OBJECTS:${CORENRN_MPI_LIB_NAME}>)
 endif()
 
 # main coreneuron library
@@ -190,17 +191,78 @@ add_library(
   ${cudacorenrn_objs}
   ${NMODL_INBUILT_MOD_OUTPUTS}
   ${MPI_CORE_FILES}
-  ${OBJ_MPI})
-if(CORENRN_ENABLE_MPI)
-  if(CORENRN_ENABLE_DYNAMIC_MPI)
-    target_link_libraries(coreneuron ${CMAKE_DL_LIBS})
-    target_link_libraries(corenrn_mpi ${MPI_CXX_LIBRARIES})
-    target_compile_definitions(coreneuron
-                               PUBLIC CORENRN_SHARED_LIBRARY_SUFFIX=${CMAKE_SHARED_LIBRARY_SUFFIX})
+  ${CORENRN_MPI_OBJ})
+
+# we can link to MPI libraries in non-dynamic-mpi build
+if(CORENRN_ENABLE_MPI AND NOT CORENRN_ENABLE_MPI_DYNAMIC)
+  target_link_libraries(coreneuron ${MPI_CXX_LIBRARIES})
+endif()
+
+# this is where we handle dynamic mpi library build
+if(CORENRN_ENABLE_MPI AND CORENRN_ENABLE_MPI_DYNAMIC)
+  # ~~~
+  # main coreneuron library needs to be linked to libdl.so and
+  # and should be aware of shared library suffix on different platforms.
+  # ~~~
+  target_link_libraries(coreneuron ${CMAKE_DL_LIBS})
+
+  # store mpi library targets that will be built
+  list(APPEND corenrn_mpi_targets "")
+
+  # ~~~
+  # if coreneuron is built as a submodule of neuron then check if NEURON has created
+  # list of libraries that needs to be built. We use neuron cmake variables here because
+  # we don't need to duplicate CMake code into coreneuron (we want to have unified cmake
+  # project soon). In the absense of neuron just build a single library libcorenrn_mpi.
+  # This is mostly used for the testing.
+  # ~~~
+  if(NOT CORENEURON_AS_SUBPROJECT)
+    add_library(${CORENRN_MPI_LIB_NAME} SHARED ${MPI_LIB_FILES})
+    target_link_libraries(${CORENRN_MPI_LIB_NAME} ${MPI_CXX_LIBRARIES})
+    target_include_directories(${CORENRN_MPI_LIB_NAME} PRIVATE ${MPI_INCLUDE_PATH})
+    set_property(TARGET ${CORENRN_MPI_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON)
+    list(APPEND corenrn_mpi_targets ${CORENRN_MPI_LIB_NAME})
   else()
-    target_link_libraries(coreneuron ${MPI_CXX_LIBRARIES})
+    # ~~~
+    # from neuron we know how many different libraries needs to be built, their names
+    # include paths to be used for building shared libraries. Iterate through those
+    # and build separate library for each MPI distribution. For example, following
+    # libraries are created:
+    # - libcorenrn_mpich.so
+    # - libcorenrn_ompi.so
+    # - libcorenrn_mpt.so
+    # ~~~
+    list(LENGTH NRN_MPI_LIBNAME_LIST _num_mpi)
+    math(EXPR num_mpi "${_num_mpi} - 1")
+    foreach(val RANGE ${num_mpi})
+      list(GET NRN_MPI_INCLUDE_LIST ${val} include)
+      list(GET NRN_MPI_LIBNAME_LIST ${val} libname)
+
+      add_library(core${libname}_lib SHARED ${MPI_LIB_FILES})
+      target_include_directories(core${libname}_lib PUBLIC ${include})
+
+      # ~~~
+      # TODO: somehow mingw requires explicit linking. This needs to be verified
+      # when we will test coreneuron on windows.
+      # ~~~
+      if(MINGW) # type msmpi only
+        add_dependencies(core${libname}_lib coreneuron)
+        target_link_libraries(core${libname}_lib ${MPI_C_LIBRARIES})
+        target_link_libraries(core${libname}_lib coreneuron)
+      endif()
+      set_property(TARGET core${libname}_lib PROPERTY OUTPUT_NAME core${libname})
+      list(APPEND corenrn_mpi_targets "core${libname}_lib")
+    endforeach(val)
   endif()
+
+  set_target_properties(
+    ${corenrn_mpi_targets}
+    PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
+               LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
+               POSITION_INDEPENDENT_CODE ON)
+  install(TARGETS ${corenrn_mpi_targets} DESTINATION lib)
 endif()
+
 # Prevent CMake from running a device code link step when assembling libcoreneuron.a in GPU builds.
 # The device code linking needs to be deferred to the final step, where it is done by `nvc++ -cuda`.
 set_target_properties(coreneuron PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
@@ -222,13 +284,6 @@ target_include_directories(coreneuron SYSTEM
 target_include_directories(coreneuron SYSTEM
                            PRIVATE ${CORENEURON_PROJECT_SOURCE_DIR}/external/CLI11/include)
 
-if(CORENRN_ENABLE_MPI)
-  set_target_properties(
-    corenrn_mpi
-    PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
-               LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
-               POSITION_INDEPENDENT_CODE ON)
-endif()
 set_target_properties(
   coreneuron scopmath
   PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
@@ -253,6 +308,12 @@ add_custom_target(nrniv-core ALL DEPENDS ${output_binaries})
 
 include_directories(${CORENEURON_PROJECT_SOURCE_DIR})
 
+if(CORENRN_ENABLE_GPU)
+  separate_arguments(CORENRN_ACC_FLAGS UNIX_COMMAND "${NVHPC_ACC_COMP_FLAGS}")
+  target_compile_options(coreneuron BEFORE PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${CORENRN_ACC_FLAGS}>)
+  target_compile_options(scopmath BEFORE PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${CORENRN_ACC_FLAGS}>)
+endif()
+
 # =============================================================================
 # Extract link definitions to be used with nrnivmodl-core
 # =============================================================================

coreneuron/apps/corenrn_parameters.cpp

@@ -25,6 +25,10 @@ corenrn_parameters::corenrn_parameters() {
         "--mpi",
         this->mpi_enable,
         "Enable MPI. In order to initialize MPI environment this argument must be specified.");
+    app.add_option("--mpi-lib",
+                   this->mpi_lib,
+                   "CoreNEURON MPI library to load for dynamic MPI support",
+                   false);
     app.add_flag("--gpu", this->gpu, "Activate GPU computation.");
     app.add_option("--dt",
                    this->dt,
@@ -187,6 +191,7 @@ void corenrn_parameters::parse(int argc, char** argv) {
 std::ostream& operator<<(std::ostream& os, const corenrn_parameters& corenrn_param) {
     os << "GENERAL PARAMETERS" << std::endl
        << "--mpi=" << (corenrn_param.mpi_enable ? "true" : "false") << std::endl
+       << "--mpi-lib=" << corenrn_param.mpi_lib << std::endl
        << "--gpu=" << (corenrn_param.gpu ? "true" : "false") << std::endl
        << "--dt=" << corenrn_param.dt << std::endl
        << "--tstop=" << corenrn_param.tstop << std::endl

coreneuron/apps/corenrn_parameters.hpp

@@ -85,6 +85,7 @@ struct corenrn_parameters {
     std::string reportfilepath;          /// Reports configuration file.
     std::string checkpointpath;  /// Enable checkpoint and specify directory to store related files.
     std::string writeParametersFilepath;  /// Write parameters to this file
+    std::string mpi_lib;                  /// Name of CoreNEURON MPI library to load dynamically.
 
     CLI::App app{"CoreNeuron - Optimised Simulator Engine for NEURON."};  /// CLI app that performs
                                                                           /// CLI parsing

coreneuron/apps/main1.cpp

@@ -83,7 +83,7 @@ void set_openmp_threads(int nthread) {
  * Convert char* containing arguments from neuron to char* argv[] for
  * coreneuron command line argument parser.
  */
-char* prepare_args(int& argc, char**& argv, int use_mpi, const char* arg) {
+char* prepare_args(int& argc, char**& argv, int use_mpi, const char* mpi_lib, const char* arg) {
     // first construct all arguments as string
     std::string args(arg);
     args.insert(0, " coreneuron ");
@@ -92,6 +92,14 @@ char* prepare_args(int& argc, char**& argv, int use_mpi, const char* arg) {
         args.append(" --mpi ");
     }
 
+    // if neuron has passed name of MPI library then add it to CLI
+    std::string corenrn_mpi_lib{mpi_lib};
+    if (!corenrn_mpi_lib.empty()) {
+        args.append(" --mpi-lib ");
+        corenrn_mpi_lib += " ";
+        args.append(corenrn_mpi_lib);
+    }
+
     // we can't modify string with strtok, make copy
     char* first = strdup(args.c_str());
     const char* sep = " ";
@@ -452,12 +460,9 @@ std::unique_ptr<ReportHandler> create_report_handler(ReportConfiguration& config
 using namespace coreneuron;
 
 #if NRNMPI
-#define STRINGIFY(x) #x
-#define TOSTRING(x)  STRINGIFY(x)
-static void* load_dynamic_mpi() {
+static void* load_dynamic_mpi(const std::string& libname) {
     dlerror();
-    void* handle = dlopen("libcorenrn_mpi" TOSTRING(CORENRN_SHARED_LIBRARY_SUFFIX),
-                          RTLD_NOW | RTLD_GLOBAL);
+    void* handle = dlopen(libname.c_str(), RTLD_NOW | RTLD_GLOBAL);
     const char* error = dlerror();
     if (error) {
         std::string err_msg = std::string("Could not open dynamic MPI library: ") + error + "\n";
@@ -473,11 +478,26 @@ extern "C" void mk_mech_init(int argc, char** argv) {
 
 #if NRNMPI
     if (corenrn_param.mpi_enable) {
-#ifdef CORENRN_ENABLE_DYNAMIC_MPI
-        auto mpi_handle = load_dynamic_mpi();
-        mpi_manager().resolve_symbols(mpi_handle);
+#ifdef CORENRN_ENABLE_MPI_DYNAMIC
+        // coreneuron rely on neuron to detect mpi library distribution and
+        // the name of the library itself. Make sure the library name is specified
+        // via CLI option.
+        if (corenrn_param.mpi_lib.empty()) {
+            throw std::runtime_error(
+                "For dynamic MPI support you must pass '--mpi-lib "
+                "/path/libcorenrnmpi_<name>.<suffix>` argument!\n");
+        }
+
+        // neuron can call coreneuron multiple times and hence we do not
+        // want to initialize/load mpi library multiple times
+        static bool mpi_lib_loaded = false;
+        if (!mpi_lib_loaded) {
+            auto mpi_handle = load_dynamic_mpi(corenrn_param.mpi_lib);
+            mpi_manager().resolve_symbols(mpi_handle);
+            mpi_lib_loaded = true;
+        }
 #endif
-        auto ret = nrnmpi_init(&argc, &argv);
+        auto ret = nrnmpi_init(&argc, &argv, corenrn_param.is_quiet());
         nrnmpi_numprocs = ret.numprocs;
         nrnmpi_myid = ret.myid;
     }

coreneuron/mechanism/mech/enginemech.cpp

@@ -58,7 +58,11 @@ extern bool corenrn_embedded;
 extern int corenrn_embedded_nthread;
 
 /// parse arguments from neuron and prepare new one for coreneuron
-char* prepare_args(int& argc, char**& argv, int use_mpi, const char* nrn_arg);
+char* prepare_args(int& argc,
+                   char**& argv,
+                   int use_mpi,
+                   const char* mpi_lib,
+                   const char* nrn_arg);
 
 /// initialize standard mechanisms from coreneuron
 void mk_mech_init(int argc, char** argv);
@@ -80,6 +84,7 @@ int corenrn_embedded_run(int nthread,
                          int have_gaps,
                          int use_mpi,
                          int use_fast_imem,
+                         const char* mpi_lib,
                          const char* nrn_arg) {
     // set coreneuron's internal variable based on neuron arguments
     corenrn_embedded = true;
@@ -93,7 +98,7 @@ int corenrn_embedded_run(int nthread,
     // pre-process argumnets from neuron and prepare new for coreneuron
     int argc;
     char** argv;
-    char* new_arg = prepare_args(argc, argv, use_mpi, nrn_arg);
+    char* new_arg = prepare_args(argc, argv, use_mpi, mpi_lib, nrn_arg);
 
     // initialize internal arguments
     mk_mech_init(argc, argv);

coreneuron/mpi/lib/nrnmpi.cpp

@@ -34,7 +34,7 @@ static void nrn_fatal_error(const char* msg) {
     nrnmpi_abort_impl(-1);
 }
 
-nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv) {
+nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv, bool is_quiet) {
     nrnmpi_under_nrncontrol_ = true;
 
     if (!nrnmpi_initialized_impl()) {
@@ -54,7 +54,7 @@ nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv) {
     nrn_assert(MPI_Comm_size(nrnmpi_world_comm, &nrnmpi_numprocs_) == MPI_SUCCESS);
     nrnmpi_spike_initialize();
 
-    if (nrnmpi_myid_ == 0) {
+    if (nrnmpi_myid_ == 0 && !is_quiet) {
 #if defined(_OPENMP)
         printf(" num_mpi=%d\n num_omp_thread=%d\n\n", nrnmpi_numprocs_, omp_get_max_threads());
 #else

coreneuron/mpi/nrnmpi.h

@@ -76,7 +76,7 @@ struct mpi_function<std::integral_constant<function_ptr, fptr>> : mpi_function_b
     using mpi_function_base::mpi_function_base;
     template <typename... Args> // in principle deducible from `function_ptr`
     auto operator()(Args&&... args) const {
-#ifdef CORENRN_ENABLE_DYNAMIC_MPI
+#ifdef CORENRN_ENABLE_MPI_DYNAMIC
         // Dynamic MPI, m_fptr should have been initialised via dlsym.
         assert(m_fptr);
         return (*reinterpret_cast<decltype(fptr)>(m_fptr))(std::forward<Args>( args )...);

coreneuron/mpi/nrnmpidec.h

@@ -21,7 +21,7 @@ struct nrnmpi_init_ret_t {
     int numprocs;
     int myid;
 };
-extern "C" nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv);
+extern "C" nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv, bool is_quiet);
 extern mpi_function<cnrn_make_integral_constant_t(nrnmpi_init_impl)> nrnmpi_init;
 extern "C" void nrnmpi_finalize_impl(void);
 extern mpi_function<cnrn_make_integral_constant_t(nrnmpi_finalize_impl)> nrnmpi_finalize;