Merge branch 'master' into olupton/ubsan-fixes

neuronsimulator · Mar 19, 2022 · 6b2b7fc · 6b2b7fc
2 parents 2c16b0c + 4488df4
commit 6b2b7fc
Show file tree

Hide file tree

Showing 501 changed files with 24,900 additions and 5,710 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,108 @@
+version: 2.1
+
+orbs:
+  python: circleci/[email protected]
+
+jobs:
+  manylinux2014-aarch64:
+
+    parameters:
+      NRN_PYTHON_VERSION:
+        type: string
+      NRN_NIGHTLY_UPLOAD:
+        type: string
+
+    machine:
+      image: ubuntu-2004:202101-01
+
+    resource_class: arm.medium
+
+    steps:
+      - checkout
+      - run:
+          name: Build manylinux AArch64 wheel
+          command: |
+            docker run --rm \
+                -w /root/nrn \
+                -v $PWD:/root/nrn \
+                -v /opt/nrnwheel/mpt:/nrnwheel/mpt \
+                -e NEURON_NIGHTLY_TAG \
+                -e NRN_NIGHTLY_UPLOAD \
+                -e NRN_RELEASE_UPLOAD \
+                -e NEURON_WHEEL_VERSION \
+                -e NRN_BUILD_FOR_UPLOAD=1 \
+                'neuronsimulator/neuron_wheel:latest-aarch64' \
+                packaging/python/build_wheels.bash linux << parameters.NRN_PYTHON_VERSION >> coreneuron
+
+      - store_artifacts:
+          path: ./wheelhouse
+          destination: artifacts
+
+      - run:
+          name: Test manylinux AArch64 wheel
+          command: |
+
+            # install mpi dependencies
+            sudo apt update
+            sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev
+
+            # choose available python versions from pyenv
+            pyenv_py_ver=""
+            case << parameters.NRN_PYTHON_VERSION >> in
+              36) pyenv_py_ver="3.6.12" ;;
+              37) pyenv_py_ver="3.7.9" ;;
+              38) pyenv_py_ver="3.8.7" ;;
+              39) pyenv_py_ver="3.9.1" ;;
+              310) pyenv_py_ver="3.10.1" ;;
+              *) echo "Error: pyenv python version not specified!" && exit 1;;
+            esac
+
+            # install python dependencies: .10 is not available pyenv
+            if [ "<< parameters.NRN_PYTHON_VERSION >>" == "310" ]; then
+              sudo apt install software-properties-common -y
+              sudo add-apt-repository ppa:deadsnakes/ppa -y
+              sudo apt install python3.10 libpython3.10 python3.10-venv
+              export PYTHON_EXE=$(which python3.10)
+            else
+              env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $pyenv_py_ver --force
+              pyenv global $pyenv_py_ver
+              export PYTHON_EXE=$(which python)
+            fi
+
+            # test wheel
+            packaging/python/test_wheels.sh $PYTHON_EXE $(ls -t wheelhouse/*.whl)
+
+      - run:
+          name: Upload nightly wheel to pypi.org
+          command: |
+            if [ "<< parameters.NRN_NIGHTLY_UPLOAD >>" == "true" ]; then
+              python -m pip install twine
+              python -m twine upload --verbose --skip-existing -u $TWINE_USERNAME -p $TWINE_PASSWORD  wheelhouse/*.whl
+            else
+              echo "Skipping pypi.org upload!"
+            fi
+
+workflows:
+
+  build-workflow:
+    jobs:
+      - manylinux2014-aarch64:
+          matrix:
+            parameters:
+              NRN_PYTHON_VERSION: ["310"]
+              NRN_NIGHTLY_UPLOAD: ["false"]
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - manylinux2014-aarch64:
+          matrix:
+            parameters:
+              NRN_PYTHON_VERSION: ["36", "37", "38", "39", "310"]
+              NRN_NIGHTLY_UPLOAD: ["true"]
diff --git a/.cmake-format.changes.yaml b/.cmake-format.changes.yaml
@@ -3,26 +3,28 @@ additional_commands:
     pargs: 0
     kwargs:
       NAME: 1
-      SUBMODULE: '?'
-      MODFILE_PATTERNS: '*'
-      SCRIPT_PATTERNS: '*'
-      OUTPUT: '*'
+      MODFILE_PATTERNS: '+'
+      NRNIVMODL_ARGS: '+'
+      OUTPUT: '+'
+      SCRIPT_PATTERNS: '+'
+      SIM_DIRECTORY: 1
+      SUBMODULE: 1
   nrn_add_test:
     pargs: 0
     kwargs:
       GROUP: 1
       NAME: 1
-      SUBMODULE: '?'
-      MODFILE_PATTERNS: '*'
-      PRECOMMAND: '+'
       COMMAND: '+'
-      SCRIPT_PATTERNS: '*'
-      REQUIRES: '*'
-      CONFLICTS: '*'
+      CONFLICTS: '+'
+      PRECOMMAND: '+'
+      PROCESSORS: 1
+      REQUIRES: '+'
+      MODFILE_PATTERNS: '*'
+      NRNIVMODL_ARGS: '*'
       OUTPUT: '*'
-      PROCESSORS: '?'
+      SCRIPT_PATTERNS: '*'
       SIM_DIRECTORY: '?'
-      NRNIVMODL_ARGS: '+'
+      SUBMODULE: '?'
   nrn_add_test_group_comparison:
     pargs: 0
     kwargs:

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -20,8 +20,8 @@ on:
 #      - 'docs/**'
 
 env:
-  PY_MIN_VERSION: 3.6
-  PY_MAX_VERSION: 3.9
+  PY_MIN_VERSION: '3.6'
+  PY_MAX_VERSION: '3.10'
 
 jobs:
   coverage:

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -15,11 +15,11 @@ on:
       - release/**
 
 env:
-  DEFAULT_PY_VERSION: 3.8
+  DEFAULT_PY_VERSION: '3.10'
 
 jobs:
   documentation:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     name: Documentation
 
@@ -31,7 +31,7 @@ jobs:
         run: |
           sudo apt-get update --fix-missing
           sudo apt-get install build-essential libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin patchelf
-          sudo apt-get install ffmpeg doxygen
+          sudo apt-get install ffmpeg doxygen pandoc
         shell: bash
 
       - name: Set up Python@${{ env.DEFAULT_PY_VERSION }}

diff --git a/.github/workflows/neuron-ci.yml b/.github/workflows/neuron-ci.yml
@@ -23,8 +23,8 @@ env:
   BUILD_TYPE: Release
   DEVELOPER_DIR: /Applications/Xcode_11.3.1.app/Contents/Developer
   DESIRED_CMAKE_VERSION: 3.15.0
-  PY_MIN_VERSION: 3.6
-  PY_MAX_VERSION: 3.9
+  PY_MIN_VERSION: '3.6'
+  PY_MAX_VERSION: '3.10'
 
 jobs:
   ci:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -34,7 +34,7 @@ jobs:
     - name: Set up Python3
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: '3.10'
 
     - name: Download Dependencies
       run: |

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -76,8 +76,10 @@ option(NRN_DYNAMIC_UNITS_USE_LEGACY "Use legacy units as default for dynamic uni
 # note that if CoreNEURON is enabled then it is not necessary to enable this option
 option(NRN_ENABLE_MOD_COMPATIBILITY "Enable CoreNEURON compatibility for MOD files" ${NRN_ENABLE_MOD_COMPATIBILITY_DEFAULT})
 option(NRN_ENABLE_REL_RPATH "Use relative RPATH in binaries. for relocatable installs/Python" ${NRN_ENABLE_REL_RPATH_DEFAULT})
+option(NRN_WHEEL_BUILD ${NRN_WHEEL_BUILD_DEFAULT})
 option(NRN_WHEEL_STATIC_READLINE "Use static readline libraries for the wheels." ${NRN_WHEEL_STATIC_READLINE_DEFAULT})
 mark_as_advanced(NRN_ENABLE_REL_RPATH)
+mark_as_advanced(NRN_WHEEL_BUILD)
 
 # =============================================================================
 # Build options (string)
@@ -252,15 +254,15 @@ if(SPHINX_FOUND)
   # Note that neuron must be available for python import. See docs/README.md.
   add_custom_target(
           notebooks
-          COMMAND bash ${PROJECT_SOURCE_DIR}/docs/notebooks.sh execute
+          COMMAND ${CMAKE_COMMAND} -E env NEURON_MODULE_OPTIONS="-nogui" bash ${PROJECT_SOURCE_DIR}/docs/notebooks.sh
           WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
   )
 
-  # Executing notebooks can take a fair amount of time. Use this target
-  # if you want to skip the execution.
+  # Executing notebooks will add outputs in-place.
+  # We don't want those committed to the repo. This commands cleans them out.
   add_custom_target(
-          notebooks-noexec
-          COMMAND bash ${PROJECT_SOURCE_DIR}/docs/notebooks.sh
+          notebooks-clean
+          COMMAND bash ${PROJECT_SOURCE_DIR}/docs/notebooks.sh --clean
           WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
   )
 
@@ -271,8 +273,7 @@ if(SPHINX_FOUND)
           sphinx
           COMMAND ${SPHINX_EXECUTABLE} -b html
           ${SPHINX_SOURCE} ${SPHINX_BUILD}
-          COMMAND echo "The HTML docs are at file://${PROJECT_SOURCE_DIR}/docs/_build/index.html"
-          WORKING_DIRECTORY WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
+          WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
           COMMENT "Generating documentation with Sphinx"
   )
 endif()
@@ -291,6 +292,9 @@ if(DOXYGEN_FOUND AND SPHINX_FOUND AND JUPYTER_EXE)
           WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/docs
           COMMAND ${CMAKE_COMMAND} --build . --target sphinx
           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+          COMMAND ${CMAKE_COMMAND} --build . --target notebooks-clean
+          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+          COMMAND echo "The HTML docs are at file://${PROJECT_SOURCE_DIR}/docs/_build/index.html"
           COMMENT "Generating full documentation"
   )
 else()
@@ -305,7 +309,9 @@ endif()
 # Enable MPI
 # =============================================================================
 if(NRN_ENABLE_MPI)
-  find_package(MPI REQUIRED)
+
+  # find_package(MPI REQUIRED) has a CMAKE_OSX_ARCHITECTURES edge case
+  nrn_mpi_find_package()
   set(NRNMPI 1)
   set(PARANEURON 1)
   # avoid linking to C++ bindings
@@ -386,6 +392,15 @@ endif()
 # =============================================================================
 # Enable Python support
 # =============================================================================
+
+# Since PythonInterp module prefers system-wide python,
+# if PYTHON_EXECUTABLE is not set, look it up in the PATH exclusively.
+if(NOT PYTHON_EXECUTABLE)
+  message(STATUS "-DPYTHON_EXECUTABLE not specified. Looking for `python3` in the PATH exclusively...")
+  find_program(PYTHON_EXECUTABLE python3 PATHS ENV PATH  NO_DEFAULT_PATH)
+  message(STATUS "\tSetting PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}")
+endif()
+
 find_package(PythonInterp 3.6 REQUIRED)
 if(NRN_ENABLE_PYTHON)
   # start afresh with PythonLibsNew's find_library
@@ -442,7 +457,7 @@ else()
 endif()
 
 # =============================================================================
-# Add helpder CMake modules AFTER setting options
+# Add helper CMake modules AFTER setting options
 # =============================================================================
 include(NeuronFileLists)
 include(MPIDynamicHelper)
@@ -456,10 +471,12 @@ if(NRN_ENABLE_CORENEURON)
   find_package(coreneuron QUIET PATHS ${CORENEURON_DIR}/share/cmake ${CORENEURON_DIR})
   if(coreneuron_FOUND)
     message(STATUS "Using external CoreNEURON from ${CORENEURON_DIR}")
+    set(nrn_using_ext_corenrn TRUE)
     # By default `nrnivmodl` should look for `nrnivmodl-core` under the external prefix.
     set(cnrn_prefix "${CORENEURON_DIR}")
   else()
     message(STATUS "Building CoreNEURON from submodule")
+    set(nrn_using_ext_corenrn FALSE)
     # If NEURON tests are enabled then enable CoreNEURON tests too
     set(CORENRN_ENABLE_UNIT_TESTS ${NRN_ENABLE_TESTS} CACHE BOOL "" FORCE)
     set(CORENRN_ENABLE_LEGACY_UNITS ${NRN_DYNAMIC_UNITS_USE_LEGACY} CACHE BOOL "" FORCE)
@@ -485,6 +502,8 @@ if(NRN_ENABLE_CORENEURON)
     # variable in this scope. If CoreNEURON is installed externally then this
     # is exported into coreneuron-config.cmake.
     get_property(CORENEURON_LIB_LINK_FLAGS GLOBAL PROPERTY CORENEURON_LIB_LINK_FLAGS)
+    # NEURON tests that link against CoreNEURON need to depend on it.
+    set(CORENEURON_TARGET_TO_DEPEND coreneuron)
   endif()
 endif()
 
@@ -650,6 +669,11 @@ file(COPY ${NRN_HEADERS_PATHS} ${PROJECT_BINARY_DIR}/src/oc/nrnpthread.h
      DESTINATION ${PROJECT_BINARY_DIR}/include)
 install(DIRECTORY ${PROJECT_BINARY_DIR}/include DESTINATION ${CMAKE_INSTALL_PREFIX})
 
+if (NRN_MACOS_BUILD)
+  # universal build for neurondemo needs to be after, or at end of, install
+  nrn_macos_after_install()
+endif()
+
 # =============================================================================
 # Copy bash executable for windows
 # =============================================================================
@@ -705,7 +729,7 @@ message(STATUS "--------------+-------------------------------------------------
 message(STATUS "install       | Will install NEURON to: ${CMAKE_INSTALL_PREFIX}")
 message(STATUS "              | Change the install location of NEURON using:")
 message(STATUS "              |   cmake <src_path> -DCMAKE_INSTALL_PREFIX=<install_path>")
-message(STATUS "docs          | Build full docs. Calls targets: doxygen, notebooks, sphinx")
+message(STATUS "docs          | Build full docs. Calls targets: doxygen, notebooks, sphinx, notebooks-clean")
 message(STATUS "uninstall     | Removes files installed by make install (todo)")
 message(STATUS "--------------+--------------------------------------------------------------")
 message(STATUS " Build option | Status")
@@ -722,8 +746,6 @@ else()
 endif()
 message(STATUS "MPI           | ${NRN_ENABLE_MPI}")
 if(NRN_ENABLE_MPI)
-  message(STATUS "  INC         | ${MPI_INCLUDE_PATH}")
-  message(STATUS "  LIB         | ${MPI_LIBRARY}")
   message(STATUS "  DYNAMIC     | ${NRN_ENABLE_MPI_DYNAMIC}")
   if(NRN_ENABLE_MPI_DYNAMIC)
     list(LENGTH NRN_MPI_LIBNAME_LIST _num_mpi)
@@ -734,6 +756,9 @@ if(NRN_ENABLE_MPI)
       message(STATUS "    LIBNAME   | ${libname}")
       message(STATUS "    INC       | ${include}")
     endforeach(val)
+  else()
+    message(STATUS "  INC         | ${MPI_INCLUDE_PATH}")
+    message(STATUS "  LIB         | ${MPI_LIBRARY}")
   endif()
 endif()
 message(STATUS "Python        | ${NRN_ENABLE_PYTHON}")
@@ -792,6 +817,9 @@ if(NRN_ENABLE_CORENEURON)
     message(STATUS "  Legacy Units| ${CORENRN_ENABLE_LEGACY_UNITS}")
   endif()
 endif()
+if (NRN_UNIVERSAL2_BUILD)
+  message(STATUS "CMAKE_OSX_ARCH| ${CMAKE_OSX_ARCHITECTURES}")
+endif()
 message(STATUS "Tests         | ${NRN_ENABLE_TESTS}")
 if(NRN_ENABLE_COVERAGE)
   message(STATUS "Coverage      | Enabled")

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -122,7 +122,7 @@ jobs:
         -e NRN_RELEASE_UPLOAD \
         -e NEURON_WHEEL_VERSION \
         -e NRN_BUILD_FOR_UPLOAD=1 \
-        'neuronsimulator/neuron_wheel_gpu' \
+        'neuronsimulator/neuron_wheel_gpu:nvhpc-22.1-cuda-11.5' \
         packaging/python/build_wheels.bash linux $(python.version) coreneuron-gpu
     displayName: 'Building ManyLinux Wheel'
 

diff --git a/bin/bbswork.sh.in b/bin/bbswork.sh.in
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 prefix=@prefix@
 exec_prefix=@exec_prefix@

diff --git a/bin/hel2mos1.sh.in b/bin/hel2mos1.sh.in
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 #NETSCAPE='/u3/local/hines/netscape/netscape'
 NETSCAPE=`which netscape`

diff --git a/bin/mkthreadsafe b/bin/mkthreadsafe
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 if test $# = 0 ; then
 	files="`ls *.mod`"

diff --git a/bin/mos2nrn2.sh.in b/bin/mos2nrn2.sh.in
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # mos2nrn2 zipfile simdir running