more robust python requirements management

* single requirements file across CIs: `nrn_requirements.txt` * env variables for python versions
neuronsimulator · Jan 24, 2022 · 00b8ba0 · 00b8ba0
1 parent d8322f8
commit 00b8ba0
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 31 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -14,6 +14,10 @@ on:
       - '**.rst'
       - 'docs/**'
 
+env:
+  PY_MIN_VERSION: 2.7
+  PY_MAX_VERSION: 3.8
+
 jobs:
   coverage:
     runs-on: ubuntu-18.04
@@ -29,22 +33,24 @@ jobs:
           sudo apt-get install  build-essential doxygen autoconf automake libtool lcov autotools-dev libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin patchelf python-pip python-tk python-numpy python3-numpy python3-pip g++-5 g++-6 g++-8 g++-9
         shell: bash
 
-      - name: Set up Python2
+      - name: Set up Python@${{ env.PY_MIN_VERSION }}
         uses: actions/setup-python@v2
         with:
-          python-version: '2.x'
+          python-version: ${{ env.PY_MIN_VERSION }}
 
-      - name: Set up Python3
+      - name: Set up Python@${{ env.PY_MAX_VERSION }}
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
-
-      - name: Install Python3 dependencies
-        run: python3 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py
+          python-version: ${{ env.PY_MAX_VERSION }}
 
       - uses: actions/checkout@v2
         with:
-           fetch-depth: 2
+          fetch-depth: 2
+
+      - name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
+        working-directory: ${{runner.workspace}}/nrn
+        run: |
+          python -m pip install --upgrade pip -r nrn_requirements.txt
 
       - name: Build & Test
         id: build-test
@@ -58,14 +64,14 @@ jobs:
           export CXX=g++
 
           # Python setup
-          export PYTHON2=$(which python2);
-          export PYTHON3=$(which python3);
+          export PYTHON_MIN=$(which $PYTHON_MIN_NAME);
+          export PYTHON_MAX=$(which $PYTHON_MAX_NAME);
 
           mkdir build && cd build;
 
           # CMake options & flags
           export COVERAGE_FLAGS="--coverage -O0 -fno-inline -g";
-          export CMAKE_OPTION="-DNRN_ENABLE_BINARY_SPECIAL=ON -DNRN_ENABLE_MPI=ON -DNRN_ENABLE_INTERVIEWS=ON -DNRN_ENABLE_PYTHON=ON -DNRN_ENABLE_PYTHON_DYNAMIC=ON -DNRN_PYTHON_DYNAMIC=${PYTHON2};${PYTHON3} -DNRN_ENABLE_CORENEURON=ON";
+          export CMAKE_OPTION="-DNRN_ENABLE_BINARY_SPECIAL=ON -DNRN_ENABLE_MPI=ON -DNRN_ENABLE_INTERVIEWS=ON -DNRN_ENABLE_PYTHON=ON -DNRN_ENABLE_PYTHON_DYNAMIC=ON -DNRN_PYTHON_DYNAMIC=${PYTHON_MIN};${PYTHON_MAX} -DNRN_ENABLE_CORENEURON=ON";
           cmake $CMAKE_OPTION -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DNRN_ENABLE_TESTS=ON -DCMAKE_C_FLAGS="${COVERAGE_FLAGS}" -DCMAKE_CXX_FLAGS="${COVERAGE_FLAGS}" ..;
 
           # Coverage
@@ -82,6 +88,8 @@ jobs:
           bash <(curl -s https://codecov.io/bash)
         env:
           MATRIX_EVAL: "CC=gcc CXX=g++"
+          PYTHON_MIN_NAME: "python${{ env.PY_MIN_VERSION }}"
+          PYTHON_MAX_NAME: "python${{ env.PY_MAX_VERSION }}"
 
       # This step will set up an SSH connection on tmate.io for live debugging.
       # To trigger it, simply add 'live-debug-coverage' to your last pushed commit message.

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -10,6 +10,9 @@ on:
       - master
       - release/**
 
+env:
+  DEFAULT_PY_VERSION: 3.8
+
 jobs:
   documentation:
     runs-on: ubuntu-18.04
@@ -22,38 +25,39 @@ jobs:
 
       - name: Install apt packages
         run: |
-          sudo apt-get install build-essential autoconf automake libtool autotools-dev libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin patchelf python-pip python-tk python-numpy python3-numpy python3-pip g++-5 g++-6 g++-8 g++-9
+          sudo apt-get install build-essential autoconf automake libtool autotools-dev libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin patchelf g++-5 g++-6 g++-8 g++-9
           sudo apt-get install ffmpeg doxygen
         shell: bash
 
-      - name: Set up Python3
+      - name: Set up Python@${{ env.DEFAULT_PY_VERSION }}
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: ${{ env.DEFAULT_PY_VERSION }}
+
+      - uses: actions/checkout@v2
 
-      - name: Install Python3 dependencies
+      - name: Install Python dependencies
         working-directory: ${{runner.workspace}}/nrn
         run: |
-          python3 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py
+          python -m pip install --upgrade pip -r nrn_requirements.txt
 
-      - uses: actions/checkout@v2
-
-      - name: Install Python3 documentation dependencies
+      - name: Install Python documentation dependencies
         working-directory: ${{runner.workspace}}/nrn
-        run: python3 -m pip install --upgrade -r docs/docs_requirements.txt
+        run: |
+          python -m pip install --upgrade -r docs/docs_requirements.txt
 
       - name: Documentation
         id: documentation
         shell: bash
         working-directory: ${{runner.workspace}}/nrn
         run:  |
           echo "-------- NEURON wheel --------";
-          python3 setup.py build_ext bdist_wheel;
+          python setup.py build_ext bdist_wheel;
           neuron_wheel=dist/NEURON*.whl;
           echo "-------- install wheel --------"
-          python3 -m pip install $neuron_wheel;
+          python -m pip install $neuron_wheel;
           echo "-------- now build docs--------";
-          python3 setup.py docs;
+          python setup.py docs;
           echo "-------- disable jekyll--------";
           pushd docs/_build;
           touch .nojekyll;

diff --git a/.github/workflows/neuron-ci.yml b/.github/workflows/neuron-ci.yml
@@ -19,6 +19,7 @@ env:
   BUILD_TYPE: Release
   DEVELOPER_DIR: /Applications/Xcode_11.3.1.app/Contents/Developer
   DESIRED_CMAKE_VERSION: 3.8.2
+  DEFAULT_PY_VERSION: 3.8
 
 jobs:
   ci:
@@ -65,24 +66,28 @@ jobs:
           sudo apt-get install build-essential autoconf automake libtool autotools-dev libopenmpi-dev libmpich-dev libx11-dev libxcomposite-dev mpich openmpi-bin patchelf python-pip python-tk python-numpy python3-numpy python3-pip g++-5 g++-6 g++-8 g++-9
         shell: bash
 
+      - uses: actions/checkout@v2
+
       - name: Set up Python2
         uses: actions/setup-python@v2
         with:
           python-version: '2.x'
 
       - name: Install Python2 dependencies
         if: ${{ matrix.config.use_python2 == 'ON' || matrix.config.python_dynamic == 'ON' }}
-        run: python2 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py pathlib
+        working-directory: ${{runner.workspace}}/nrn
+        run: |
+          python2 -m pip install --upgrade pip pathlib -r nrn_requirements.txt
 
-      - name: Set up Python3
+      - name: Set up Python@${{ env.DEFAULT_PY_VERSION }}
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: ${{ env.DEFAULT_PY_VERSION }}
 
-      - name: Install Python3 dependencies
-        run: python3 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py
-
-      - uses: actions/checkout@v2
+      - name: Install Python@${{ env.DEFAULT_PY_VERSION }} dependencies
+        working-directory: ${{runner.workspace}}/nrn
+        run: |
+          python -m pip install --upgrade pip -r nrn_requirements.txt
 
       - name: Build and Test
         id: build-test

diff --git a/nrn_requirements.txt b/nrn_requirements.txt
@@ -0,0 +1,11 @@
+wheel
+setuptools
+scikit-build
+matplotlib
+bokeh
+ipython
+cython
+pytest
+pytest-cov
+mpi4py
+numpy
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -110,6 +110,7 @@ if(NRN_ENABLE_PYTHON AND PYTEST_FOUND)
   if(PYTEST_COV_FOUND)
     set(PYTEST_COVERAGE_ARGS --cov-report=xml --cov=neuron)
   else()
+    unset(PYTEST_COVERAGE_ARGS)
   endif()
   # TODO: consider allowing the group-related parts to be dropped here
   nrn_add_test_group(NAME pynrn_tests)
@@ -193,7 +194,7 @@ if(NRN_ENABLE_PYTHON AND PYTEST_FOUND)
         -pyexe
         ${PYTHON_EXECUTABLE})
   else()
-    set(py_args -m pytest ${PYTEST_COVERAGE_ARGS}-s)
+    set(py_args -m pytest ${PYTEST_COVERAGE_ARGS} -s)
     set(modtests_launch_py ${nrnpython_mpi_env} ${PYTHON_EXECUTABLE} ${py_args})
     set(modtests_launch_hoc ${CMAKE_BINARY_DIR}/bin/nrniv ${nrniv_mpi_arg})
     # This tries to use pytest and mpi4py in parallel, which might be fragile.