fix: Separate dependencies into groups for Poetry installation (#175)

* fix: Separate dependencies into groups for Poetry installation

* fix: Change setup.cfg

* Update config for ci workflow

* feat(makefile): Introduce new targets for conda environment setup

* feat(makefile): Add optional variable to create conda env

* fix(ci): Install dev dependencies

* chore(ci): Add geos dependencies

* Fix broken poetry install

* feat(ci): update workflow to support Python 3.10 and 3.11

.github/workflows/python-package.yml

@@ -1,25 +1,32 @@
-name: tests
+name: main
 
 on: [push, pull_request]
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 10
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python_version: ["3.10", "3.11"]
+
     concurrency:
-      group: ci-${{ github.ref }}
+      group: ci-tests-${{ matrix.python_version }}-${{ github.ref }}
       cancel-in-progress: true
-
-    defaults:
-      run:
-        shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v2
-
+    - uses: actions/checkout@v3
+
+    - name: Prepare conda environment
+      run: |
+        sed -i s/python\ 3\.11/python\ ${{ matrix.python_version }}/ conda/dev.yaml
+        cat conda/dev.yaml
+
     - uses: conda-incubator/setup-miniconda@v2
       with:
         miniconda-version: "latest"
@@ -30,15 +37,20 @@ jobs:
         use-mamba: true
         miniforge-variant: Mambaforge
 
+
     - name: Install dependencies
       run: |
         curl -sSL https://install.python-poetry.org/ | python -
         export PATH="$HOME/.poetry/bin:$PATH"
-        poetry config virtualenvs.create false 
-        poetry build && poetry install
+        poetry config virtualenvs.create false
+
+    - name: Install main dependencies
+      run: |
+        make conda-install-main ENVCREATE=conda-env
 
     - name: Lint with flake8
       run: |
+        make conda-install-dev
         make check-codestyle
 
     - name: Run jupyterlab with PySUS
@@ -47,4 +59,5 @@ jobs:
 
     - name: Test with pytest
       run: |
-        make test
+        make conda-install-geo
+        make test-pysus

Makefile

@@ -1,6 +1,7 @@
 SHELL := /usr/bin/env bash
 PYTHON := python
 PYTHONPATH := ${PWD}
+ENVCREATE:=
 
 
 .PHONY: clean clean-test clean-pyc clean-build help
@@ -36,11 +37,34 @@ SEMANTIC_RELEASE = npx --yes \
           semantic-release
 
 
+# Create a Conda environment and install dependencies for development.
+.PHONY: conda-env
+conda-env:
+	mamba env create -f conda/dev.yaml --force
+
+# Install main project dependencies using Poetry.
+.PHONY:  conda-install-main
+conda-install-main: ${ENVCREATE}
+	conda run -n pysus poetry install --only main
+
+.PHONY: conda-install-dev
+conda-install-dev:
+	conda run -n pysus poetry install --only dev
+
+.PHONY: conda-install-docs
+conda-install-docs:
+	conda run -n pysus poetry install --only docs
+
+.PHONY: conda-install-geo
+conda-install-geo:
+	conda run -n pysus pip install --no-use-pep517 shapely==1.8.5.post1
+	conda run -n pysus poetry install --only geo
+
+# Linting
 .PHONY: pre-commit-install
 pre-commit-install:
 	poetry run pre-commit install
 
-#* Linting
 .PHONY: check-codestyle
 check-codestyle: ## check style with flake8
 	# stop the build if there are Python syntax errors or undefined names
@@ -62,11 +86,10 @@ down-jupyter-pysus: ## stop and remove containers for all services
 test-jupyter-pysus: ## run pytest for notebooks inside jupyter container
 	$(DOCKER) exec -T jupyter bash /test_notebooks.sh
 
-.PHONY: test
-test: ## run tests quickly with the default Python
+.PHONY: test-pysus
+test-pysus: ## run tests quickly with the default Python
 	cp docs/source/**/*.ipynb pysus/Notebooks
 	poetry run pytest -vv pysus/tests/
-	poetry run pytest --nbmake --nbmake-timeout=800 pysus/Notebooks/*.ipynb
 
 coverage: ## check code coverage quickly with the default Python
 	coverage run --source pysus/tests/ -m pytest

conda/dev.yaml

@@ -1,12 +1,12 @@
 name: pysus
 channels:
   - conda-forge
-  - nodefaults
+  - defaults
 dependencies:
-  - python >= 3.11
+  - python>=3.11
+  - numpy>=1.26.2
   - cffi
   - gcc
   - make
-  - nbsphinx
+  - poetry>=1.3.2
   - pip
-  - poetry >= 1.3.2

docker/scripts/entrypoint.sh

@@ -1,3 +1,10 @@
 #!/bin/bash
 
-exec jupyter-lab --browser='firefox'  --allow-root --NotebookApp.token='' --NotebookApp.password=''
+jupyter_lab_path=$(which jupyter)
+
+if [ -z "$jupyter_lab_path" ]; then
+  echo "Jupyter not found"
+  exit 1
+fi
+
+$jupyter_lab_path lab --browser='firefox' --allow-root --NotebookApp.token='' --NotebookApp.password=''

docker/scripts/poetry-install.sh

@@ -3,4 +3,4 @@
 set -ex
 
 poetry config virtualenvs.create false
-poetry install --all-extras --with docs
+poetry install --without geo

docs/source/tutorials/Dengue.ipynb

@@ -30,7 +30,6 @@
    ],
    "source": [
     "from pysus.ftp.databases.sinan import SINAN\n",
-    "import keplergl\n",
     "import pandas as pd\n",
     "import geopandas as gpd\n",
     "%pylab inline\n",

pyproject.toml

@@ -13,7 +13,7 @@ python-dateutil = "2.8.2"
 cffi = "1.15.1"
 dbfread = "2.0.7"
 fastparquet = "^0.8.1"
-numpy = "1.23.2"
+numpy = "1.26.2"
 pyarrow = ">=11.0.0"
 pycparser = "2.21"
 pyreaddbc = ">=1.1.0"
@@ -24,38 +24,38 @@ Unidecode = "^1.3.6"
 dateparser = "^1.1.8"
 pandas = ">=1.5.3"
 urwid = "^2.1.2"
-# Preprocessing Packages
 elasticsearch = { version = "7.16.2", extras=["preprocessing"] }
-geobr = { version = "^0.2.0", extras=["preprocessing"] }
-geocoder = { version = "^1.38.1", extras=["preprocessing"] }
-jsonschema = "^4.19.0"
 # FTP
 bigtree = "^0.12.2"
 aioftp = "^0.21.4"
 humanize = "^4.8.0"
 
 [tool.poetry.group.dev.dependencies]
+compose-go = "^2.21.0"
 pytest = ">=6.1.0"
 black = "^22.6.0"
 flake8 = "^5.0.4"
 isort = "^5.10.1"
 pre-commit = "^2.20.0"
 pytest-timeout = "^2.1.0"
-nbsphinx = "^0.9.3"
 
 [tool.poetry.group.docs.dependencies]
-compose-go = "^2.21.0"
 sphinx = "^5.1.1"
 nbmake = "^1.4.1"
 matplotlib = "^3.7.1"
 jupyterlab = "^4.0.5"
 ipykernel = "^6.25.1"
 seaborn = "^0.12.2"
-descartes = "^1.1.0"
-folium = "^0.14.0"
-keplergl = "^0.3.2"
 tomli = "^2.0.1"
 sphinx-rtd-theme = "^1.3.0"
+nbsphinx = "^0.9.3"
+
+[tool.poetry.group.geo.dependencies]
+geobr = { version = "^0.2.0", extras=["preprocessing"] }
+geocoder = { version = "^1.38.1", extras=["preprocessing"] }
+jsonschema = "^4.19.0"
+descartes = "^1.1.0"
+folium = "^0.14.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -84,4 +84,3 @@ exclude = ["*.git", "docs/"]
 
 [tool.poetry.extras]
 preprocessing = ["geobr", "geocoder"]
-

setup.cfg

@@ -1,11 +1,16 @@
 [metadata]
-license-file = LICENSE
+license_file = LICENSE
+
+[options]
+packages = find:
+include = 
+    pysus
 
 [build_sphinx]
 project = 'PySUS'
 version = 0.6
 release = 0.6.3
-source-dir = './docs/source'
+source-dir = ./docs/source
 
 [flake8]
 exclude = tests,build,dist,docs,.git,__pycache__,.tox,.eggs,*.egg,.asv
@@ -21,9 +26,7 @@ include_trailing_comma = true
 skip = docs/
 
 [aliases]
-# Define setup.py command aliases here
 test = pytest
 
 [tool:pytest]
-# collect_ignore = ['setup.py']
-addopts = --ignore=setup.py
+addopts = --ignore=setup.py