Use poetry for dependency management

.dockerignore

@@ -4,8 +4,11 @@
 # Specifically add all files that should not be ignored
 !requirements.txt
 !setup.py
+!pyproject.toml
+!poetry.lock
+!README.md
 !src
 !test
 !pytest.ini
 !.flake8
-!scripts
+!scripts

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
           - "flake8-bugbear==21.9.2"
           - "flake8-docstrings==1.5.0"
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.26.1
+    rev: v1.27.1
     hooks:
       - id: yamllint # Check YAML Files
         args: ["-d", "{extends: relaxed, rules: {line-length: {max: 120 }}}"]
@@ -47,3 +47,13 @@ repos:
     hooks:
       - id: nbstripout
         additional_dependencies: ["--index-url=https://pypi.org/simple/"]
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.3.2
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+        args: [--check]
+        files: ^pyproject.toml$
+      - id: poetry-export
+        args: [-f, requirements.txt, -o, requirements.txt, --with=dev, --with=doc, -Eall]
+        files: ^(pyproject.toml|poetry.lock)$

.readthedocs.yaml

@@ -7,23 +7,16 @@ version: 2
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.8"
-    # You can also specify other tool versions:
-    # nodejs: "16"
-    # rust: "1.55"
-    # golang: "1.17"
+    python: "3.9"
+  jobs:
+    post_create_environment:
+      - pip install poetry
+      - poetry config virtualenvs.create false
+    post_install:
+      - poetry install --with doc
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
    configuration: docs/conf.py
-
-# If using Sphinx, optionally build your docs in additional formats such as PDF
-# formats:
-#    - pdf
-
-# Optionally declare the Python requirements required to build your docs
-python:
-  install:
-    - requirements: requirements.doc.in

docs/contribute.rst

@@ -3,24 +3,34 @@ Contribute
 
 To contribute to developing this package, check out its Github repository and push commits there.
 
-How do we handle pip requirements?
+How do we handle dependencies?
 -------------------------------------
 
-We mostly follow `this workflow <https://kennethreitz.org/essays/2016/02/25/a-better-pip-workflow>`_
+We use poetry for resolving and installing dependencies.
+For an overview of poetry basic commands, visit the `official documentation: <https://python-poetry.org/docs>`_
 
-#. Add packages to ``requirements.in``. Only pin versions that need to be pinned to make the code runnable. 
-   Always specify the largest range of possible versions to ensure a maximum of compatibility with other packages.
-#. Ask our devs to freeze your requirements into ``requirements.txt``. This is not allowed from external users for
-   security reasons.
-#. Commit ``requirements.in`` and ``requirements.txt`` in a PR. Once merged to main, Cloudbuild will build the
-   image with the new dependencies.
+#. `Install poetry <https://python-poetry.org/docs/#installation>`_
+#. Install the dependencies: ``poetry install --all-extras``. Poetry creates a virtual environment for you.
+#. You can activate the venv using ``poetry shell`` or temporarily ``poetry run [command]``.
+#. When adding new dependencies, use ``poetry add [my-package]`` or
+   add them manually to ``pyproject.toml`` and update the lockfile ``poetry lock --no-update``.
+#. ``requirements.txt`` will be updated via a pre-commit hook.
+#. Commit ``poetry.lock`` and ``requirements.txt`` in a PR.
+   Once merged to main, Cloudbuild will build the image with the new dependencies.
 
 
 Tests
 -------------------------------------
 
-You can run tests by executing ``pytest``. Prior make sure that you installed the testing extras i.e. via
-``pip install -e .[dev]``.
+You can run tests by executing ``poetry run pytest -n auto``.
+
+Build the documentation locally
+-------------------------------------
+
+Running ``poetry run sphinx-build ./docs ./docs/build`` from the root directory will generate the documentation.
+Currently, this only works on python3.9.
+You can use poetry with python3.9 by running ``poetry env use 3.9`` before ``poetry install --all-extras``.
+
 
 Python Code Style Guide
 --------------------------

infrastructure/docker/Dockerfile

@@ -23,8 +23,10 @@ ENV SPARK_HOME="/opt/spark"
 # add shaded gcs connector
 ADD https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/hadoop3-2.2.2/gcs-connector-hadoop3-2.2.2-shaded.jar /opt/spark/jars/gcs-connector-hadoop3-latest.jar
 
-ADD . .
+ADD requirements.txt requirements.txt
 
 RUN pip3 install --upgrade pip && \
     pip3 install --require-hashes -r requirements.txt --no-deps --disable-pip-version-check && \
-    pip3 cache purge
+    pip3 cache purge
+
+ADD . .