ci: unify dependency management + hatch scripts (#7079)

* ci: unify dependency management + hatch scripts * ci: migrate readme sync * build: migrate snippets * ci: pin hatch * ci: make Python version more explicit + quote * ci: add scripts with parameters to hatch --------- Co-authored-by: Silvano Cerza <[email protected]>
deepset-ai · Feb 26, 2024 · ba49905 · ba49905
1 parent d66b535
commit ba49905
Show file tree

Hide file tree

Showing 11 changed files with 118 additions and 91 deletions.
diff --git a/.github/workflows/docstring_labeler.yml b/.github/workflows/docstring_labeler.yml
@@ -5,6 +5,9 @@ on:
     paths:
       - "haystack/**/*.py"
 
+env:
+  PYTHON_VERSION: "3.11"
+
 jobs:
   label:
     runs-on: ubuntu-latest
@@ -24,7 +27,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Get docstrings
         id: base-docstrings

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -18,6 +18,7 @@ on:
 env:
   PYTHON_VERSION: "3.8"
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   run:
@@ -28,15 +29,10 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ env.PYTHON_VERSION }}
+        python-version: "${{ env.PYTHON_VERSION }}"
 
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt install ffmpeg  # for local Whisper tests
-
-    - name: Install Haystack and the dependencies needed for tests
-      run: pip install -r test/test_requirements.txt
+    - name: Install Hatch
+      run: pip install hatch==${{ env.HATCH_VERSION }}
 
     - name: Run tests
-      run: pytest e2e
+      run: hatch run test:e2e
diff --git a/.github/workflows/license_compliance.yml b/.github/workflows/license_compliance.yml
@@ -11,6 +11,7 @@ on:
 
 env:
   CORE_DATADOG_API_KEY: ${{ secrets.CORE_DATADOG_API_KEY }}
+  PYTHON_VERSION: "3.10"
 
 jobs:
   license_check_direct:
@@ -25,7 +26,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Get direct dependencies
         run: |

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -12,6 +12,7 @@ on:
 
 env:
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   mypy:
@@ -35,19 +36,16 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack
-        run: pip install .[dev]
-
-      - name: Install the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Mypy
         if: steps.files.outputs.any_changed == 'true'
         run: |
           mkdir .mypy_cache
-          mypy --install-types --non-interactive --cache-dir=.mypy_cache/ ${{ steps.files.outputs.all_changed_files }}
+          hatch run test:types ${{ steps.files.outputs.all_changed_files }}
 
   pylint:
     runs-on: ubuntu-latest
@@ -68,12 +66,12 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack and the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Pylint
         if: steps.files.outputs.any_changed == 'true'
         run: |
-          pylint -ry -j 0 ${{ steps.files.outputs.all_changed_files }}
+          hatch run test:lint ${{ steps.files.outputs.all_changed_files }}
diff --git a/.github/workflows/minor_version_release.yml b/.github/workflows/minor_version_release.yml
@@ -49,7 +49,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install release_docs.py dependencies
         run: pip install requests

diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml
@@ -5,6 +5,9 @@ on:
     tags:
       - "v[0-9].[0-9]+.[0-9]+*"
 
+env:
+  HATCH_VERSION: "1.9.3"
+
 jobs:
   release-on-pypi:
     runs-on: ubuntu-latest
@@ -14,7 +17,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Hatch
-        run: pip install hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Build Haystack
         run: hatch build

diff --git a/.github/workflows/readme_sync.yml b/.github/workflows/readme_sync.yml
@@ -8,6 +8,10 @@ on:
     branches:
       - main
 
+env:
+  HATCH_VERSION: "1.9.3"
+  PYTHON_VERSION: "3.10"
+
 jobs:
   sync:
     runs-on: ubuntu-latest
@@ -18,20 +22,18 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -U haystack-pydoc-tools
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Generate API docs
         env:
           # This is necessary to fetch the documentation categories
           # from Readme.io as we need them to associate the slug
           # in config files with their id.
           README_API_KEY: ${{ secrets.README_API_KEY }}
-        run: ./.github/utils/pydoc-markdown.sh
+        run: hatch run readme:sync
 
       - name: Sync docs with 2.0
         # Sync the docs to the `2.0` version on Readme

diff --git a/.github/workflows/snippets_tests.yml b/.github/workflows/snippets_tests.yml
@@ -17,6 +17,7 @@ on:
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   black:
@@ -26,7 +27,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install Black
         run: |
@@ -100,13 +101,10 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install snippets dependencies
-        run: |
-          pip install --upgrade pip
-          pip install "." torch
-          pip install pydantic
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Get changed files
         id: files
@@ -119,7 +117,7 @@ jobs:
         run: |
           CHANGED_FILES=${{ steps.files.outputs.all_changed_files }}
           for file in $CHANGED_FILES; do
-            python "$file"
+            hatch run snippets:python "$file"
           done
 
       - name: Calculate alert data

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -29,6 +29,7 @@ env:
   AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   black:
@@ -38,19 +39,13 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Black
-        run: |
-          pip install --upgrade pip
-          pip install .[dev]
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Check status
-        run: |
-          if ! black . --check; then
-            git status
-            exit 1
-          fi
+        run: hatch run default:format-check
 
       - name: Calculate alert data
         id: calculator
@@ -96,10 +91,14 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack and the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
+
+      - name: Install dependencies
+        # To actually install and sync the dependencies
+        run: hatch run test:pip list
 
       - uses: actions/cache@v4
         with:
@@ -122,7 +121,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Restore Python dependencies
         uses: actions/cache/restore@v4
@@ -131,7 +130,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test
+        run: hatch run test:unit
 
       - name: Coveralls
         # We upload only coverage for ubuntu as handling both os
@@ -186,7 +185,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install dependencies
         run: |
@@ -200,7 +199,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test
+        run: hatch run test:integration
 
       - name: Calculate alert data
         id: calculator
@@ -245,7 +244,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install dependencies
         run: |
@@ -258,7 +257,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test -k 'not tika'
+        run: hatch run test:integration-mac
 
       - name: Calculate alert data
         id: calculator
@@ -300,7 +299,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Restore Python dependencies
         uses: actions/cache/restore@v4
@@ -309,7 +308,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test -k 'not tika'
+        run: hatch run test:integration-windows
 
       - name: Calculate alert data
         id: calculator