From ba49905eff2bee7617bb338dcc5060088e0116ff Mon Sep 17 00:00:00 2001 From: Tobias Wochinger Date: Mon, 26 Feb 2024 15:40:10 +0100 Subject: [PATCH] ci: unify dependency management + hatch scripts (#7079) * ci: unify dependency management + hatch scripts * ci: migrate readme sync * build: migrate snippets * ci: pin hatch * ci: make Python version more explicit + quote * ci: add scripts with parameters to hatch --------- Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> --- .github/workflows/docstring_labeler.yml | 5 +- .github/workflows/e2e.yml | 14 ++--- .github/workflows/license_compliance.yml | 3 +- .github/workflows/linting.yml | 20 +++---- .github/workflows/minor_version_release.yml | 2 +- .github/workflows/pypi_release.yml | 5 +- .github/workflows/readme_sync.yml | 14 +++-- .github/workflows/snippets_tests.yml | 14 ++--- .github/workflows/tests.yml | 41 +++++++------- pyproject.toml | 61 ++++++++++++++++++++- test/test_requirements.txt | 30 ---------- 11 files changed, 118 insertions(+), 91 deletions(-) delete mode 100644 test/test_requirements.txt diff --git a/.github/workflows/docstring_labeler.yml b/.github/workflows/docstring_labeler.yml index 96384817d4..1dc5ddd9dc 100644 --- a/.github/workflows/docstring_labeler.yml +++ b/.github/workflows/docstring_labeler.yml @@ -5,6 +5,9 @@ on: paths: - "haystack/**/*.py" +env: + PYTHON_VERSION: "3.11" + jobs: label: runs-on: ubuntu-latest @@ -24,7 +27,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "${{ env.PYTHON_VERSION }}" - name: Get docstrings id: base-docstrings diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 694b39e16b..22440932c2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -18,6 +18,7 @@ on: env: PYTHON_VERSION: "3.8" OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + HATCH_VERSION: "1.9.3" jobs: run: @@ -28,15 +29,10 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install dependencies - run: | - sudo apt-get update - sudo apt install ffmpeg # for local Whisper tests - - - name: Install Haystack and the dependencies needed for tests - run: pip install -r test/test_requirements.txt + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Run tests - run: pytest e2e + run: hatch run test:e2e diff --git a/.github/workflows/license_compliance.yml b/.github/workflows/license_compliance.yml index aeb5abcdb4..9fa159d830 100644 --- a/.github/workflows/license_compliance.yml +++ b/.github/workflows/license_compliance.yml @@ -11,6 +11,7 @@ on: env: CORE_DATADOG_API_KEY: ${{ secrets.CORE_DATADOG_API_KEY }} + PYTHON_VERSION: "3.10" jobs: license_check_direct: @@ -25,7 +26,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "${{ env.PYTHON_VERSION }}" - name: Get direct dependencies run: | diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 4557f9206e..e3ece21040 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -12,6 +12,7 @@ on: env: PYTHON_VERSION: "3.8" + HATCH_VERSION: "1.9.3" jobs: mypy: @@ -35,19 +36,16 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Haystack - run: pip install .[dev] - - - name: Install the dependencies needed for tests - run: pip install -r test/test_requirements.txt + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Mypy if: steps.files.outputs.any_changed == 'true' run: | mkdir .mypy_cache - mypy --install-types --non-interactive --cache-dir=.mypy_cache/ ${{ steps.files.outputs.all_changed_files }} + hatch run test:types ${{ steps.files.outputs.all_changed_files }} pylint: runs-on: ubuntu-latest @@ -68,12 +66,12 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Haystack and the dependencies needed for tests - run: pip install -r test/test_requirements.txt + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Pylint if: steps.files.outputs.any_changed == 'true' run: | - pylint -ry -j 0 ${{ steps.files.outputs.all_changed_files }} + hatch run test:lint ${{ steps.files.outputs.all_changed_files }} diff --git a/.github/workflows/minor_version_release.yml b/.github/workflows/minor_version_release.yml index 5b174fa0d0..6233ccea73 100644 --- a/.github/workflows/minor_version_release.yml +++ b/.github/workflows/minor_version_release.yml @@ -49,7 +49,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Install release_docs.py dependencies run: pip install requests diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index 86f983cc33..c784c08f75 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -5,6 +5,9 @@ on: tags: - "v[0-9].[0-9]+.[0-9]+*" +env: + HATCH_VERSION: "1.9.3" + jobs: release-on-pypi: runs-on: ubuntu-latest @@ -14,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Install Hatch - run: pip install hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Build Haystack run: hatch build diff --git a/.github/workflows/readme_sync.yml b/.github/workflows/readme_sync.yml index a6126f7b3a..679eb75ce7 100644 --- a/.github/workflows/readme_sync.yml +++ b/.github/workflows/readme_sync.yml @@ -8,6 +8,10 @@ on: branches: - main +env: + HATCH_VERSION: "1.9.3" + PYTHON_VERSION: "3.10" + jobs: sync: runs-on: ubuntu-latest @@ -18,12 +22,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U haystack-pydoc-tools + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Generate API docs env: @@ -31,7 +33,7 @@ jobs: # from Readme.io as we need them to associate the slug # in config files with their id. README_API_KEY: ${{ secrets.README_API_KEY }} - run: ./.github/utils/pydoc-markdown.sh + run: hatch run readme:sync - name: Sync docs with 2.0 # Sync the docs to the `2.0` version on Readme diff --git a/.github/workflows/snippets_tests.yml b/.github/workflows/snippets_tests.yml index d64c2fd700..148a1acb43 100644 --- a/.github/workflows/snippets_tests.yml +++ b/.github/workflows/snippets_tests.yml @@ -17,6 +17,7 @@ on: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} PYTHON_VERSION: "3.8" + HATCH_VERSION: "1.9.3" jobs: black: @@ -26,7 +27,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Install Black run: | @@ -100,13 +101,10 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install snippets dependencies - run: | - pip install --upgrade pip - pip install "." torch - pip install pydantic + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Get changed files id: files @@ -119,7 +117,7 @@ jobs: run: | CHANGED_FILES=${{ steps.files.outputs.all_changed_files }} for file in $CHANGED_FILES; do - python "$file" + hatch run snippets:python "$file" done - name: Calculate alert data diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39d4c9fe65..71866c4eb4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,7 @@ env: AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PYTHON_VERSION: "3.8" + HATCH_VERSION: "1.9.3" jobs: black: @@ -38,19 +39,13 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Black - run: | - pip install --upgrade pip - pip install .[dev] + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Check status - run: | - if ! black . --check; then - git status - exit 1 - fi + run: hatch run default:format-check - name: Calculate alert data id: calculator @@ -96,10 +91,14 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Haystack and the dependencies needed for tests - run: pip install -r test/test_requirements.txt + - name: Install Hatch + run: pip install hatch==${{ env.HATCH_VERSION }} + + - name: Install dependencies + # To actually install and sync the dependencies + run: hatch run test:pip list - uses: actions/cache@v4 with: @@ -122,7 +121,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Restore Python dependencies uses: actions/cache/restore@v4 @@ -131,7 +130,7 @@ jobs: key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }} - name: Run - run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test + run: hatch run test:unit - name: Coveralls # We upload only coverage for ubuntu as handling both os @@ -186,7 +185,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Install dependencies run: | @@ -200,7 +199,7 @@ jobs: key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }} - name: Run - run: pytest --maxfail=5 -m "integration" test + run: hatch run test:integration - name: Calculate alert data id: calculator @@ -245,7 +244,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Install dependencies run: | @@ -258,7 +257,7 @@ jobs: key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }} - name: Run - run: pytest --maxfail=5 -m "integration" test -k 'not tika' + run: hatch run test:integration-mac - name: Calculate alert data id: calculator @@ -300,7 +299,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "${{ env.PYTHON_VERSION }}" - name: Restore Python dependencies uses: actions/cache/restore@v4 @@ -309,7 +308,7 @@ jobs: key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }} - name: Run - run: pytest --maxfail=5 -m "integration" test -k 'not tika' + run: hatch run test:integration-windows - name: Calculate alert data id: calculator diff --git a/pyproject.toml b/pyproject.toml index ba5b8dec77..2372310efb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ dependencies = [ "boilerpy3", # Fulltext extraction from HTML pages ] -[project.optional-dependencies] -dev = [ +[tool.hatch.envs.default] +dependencies = [ "pre-commit", # Type check "mypy", @@ -91,6 +91,63 @@ dev = [ "black[jupyter]~=23.0", ] +[tool.hatch.envs.default.scripts] +format = "black ." +format-check = "black --check ." + +[tool.hatch.envs.test] +extra-dependencies = [ + "transformers[torch,sentencepiece]==4.37.2", # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators... + "spacy>=3.7,<3.8", # NamedEntityExtractor + "spacy-curated-transformers>=0.2,<=0.3", # NamedEntityExtractor + "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.7.3/en_core_web_trf-3.7.3-py3-none-any.whl", # NamedEntityExtractor + + # Converters + "pypdf", # PyPDFConverter + "markdown-it-py", # MarkdownToDocument + "mdit_plain", # MarkdownToDocument + "tika", # TikaDocumentConverter + "azure-ai-formrecognizer>=3.2.0b2", # AzureOCRDocumentConverter + "langdetect", # TextLanguageRouter and DocumentLanguageClassifier + "sentence-transformers>=2.2.0", # SentenceTransformersTextEmbedder and SentenceTransformersDocumentEmbedder + "openai-whisper>=20231106", # LocalWhisperTranscriber + + # OpenAPI + "jsonref", # OpenAPIServiceConnector, OpenAPIServiceToFunctions + "openapi3", + + # Validation + "jsonschema", + + # Tracing + "opentelemetry-sdk", + "ddtrace", +] + +[tool.hatch.envs.test.scripts] +e2e = "pytest e2e" +unit = 'pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test' +integration = 'pytest --maxfail=5 -m "integration" test' +integration-mac = 'pytest --maxfail=5 -m "integration" test -k "not tika"' +integration-windows = 'pytest --maxfail=5 -m "integration" test -k "not tika"' +types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack}" +lint = "pylint -ry -j 0 {args:haystack}" + +[tool.hatch.envs.readme] +detached = true # To avoid installing the dependencies from the default environment +dependencies = [ + "haystack-pydoc-tools", +] + +[tool.hatch.envs.readme.scripts] +sync = "./.github/utils/pydoc-markdown.sh" + +[tool.hatch.envs.snippets] +extra-dependencies = [ + "torch", + "pydantic", +] + [project.urls] "CI: GitHub" = "https://github.com/deepset-ai/haystack/actions" "Docs: RTD" = "https://haystack.deepset.ai/overview/intro" diff --git a/test/test_requirements.txt b/test/test_requirements.txt deleted file mode 100644 index 2e16267294..0000000000 --- a/test/test_requirements.txt +++ /dev/null @@ -1,30 +0,0 @@ -.[dev] - -# Package Components - -transformers[torch,sentencepiece]==4.37.2 # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators... -spacy>=3.7,<3.8 # NamedEntityExtractor -spacy-curated-transformers>=0.2,<=0.3 # NamedEntityExtractor -https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.7.3/en_core_web_trf-3.7.3-py3-none-any.whl # NamedEntityExtractor - -# Converters -pypdf # PyPDFConverter -markdown-it-py # MarkdownToDocument -mdit_plain # MarkdownToDocument -tika # TikaDocumentConverter -azure-ai-formrecognizer>=3.2.0b2 # AzureOCRDocumentConverter - -langdetect # TextLanguageRouter and DocumentLanguageClassifier -sentence-transformers>=2.2.0 # SentenceTransformersTextEmbedder and SentenceTransformersDocumentEmbedder -openai-whisper>=20231106 # LocalWhisperTranscriber - -# OpenAPI -jsonref # OpenAPIServiceConnector, OpenAPIServiceToFunctions -openapi3 - -# Validation -jsonschema - -# Tracing -opentelemetry-sdk -ddtrace