chore(version): the next version will be 3.27.8 #5910
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| #on: | |
| # push: | |
| # branches-ignore: | |
| # - '**' # temporally ignore all | |
| jobs: | |
| update-schema: | |
| if: | | |
| !startsWith(github.event.head_commit.message, 'chore') && | |
| !startsWith(github.event.head_commit.message, 'build: hotfix') && | |
| !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| # submodules: true | |
| fetch-depth: 100 | |
| - run: | | |
| truncate --size=24KB README.md > README-trunc.md | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} | |
| password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} | |
| repository: jinaai/jina | |
| readme-filepath: ./README-trunc.md | |
| - uses: actions/[email protected] | |
| with: | |
| repository: jina-ai/api | |
| path: schema | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - uses: actions/setup-node@v2 | |
| with: | |
| node-version: '14' | |
| - run: | | |
| pip install . --no-cache-dir | |
| pip install docarray==0.21.0 # only valid for this version. I think we should remove this schema loading | |
| JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2)-master | |
| echo "JINA_VERSION=${JINA_VERSION}" >> $GITHUB_ENV | |
| cd schema | |
| mkdir -p schemas | |
| jina export schema --schema-path schemas/"$JINA_VERSION.json" schemas/master.json schemas/master --yaml-path "$JINA_VERSION.yml" master.yml --json-path "$JINA_VERSION.json" master.json master | |
| python ../scripts/get-openapi-schemas.py | |
| npm install --prefix ~ snippet-enricher-cli | |
| ~/node_modules/.bin/snippet-enricher-cli --input=gateway.json --targets=shell_curl > gateway-with-code.json | |
| cd - | |
| - name: redoc-cli-gateway | |
| uses: seeebiii/redoc-cli-github-action@v10 | |
| with: | |
| args: 'bundle schema/gateway-with-code.json -o rest.html' | |
| - name: push-to-api-repo | |
| run: | | |
| mkdir -p schema/rest/ | |
| cp rest.html schema/rest/master.html | |
| cp schema/gateway.json schema/rest/master.json | |
| rm schema/gateway.json schema/gateway-with-code.json | |
| cd schema | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "Jina Dev Bot" | |
| if [[ `git status --porcelain` ]]; then | |
| git pull && git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push | |
| fi | |
| update-docker: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: benc-uk/workflow-dispatch@v1 | |
| with: | |
| workflow: Manual Docker Build | |
| inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "CD"}' | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| env: | |
| release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} | |
| prep-testbed: | |
| runs-on: ubuntu-latest | |
| needs: update-schema | |
| env: | |
| release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - id: set-matrix | |
| run: | | |
| sudo apt-get install jq | |
| export value=$(bash scripts/get-all-test-paths.sh) | |
| echo "matrix=$value" >> $GITHUB_OUTPUT | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| docarray-v-two-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| env: | |
| JINA_RANDOM_PORT_MIN: 16384 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.8] | |
| protobuf-version: ['==3.19.6', ''] | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp38-cp38 | |
| - name: Prepare environment | |
| run: | | |
| docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir | |
| if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 | |
| else | |
| pip install -U protobuf${{ matrix.protobuf-version }} | |
| fi | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test | |
| id: test | |
| run: | | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/unit/serve/runtimes/test_helper.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_v2.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_singleton.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_parameters_as_pydantic.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_streaming.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/csp | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/docker | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 45 | |
| env: | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| stateful-docarray-v-two-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| env: | |
| JINA_RANDOM_PORT_MIN: 16384 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.8] | |
| protobuf-version: ['==3.19.6', ''] | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp38-cp38 | |
| - name: Prepare environment | |
| run: | | |
| docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir | |
| if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 | |
| else | |
| pip install -U protobuf${{ matrix.protobuf-version }} | |
| fi | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test stateful | |
| id: test_stateful | |
| run: | | |
| JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 30 | |
| env: | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| mark-dev-N: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 200 | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - id: computedevdelta | |
| run: | | |
| git fetch --depth=200 | |
| LAST_VER=$(git tag -l | sort -V | tail -n1) | |
| COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count) | |
| echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT | |
| INIT_FILE='jina/__init__.py' | |
| RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2) | |
| echo "LAST_VER $LAST_VER" | |
| echo "COMMITS_SINCE_LAST_VER $COMMITS_SINCE_LAST_VER" | |
| echo "RELEASE_VER $RELEASE_VER" | |
| echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT | |
| echo "releasever=$RELEASE_VER" >> $GITHUB_OUTPUT | |
| outputs: | |
| devdelta: ${{ steps.computedevdelta.outputs.devdelta }} | |
| releasever: ${{ steps.computedevdelta.outputs.releasever }} | |
| # Build the wheels for Linux and macOS for Python 3.8 and newer | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| needs: [mark-dev-N] | |
| strategy: | |
| # Ensure that a wheel builder finishes even if another fails | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # linux | |
| - os: ubuntu-latest | |
| python: '3.7' | |
| python-manylinux-tag: "cp37-cp37m" | |
| - os: ubuntu-latest | |
| python: '3.8' | |
| python-manylinux-tag: "cp38-cp38" | |
| - os: ubuntu-latest | |
| python: '3.9' | |
| python-manylinux-tag: "cp39-cp39" | |
| - os: ubuntu-latest | |
| python: '3.10' | |
| python-manylinux-tag: "cp310-cp310" | |
| - os: ubuntu-latest | |
| python: '3.11' | |
| python-manylinux-tag: "cp311-cp311" | |
| # MacOS emulated | |
| - os: macos-latest | |
| python: '3.8' | |
| python-cibuildwheels: '38' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.9' | |
| python-cibuildwheels: '39' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.10' | |
| python-cibuildwheels: '310' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.11' | |
| python-cibuildwheels: '311' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| # MacOS native | |
| - os: macos-latest | |
| python: '3.8' | |
| python-cibuildwheels: '38' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.9' | |
| python-cibuildwheels: '39' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.10' | |
| python-cibuildwheels: '310' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.11' | |
| python-cibuildwheels: '311' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Update version | |
| run: | | |
| DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| INIT_FILE='jina/__init__.py' | |
| DEV_VER=$RELEASE_VER".dev"$DEVDELTA | |
| echo "DEVDELTA $DEVDELTA" | |
| echo "RELEASE_VER $RELEASE_VER" | |
| echo "DEV_VER $DEV_VER" | |
| if [[ "${{ matrix.os }}" == "macos-latest" ]]; then | |
| sed -i '' '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| else | |
| sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| fi | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }} | |
| - name: Build wheels with cibuildwheels on macos | |
| env: | |
| CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }} | |
| CIBW_ARCHS: ${{ matrix.arch }} | |
| CIBW_TEST_COMMAND: python -c "import jina; import jraft" | |
| CIBW_BUILD_VERBOSITY: 1 | |
| GOARCH: ${{ matrix.goarch }} | |
| CGO_ENABLED: 1 | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| python -m cibuildwheel --output-dir dist | |
| - name: Upload wheels as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: dist/*.whl | |
| # comment for now, do it manually if needed | |
| # pre-release: | |
| # if: | | |
| # !startsWith(github.event.head_commit.message, 'chore') && | |
| # !startsWith(github.event.head_commit.message, 'build: hotfix') && | |
| # !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') | |
| # needs: [build-wheels, mark-dev-N] | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/[email protected] | |
| # with: | |
| # # submodules: true | |
| # fetch-depth: 100 | |
| # - run: | | |
| # truncate --size=24KB README.md > README-trunc.md | |
| # - uses: actions/download-artifact@v3 | |
| # with: | |
| # name: artifact | |
| # path: dist | |
| # - name: Update version | |
| # run: | | |
| # DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| # RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| # INIT_FILE='jina/__init__.py' | |
| # DEV_VER=$RELEASE_VER".dev"$DEVDELTA | |
| # echo "DEVDELTA $DEVDELTA" | |
| # echo "RELEASE_VER $RELEASE_VER" | |
| # echo "DEV_VER $DEV_VER" | |
| # sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| # - name: Pre-release (.devN) | |
| # run: | | |
| # pip install twine wheel | |
| # ./scripts/release.sh | |
| # env: | |
| # TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} | |
| # TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} | |
| # JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} | |
| core-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.8] | |
| test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp38-cp38 | |
| - name: Prepare environment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test | |
| id: test | |
| run: | | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/stateful/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }} | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 30 | |
| env: | |
| JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }} | |
| JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check file existence | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| import-test: | |
| runs-on: ubuntu-latest | |
| needs: update-schema | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| core: ['', 'true'] | |
| perf: ['', 'true'] | |
| python-env: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| exclude: | |
| - core: 'true' | |
| perf: 'true' | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-env }} | |
| - name: Prepare enviroment | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install --no-cache-dir . | |
| env: | |
| JINA_PIP_INSTALL_CORE: ${{ matrix.core }} | |
| JINA_PIP_INSTALL_PERF: ${{ matrix.perf }} | |
| - name: Test basic import | |
| run: python -c 'from jina import Executor,requests,Client,Flow,dynamic_batching,Document,DocumentArray' | |
| docker-image-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-arch: ["linux/amd64", "linux/arm64"] | |
| steps: | |
| # - name: Cancel Previous Runs | |
| # uses: styfle/[email protected] | |
| # with: | |
| # access_token: ${{ github.token }} | |
| - uses: actions/[email protected] | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| install: true | |
| driver-opts: network=host | |
| - run: | | |
| docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 | |
| docker run --rm --privileged tonistiigi/binfmt --install all | |
| - name: Build and test | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: Dockerfiles/debianx.Dockerfile | |
| platforms: ${{ matrix.test-arch }} | |
| push: true | |
| tags: localhost:5000/jina/multiarch:latest | |
| target: jina | |
| - run: | | |
| docker run --platform ${{ matrix.test-arch }} localhost:5000/jina/multiarch:latest -v | |
| hub-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| steps: | |
| # - name: Cancel Previous Runs | |
| # uses: styfle/[email protected] | |
| # with: | |
| # access_token: ${{ github.token }} | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Test hubapp with hubpods | |
| run: | | |
| ./tests/jinahub/test_integration.sh | |
| timeout-minutes: 30 | |
| env: | |
| JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }} | |
| JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }} | |
| k8s-flow-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s Flow | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_flow.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-deployment-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s Deployment | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_deployment.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-graceful-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test graceful request handling on k8s | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_graceful_request_handling.py | |
| timeout-minutes: 45 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-failures-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s failures | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| curl --proto '=https' --tlsv1.2 -sSfL https://linkerd.github.io/linkerd-smi/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_failures.py | |
| timeout-minutes: 45 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-otel-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test K8s with OpenTelemetry | |
| run: | | |
| pytest -v -s --cov=jina --cov-report=xml ./tests/k8s_otel | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| docker-compose-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test docker compose | |
| run: | | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/docker_compose/test_*.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/[email protected] | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| benchmark-pre-release: | |
| runs-on: ubuntu-latest | |
| needs: [update-schema, mark-dev-N] | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| fetch-depth: 200 | |
| - run: | | |
| DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER | |
| echo "NEXT_VER=$NEXT_VER" >> $GITHUB_ENV | |
| - uses: benc-uk/workflow-dispatch@v1 | |
| with: | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| workflow: Benchmark Jina | |
| repo: jina-ai/jina-terraform | |
| ref: "main" | |
| inputs: '{ "pypi_releases": "[\"${{ env.NEXT_VER }}\"]"}' | |
| # just for blocking the merge until all parallel core-test are successful | |
| success-all-steps: | |
| runs-on: ubuntu-latest | |
| needs: [core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release] | |
| if: always() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v2 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Success | |
| if: ${{ success() }} | |
| run: echo "All Done" |