Skip to content

Commit

Permalink
[ENH] add image tags to db migration jobs (chroma-core#2425)
Browse files Browse the repository at this point in the history
  • Loading branch information
Anush008 committed Jun 27, 2024
2 parents 3f329e8 + 9cce6b1 commit 4782dcc
Show file tree
Hide file tree
Showing 259 changed files with 21,298 additions and 5,942 deletions.
4 changes: 4 additions & 0 deletions .github/actions/rust/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,7 @@ runs:
path: hnswlib
- name: Install Protoc
uses: arduino/setup-protoc@v2
- name: Cache
uses: Swatinem/rust-cache@v2
with:
workspaces: chroma/rust/worker
4 changes: 2 additions & 2 deletions .github/workflows/_go-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
test:
strategy:
matrix:
platform: [ubuntu-latest]
platform: [depot-ubuntu-22.04]
runs-on: ${{ matrix.platform }}
services:
postgres:
Expand Down Expand Up @@ -36,7 +36,7 @@ jobs:
working-directory: go

cluster-test:
runs-on: "16core-64gb-ubuntu-latest"
runs-on: "depot-ubuntu-22.04-16"
steps:
- name: Checkout
uses: actions/checkout@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_javascript-client-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:

jobs:
test:
runs-on: ubuntu-latest
runs-on: depot-ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
Expand Down
47 changes: 39 additions & 8 deletions .github/workflows/_python-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,25 @@ on:
required: false
default: '["3.8"]'
type: string

property_testing_preset:
description: 'Property testing preset'
required: true
type: string

jobs:
test:
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: [ubuntu-latest, windows-latest]
platform: [depot-ubuntu-22.04, windows-latest]
test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore-glob 'chromadb/test/distributed/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
"chromadb/test/auth/test_simple_rbac_authz.py",
"chromadb/test/property/test_add.py",
"chromadb/test/property/test_collections.py",
"chromadb/test/property/test_collections_with_database_tenant.py",
"chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
"chromadb/test/property/test_cross_version_persist.py",
"chromadb/test/property/test_embeddings.py",
"chromadb/test/property/test_filtering.py",
Expand All @@ -37,24 +43,33 @@ jobs:
with:
python-version: ${{ matrix.python }}
- name: Test
run: python -m pytest ${{ matrix.parallelized && '-n auto' || '' }} ${{ matrix.test-globs }}
run: python -m pytest ${{ matrix.parallelized && '-n auto' || '' }} ${{ matrix.test-globs }}
shell: bash
env:
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

# todo: break out stress integration tests
test-single-node-integration:
strategy:
fail-fast: false
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: [ubuntu-latest] # todo: should run on Windows, currently failing because Dockerfile doesn't build
platform: [depot-ubuntu-22.04, windows-latest]
test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py' --ignore-glob 'chromadb/test/distributed/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
"chromadb/test/property/test_add.py",
"chromadb/test/test_cli.py",
"chromadb/test/auth/test_simple_rbac_authz.py",
"chromadb/test/property/test_collections.py",
"chromadb/test/property/test_collections_with_database_tenant.py",
"chromadb/test/property/test_cross_version_persist.py",
"chromadb/test/property/test_embeddings.py",
"chromadb/test/property/test_filtering.py",
"chromadb/test/property/test_persist.py"]
include:
- platform: depot-ubuntu-22.04
env-file: compose-env.linux
- platform: windows-latest
env-file: compose-env.windows
runs-on: ${{ matrix.platform }}
steps:
- name: Checkout
Expand All @@ -64,15 +79,23 @@ jobs:
- name: Integration Test
run: bin/python-integration-test ${{ matrix.test-globs }}
shell: bash
env:
ENV_FILE: ${{ matrix.env-file }}
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

test-cluster-integration:
strategy:
fail-fast: false
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: ["16core-64gb-ubuntu-latest"]
platform: ["depot-ubuntu-22.04-16"]
test-globs: ["chromadb/test/db/test_system.py",
"chromadb/test/property/test_collections.py",
"chromadb/test/property/test_add.py",
"chromadb/test/property/test_filtering.py",
"chromadb/test/property/test_embeddings.py",
"chromadb/test/property/test_collections_with_database_tenant.py",
"chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
"chromadb/test/ingest/test_producer_consumer.py",
"chromadb/test/segment/distributed/test_memberlist_provider.py",
"chromadb/test/test_logservice.py",
Expand All @@ -87,12 +110,14 @@ jobs:
- name: Test
run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"'
shell: bash
env:
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

test-thin-client:
strategy:
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: [ubuntu-latest] # # todo: should run on Windows, currently failing because Dockerfile doesn't build
platform: [depot-ubuntu-22.04] # # todo: should run on Windows, currently failing because Dockerfile doesn't build
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v3
Expand All @@ -103,13 +128,15 @@ jobs:
- name: Test
run: clients/python/integration-test.sh
shell: bash
env:
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

test-stress:
timeout-minutes: 90
strategy:
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: ['16core-64gb-ubuntu-latest', '16core-64gb-windows-latest']
platform: ['depot-ubuntu-22.04-16', '16core-64gb-windows-latest']
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v3
Expand All @@ -119,12 +146,14 @@ jobs:
- name: Test
run: python -m pytest chromadb/test/stress/
shell: bash
env:
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

test-single-node-integration-stress:
strategy:
matrix:
python: ${{fromJson(inputs.python_versions)}}
platform: [ubuntu-latest] # todo: should run on Windows, currently failing because Dockerfile doesn't build
platform: [depot-ubuntu-22.04] # todo: should run on Windows, currently failing because Dockerfile doesn't build
runs-on: ${{ matrix.platform }}
steps:
- name: Checkout
Expand All @@ -134,3 +163,5 @@ jobs:
- name: Integration Test
run: bin/python-integration-test chromadb/test/stress/
shell: bash
env:
PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
2 changes: 1 addition & 1 deletion .github/workflows/_rust-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
test:
strategy:
matrix:
platform: [ubuntu-latest]
platform: [depot-ubuntu-22.04]
runs-on: ${{ matrix.platform }}
defaults:
run:
Expand Down
56 changes: 56 additions & 0 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,47 @@ jobs:
id: changes
uses: dorny/paths-filter@v3
with:
predicate-quantifier: 'every'
filters: |
outside-docs:
- '!docs/**'
docs:
- 'docs/**'
deploy-docs-preview:
name: Deploy preview of docs
needs: paths-filter
if: needs.paths-filter.outputs.docs == 'true'
runs-on: depot-ubuntu-22.04
environment:
name: Preview
url: ${{ steps.deploy.outputs.url }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-node@v3
with:
node-version: "18.x"
registry-url: "https://registry.npmjs.org"
- name: Install vercel
run: npm install -g vercel
- name: Deploy
id: deploy
run: echo "url=$(vercel deploy --token ${{ secrets.VERCEL_TOKEN }})" >> $GITHUB_OUTPUT
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_DOCS_PROJECT_ID }}


python-tests:
name: Python tests
needs: paths-filter
if: needs.paths-filter.outputs.outside-docs == 'true'
uses: ./.github/workflows/_python-tests.yml
with:
property_testing_preset: 'fast'

python-vulnerability-scan:
name: Python vulnerability scan
Expand Down Expand Up @@ -102,3 +134,27 @@ jobs:
- name: Cargo fmt check
shell: bash
run: cargo fmt -- --check

# This job exists for our branch protection rule.
# We want to require status checks to pass before merging, but the set of
# checks that run for any given PR is dynamic based on the files changed.
# When creating a branch protection rule, you have to specify a static list
# of checks.
# So since this job always runs, we can specify it in the branch protection rule.
all-required-pr-checks-passed:
if: always()
needs:
- python-tests
- python-vulnerability-scan
- javascript-client-tests
- rust-tests
- go-tests
- check-title
- lint
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
allowed-skips: python-tests,python-vulnerability-scan,javascript-client-tests,rust-tests,go-tests
26 changes: 25 additions & 1 deletion .github/workflows/release-chromadb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ jobs:
uses: ./.github/workflows/_python-tests.yml
with:
python_versions: '["3.8", "3.9", "3.10", "3.11", "3.12"]'
property_testing_preset: 'normal'

javascript-client-tests:
name: JavaScript client tests
Expand Down Expand Up @@ -238,7 +239,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Release Latest
uses: ncipollo/release-action@v1.11.1
uses: ncipollo/release-action@v1.14.0
if: ${{ needs.check-tag.outputs.tag_matches != 'true' }}
with:
tag: "latest"
Expand All @@ -252,4 +253,27 @@ jobs:
DockerHub Image: `${{ env.DOCKERHUB_IMAGE_NAME }}:${{ needs.get-version.outputs.version }}`
artifacts: "dist/chromadb-${{needs.get-version.outputs.version}}.tar.gz"
allowUpdates: true
removeArtifacts: true
prerelease: true

release-docs:
name: Deploy docs to Vercel
runs-on: ubuntu-latest
needs:
- check-tag
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-node@v3
with:
node-version: "18.x"
registry-url: "https://registry.npmjs.org"
- name: Install vercel
run: npm install -g vercel
- name: Deploy
run: vercel deploy --token ${{ secrets.VERCEL_TOKEN }} ${{ needs.check-tag.outputs.tag_matches == 'true' && '--prod' || '' }}
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_DOCS_PROJECT_ID }}
68 changes: 68 additions & 0 deletions .github/workflows/release-dev-javascript-client.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: 📦 Development Release JavaScript client

on:
push:
branches:
- main

jobs:
release-dev:
strategy:
matrix:
registry: [ "https://npm.pkg.github.com" ]
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/main' }}
permissions: write-all
steps:
- name: Check if tag matches the pattern
id: check-tag
run: |
# we don't necessarily need this
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
echo "Push to main branch, releasing dev version to GH packages"
echo "NPM_SCRIPT=release_dev" >> "$GITHUB_ENV"
else
echo "The ref does not point to main, exiting workflow" # we alredy make the check above but this is a good practice
exit 1
fi
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-node@v3
with:
node-version: "16.x"
registry-url: ${{ matrix.registry }}
check-latest: false
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install dependencies
run: npm install
working-directory: ./clients/js/
- name: Dev Version
id: dev-version
run: |
set -e
# Get current version
CURRENT_VERSION=$(node -p "require('./package.json').version")
# Generate a beta tag using commit short sha and run id
COMMIT_SHA=$(git rev-parse --short HEAD)
DEV_TAG="dev.${COMMIT_SHA}-${GITHUB_RUN_ID}"
# Create full version with beta tag
BASE_VERSION=$(echo $CURRENT_VERSION | cut -f1,2 -d.)
PATCH_VERSION=$(echo $CURRENT_VERSION | cut -f3 -d.)
# bump patch version
NEW_PATCH_VERSION=$((PATCH_VERSION + 1))
NEW_VERSION="${BASE_VERSION}.${NEW_PATCH_VERSION}-${DEV_TAG}"
echo "NEW_VERSION=${NEW_VERSION}" >> "$GITHUB_ENV"
working-directory: ./clients/js/
- name: Update package.json with organization scope
run: |
ORG_NAME="@chroma-core"
PACKAGE_NAME=$(jq -r '.name' package.json)
jq --arg org "$ORG_NAME" --arg name "$PACKAGE_NAME" '.name = "\($org)/\($name)"' package.json > tmp.$$.json && mv tmp.$$.json package.json
working-directory: ./clients/js/
- name: Test & publish
run: npm run db:run && PORT=8001 npm run $NPM_SCRIPT
working-directory: ./clients/js/
env:
NODE_AUTH_TOKEN: ${{ matrix.registry == 'https://registry.npmjs.org' && secrets.NPM_TOKEN || secrets.GITHUB_TOKEN }}
Loading

0 comments on commit 4782dcc

Please sign in to comment.