diff --git a/.github/workflows/chroma-cluster-test.yml b/.github/workflows/chroma-cluster-test.yml index e474f43ca7d..bdcbcc96827 100644 --- a/.github/workflows/chroma-cluster-test.yml +++ b/.github/workflows/chroma-cluster-test.yml @@ -11,14 +11,14 @@ on: workflow_dispatch: jobs: - test: + test-python: strategy: matrix: python: ['3.8'] platform: ['16core-64gb-ubuntu-latest'] - testfile: ["chromadb/test/ingest/test_producer_consumer.py", - "chromadb/test/db/test_system.py", - "chromadb/test/segment/distributed/test_memberlist_provider.py",] + testfile: ["chromadb/test/db/test_system.py", + "chromadb/test/ingest/test_producer_consumer.py", + "chromadb/test/segment/distributed/test_memberlist_provider.py"] runs-on: ${{ matrix.platform }} steps: - name: Checkout @@ -29,14 +29,40 @@ jobs: python-version: ${{ matrix.python }} - name: Install test dependencies run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt - - name: Start minikube - id: minikube - uses: medyagh/setup-minikube@latest - with: - minikube-version: latest - kubernetes-version: latest - driver: docker - addons: ingress, ingress-dns - start-args: '--profile chroma-test' - - name: Integration Test - run: bin/cluster-test.sh ${{ matrix.testfile }} + - name: Install Tilt + run: | + TILT_VERSION="0.33.3" + curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \ + tar -xzv -C /usr/local/bin tilt + - name: Install ctlptlc + run: | + CTLPTL_VERSION="0.8.20" + curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \ + tar -xzv -C /usr/local/bin ctlptl + - name: Set up kind + run: ctlptl create cluster kind --registry=ctlptl-registry + - name: Start Tilt + run: tilt ci + - name: Test + run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.testfile }}"' + test-go: + runs-on: '16core-64gb-ubuntu-latest' + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Tilt + run: | + TILT_VERSION="0.33.3" + curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \ + tar -xzv -C /usr/local/bin tilt + - name: Install ctlptlc + run: | + CTLPTL_VERSION="0.8.20" + curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \ + tar -xzv -C /usr/local/bin ctlptl + - name: Set up kind + run: ctlptl create cluster kind --registry=ctlptl-registry + - name: Start Tilt + run: tilt ci + - name: Test + run: bin/cluster-test.sh bash -c 'cd go/coordinator && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma/chroma-coordinator/internal/memberlist_manager' \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index 0d0777199f2..b3747e7772d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,3 +1,5 @@ +update_settings(max_parallel_updates=6) + docker_build('migration', context='.', dockerfile='./go/coordinator/Dockerfile.migration' @@ -21,7 +23,7 @@ docker_build('worker', k8s_yaml(['k8s/dev/setup.yaml']) k8s_resource( - objects=['chroma:Namespace', 'memberlist-reader:ClusterRole', 'memberlist-reader:ClusterRoleBinding', 'pod-list-role:Role', 'pod-list-role-binding:RoleBinding', 'memberlists.chroma.cluster:CustomResourceDefinition','worker-memberlist:MemberList'], + objects=['chroma:Namespace', 'memberlist-reader:ClusterRole', 'memberlist-reader:ClusterRoleBinding', 'pod-list-role:Role', 'pod-list-role-binding:RoleBinding', 'memberlists.chroma.cluster:CustomResourceDefinition','worker-memberlist:MemberList', 'test-memberlist:MemberList'], new_name='k8s_setup', labels=["infrastructure"] ) @@ -31,11 +33,12 @@ k8s_yaml(['k8s/dev/postgres.yaml']) k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"]) k8s_yaml(['k8s/dev/migration.yaml']) k8s_resource('migration', resource_deps=['postgres'], labels=["chroma"]) +k8s_yaml(['k8s/dev/logservice.yaml']) +k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"]) +k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080']) k8s_yaml(['k8s/dev/server.yaml']) -k8s_resource('server', resource_deps=['k8s_setup'],labels=["chroma"], port_forwards=8000 ) +k8s_resource('server', resource_deps=['pulsar'],labels=["chroma"], port_forwards=8000 ) k8s_yaml(['k8s/dev/coordinator.yaml']) -k8s_resource('coordinator', resource_deps=['pulsar', 'server', 'migration'], labels=["chroma"], port_forwards=50051 ) -k8s_yaml(['k8s/dev/logservice.yaml']) -k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"], port_forwards='50052:50051') +k8s_resource('coordinator', resource_deps=['pulsar'], labels=["chroma"], port_forwards=50051) k8s_yaml(['k8s/dev/worker.yaml']) k8s_resource('worker', resource_deps=['coordinator'],labels=["chroma"]) diff --git a/bin/cluster-test.sh b/bin/cluster-test.sh index d18185b8c02..467478f206b 100755 --- a/bin/cluster-test.sh +++ b/bin/cluster-test.sh @@ -1,65 +1,22 @@ #!/usr/bin/env bash - set -e -function cleanup { - # Restore the previous kube context - kubectl config use-context $PREV_CHROMA_KUBE_CONTEXT - # Kill the tunnel process - kill $TUNNEL_PID - minikube delete -p chroma-test -} - -trap cleanup EXIT - -# Save the current kube context into a variable -export PREV_CHROMA_KUBE_CONTEXT=$(kubectl config current-context) - -# Create a new minikube cluster for the test -minikube start -p chroma-test - -# Add the ingress addon to the cluster -minikube addons enable ingress -p chroma-test -minikube addons enable ingress-dns -p chroma-test - -# Setup docker to build inside the minikube cluster and build the image -eval $(minikube -p chroma-test docker-env) -docker build -t server:latest -f Dockerfile . -docker build -t migration -f go/coordinator/Dockerfile.migration . -docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile . -docker build -t worker -f rust/worker/Dockerfile . --build-arg CHROMA_KUBERNETES_INTEGRATION=1 - -# Apply the kubernetes manifests -kubectl apply -f k8s/deployment -kubectl apply -f k8s/crd -kubectl apply -f k8s/cr -kubectl apply -f k8s/test - -# Wait for the pods in the chroma namespace to be ready -kubectl wait --for=condition=complete --timeout=100s job/migration -n chroma -kubectl delete job migration -n chroma -kubectl wait --namespace chroma --for=condition=Ready pods --all --timeout=400s - -# Run mini kube tunnel in the background to expose the service -minikube tunnel -c true -p chroma-test & -TUNNEL_PID=$! - -# Wait for the tunnel to be ready. There isn't an easy way to check if the tunnel is ready. So we just wait for 10 seconds -sleep 10 - +# TODO make url configuration consistent. export CHROMA_CLUSTER_TEST_ONLY=1 -export CHROMA_SERVER_HOST=$(kubectl get svc server -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}') -export PULSAR_BROKER_URL=$(kubectl get svc pulsar-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}') -export CHROMA_COORDINATOR_HOST=$(kubectl get svc coordinator-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}') +export CHROMA_SERVER_HOST=localhost:8000 +export PULSAR_BROKER_URL=localhost +export CHROMA_COORDINATOR_HOST=localhost export CHROMA_SERVER_GRPC_PORT="50051" + + echo "Chroma Server is running at port $CHROMA_SERVER_HOST" echo "Pulsar Broker is running at port $PULSAR_BROKER_URL" echo "Chroma Coordinator is running at port $CHROMA_COORDINATOR_HOST" -echo testing: python -m pytest "$@" -python -m pytest "$@" +kubectl -n chroma port-forward svc/coordinator 50051:50051 & +kubectl -n chroma port-forward svc/pulsar 6650:6650 & +kubectl -n chroma port-forward svc/pulsar 8080:8080 & +kubectl -n chroma port-forward svc/server 8000:8000 & -export CHROMA_KUBERNETES_INTEGRATION=1 -cd go/coordinator -go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma/chroma-coordinator/internal/memberlist_manager +"$@" diff --git a/k8s/dev/pulsar.yaml b/k8s/dev/pulsar.yaml index 4038ecda209..bcddf60c113 100644 --- a/k8s/dev/pulsar.yaml +++ b/k8s/dev/pulsar.yaml @@ -17,12 +17,19 @@ spec: - name: pulsar image: apachepulsar/pulsar command: [ "/pulsar/bin/pulsar", "standalone" ] + ports: - containerPort: 6650 - containerPort: 8080 volumeMounts: - name: pulsardata mountPath: /pulsar/data + readinessProbe: + httpGet: + path: /admin/v2/brokers/health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 5 volumes: - name: pulsardata emptyDir: {} diff --git a/k8s/dev/server.yaml b/k8s/dev/server.yaml index 9d76314e693..c7ab13df6d8 100644 --- a/k8s/dev/server.yaml +++ b/k8s/dev/server.yaml @@ -49,4 +49,17 @@ spec: - name: chroma emptyDir: {} - +--- +apiVersion: v1 +kind: Service +metadata: + name: server + namespace: chroma +spec: + ports: + - name: server-port + port: 8000 + targetPort: 8000 + selector: + app: server + type: ClusterIP diff --git a/k8s/dev/setup.yaml b/k8s/dev/setup.yaml index d9e1d95cc15..75478e0b6f5 100644 --- a/k8s/dev/setup.yaml +++ b/k8s/dev/setup.yaml @@ -97,4 +97,13 @@ metadata: name: worker-memberlist namespace: chroma spec: - members: \ No newline at end of file + members: +--- +apiVersion: chroma.cluster/v1 +kind: MemberList +metadata: + name: test-memberlist + namespace: chroma +spec: + members: +--- \ No newline at end of file