[PERF]: use tilt test ci (chroma-core#1727)

## Description of changes

*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
	 - Replace minikube by tilt to match local setup
	 - Refactor workflow

---------

Co-authored-by: nicolas <[email protected]>

.github/workflows/chroma-cluster-test.yml

@@ -11,14 +11,14 @@ on:
   workflow_dispatch:
 
 jobs:
-  test:
+  test-python:
     strategy:
       matrix:
         python: ['3.8']
         platform: ['16core-64gb-ubuntu-latest']
-        testfile: ["chromadb/test/ingest/test_producer_consumer.py",
-                   "chromadb/test/db/test_system.py",
-                   "chromadb/test/segment/distributed/test_memberlist_provider.py",]
+        testfile: ["chromadb/test/db/test_system.py",
+                   "chromadb/test/ingest/test_producer_consumer.py",
+                   "chromadb/test/segment/distributed/test_memberlist_provider.py"]
     runs-on: ${{ matrix.platform }}
     steps:
     - name: Checkout
@@ -29,14 +29,40 @@ jobs:
         python-version: ${{ matrix.python }}
     - name: Install test dependencies
       run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
-    - name: Start minikube
-      id: minikube
-      uses: medyagh/setup-minikube@latest
-      with:
-        minikube-version: latest
-        kubernetes-version: latest
-        driver: docker
-        addons: ingress, ingress-dns
-        start-args: '--profile chroma-test'
-    - name: Integration Test
-      run: bin/cluster-test.sh ${{ matrix.testfile }}
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.testfile }}"'
+  test-go:
+    runs-on: '16core-64gb-ubuntu-latest'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'cd go/coordinator && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma/chroma-coordinator/internal/memberlist_manager'

Tiltfile

@@ -1,3 +1,5 @@
+update_settings(max_parallel_updates=6)
+
 docker_build('migration',
              context='.',
              dockerfile='./go/coordinator/Dockerfile.migration'
@@ -21,7 +23,7 @@ docker_build('worker',
 
 k8s_yaml(['k8s/dev/setup.yaml'])
 k8s_resource(
-  objects=['chroma:Namespace', 'memberlist-reader:ClusterRole', 'memberlist-reader:ClusterRoleBinding', 'pod-list-role:Role', 'pod-list-role-binding:RoleBinding', 'memberlists.chroma.cluster:CustomResourceDefinition','worker-memberlist:MemberList'],
+  objects=['chroma:Namespace', 'memberlist-reader:ClusterRole', 'memberlist-reader:ClusterRoleBinding', 'pod-list-role:Role', 'pod-list-role-binding:RoleBinding', 'memberlists.chroma.cluster:CustomResourceDefinition','worker-memberlist:MemberList', 'test-memberlist:MemberList'],
   new_name='k8s_setup',
   labels=["infrastructure"]
 )
@@ -31,11 +33,12 @@ k8s_yaml(['k8s/dev/postgres.yaml'])
 k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"])
 k8s_yaml(['k8s/dev/migration.yaml'])
 k8s_resource('migration', resource_deps=['postgres'], labels=["chroma"])
+k8s_yaml(['k8s/dev/logservice.yaml'])
+k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"])
+k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080'])
 k8s_yaml(['k8s/dev/server.yaml'])
-k8s_resource('server', resource_deps=['k8s_setup'],labels=["chroma"], port_forwards=8000 )
+k8s_resource('server', resource_deps=['pulsar'],labels=["chroma"], port_forwards=8000 )
 k8s_yaml(['k8s/dev/coordinator.yaml'])
-k8s_resource('coordinator', resource_deps=['pulsar', 'server', 'migration'], labels=["chroma"], port_forwards=50051 )
-k8s_yaml(['k8s/dev/logservice.yaml'])
-k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"], port_forwards='50052:50051')
+k8s_resource('coordinator', resource_deps=['pulsar'], labels=["chroma"], port_forwards=50051)
 k8s_yaml(['k8s/dev/worker.yaml'])
 k8s_resource('worker', resource_deps=['coordinator'],labels=["chroma"])

bin/cluster-test.sh

@@ -1,65 +1,22 @@
 #!/usr/bin/env bash
-
 set -e
 
-function cleanup {
-    # Restore the previous kube context
-    kubectl config use-context $PREV_CHROMA_KUBE_CONTEXT
-    # Kill the tunnel process
-    kill $TUNNEL_PID
-    minikube delete -p chroma-test
-}
-
-trap cleanup EXIT
-
-# Save the current kube context into a variable
-export PREV_CHROMA_KUBE_CONTEXT=$(kubectl config current-context)
-
-# Create a new minikube cluster for the test
-minikube start -p chroma-test
-
-# Add the ingress addon to the cluster
-minikube addons enable ingress -p chroma-test
-minikube addons enable ingress-dns -p chroma-test
-
-# Setup docker to build inside the minikube cluster and build the image
-eval $(minikube -p chroma-test docker-env)
-docker build -t server:latest -f Dockerfile .
-docker build -t migration -f go/coordinator/Dockerfile.migration .
-docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile .
-docker build -t worker -f rust/worker/Dockerfile . --build-arg CHROMA_KUBERNETES_INTEGRATION=1
-
-# Apply the kubernetes manifests
-kubectl apply -f k8s/deployment
-kubectl apply -f k8s/crd
-kubectl apply -f k8s/cr
-kubectl apply -f k8s/test
-
-# Wait for the pods in the chroma namespace to be ready
-kubectl wait --for=condition=complete --timeout=100s job/migration -n chroma
-kubectl delete job migration -n chroma
-kubectl wait --namespace chroma --for=condition=Ready pods --all --timeout=400s
-
-# Run mini kube tunnel in the background to expose the service
-minikube tunnel -c true -p chroma-test &
-TUNNEL_PID=$!
-
-# Wait for the tunnel to be ready. There isn't an easy way to check if the tunnel is ready. So we just wait for 10 seconds
-sleep 10
-
+# TODO make url configuration consistent.
 export CHROMA_CLUSTER_TEST_ONLY=1
-export CHROMA_SERVER_HOST=$(kubectl get svc server -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-export PULSAR_BROKER_URL=$(kubectl get svc pulsar-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-export CHROMA_COORDINATOR_HOST=$(kubectl get svc coordinator-lb -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export CHROMA_SERVER_HOST=localhost:8000
+export PULSAR_BROKER_URL=localhost
+export CHROMA_COORDINATOR_HOST=localhost
 export CHROMA_SERVER_GRPC_PORT="50051"
 
+
+
 echo "Chroma Server is running at port $CHROMA_SERVER_HOST"
 echo "Pulsar Broker is running at port $PULSAR_BROKER_URL"
 echo "Chroma Coordinator is running at port $CHROMA_COORDINATOR_HOST"
 
-echo testing: python -m pytest "$@"
-python -m pytest "$@"
+kubectl -n chroma port-forward svc/coordinator 50051:50051 &
+kubectl -n chroma port-forward svc/pulsar 6650:6650 &
+kubectl -n chroma port-forward svc/pulsar 8080:8080 &
+kubectl -n chroma port-forward svc/server 8000:8000 &
 
-export CHROMA_KUBERNETES_INTEGRATION=1
-cd go/coordinator
-go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma/chroma-coordinator/internal/memberlist_manager
+"$@"

k8s/dev/pulsar.yaml

@@ -17,12 +17,19 @@ spec:
         - name: pulsar
           image: apachepulsar/pulsar
           command: [ "/pulsar/bin/pulsar", "standalone" ]
+
           ports:
             - containerPort: 6650
             - containerPort: 8080
           volumeMounts:
             - name: pulsardata
               mountPath: /pulsar/data
+          readinessProbe:
+             httpGet:
+               path: /admin/v2/brokers/health
+               port: 8080
+             initialDelaySeconds: 10
+             periodSeconds: 5
       volumes:
         - name: pulsardata
           emptyDir: {}

k8s/dev/server.yaml

@@ -49,4 +49,17 @@ spec:
         - name: chroma
           emptyDir: {}
 
-
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: server
+  namespace: chroma
+spec:
+  ports:
+    - name: server-port
+      port: 8000
+      targetPort: 8000
+  selector:
+    app: server
+  type: ClusterIP

k8s/dev/setup.yaml

@@ -97,4 +97,13 @@ metadata:
   name: worker-memberlist
   namespace: chroma
 spec:
-  members:
+  members:
+---
+apiVersion: chroma.cluster/v1
+kind: MemberList
+metadata:
+  name: test-memberlist
+  namespace: chroma
+spec:
+  members:
+---