make it all work

Tiltfile

@@ -1,60 +1,93 @@
 update_settings(max_parallel_updates=6)
 
-docker_build('migration',
-             context='.',
-             dockerfile='./go/Dockerfile.migration'
+docker_build(
+  'migration',
+  context='.',
+  dockerfile='./go/Dockerfile.migration'
 )
 
-docker_build('coordinator',
-             context='.',
-             dockerfile='./go/Dockerfile'
+docker_build(
+  'coordinator',
+  context='.',
+  dockerfile='./go/Dockerfile'
 )
 
-docker_build('server',
-             context='.',
-             dockerfile='./Dockerfile',
+docker_build(
+  'server',
+  context='.',
+  dockerfile='./Dockerfile',
 )
 
-docker_build('worker',
-             context='.',
-             dockerfile='./rust/worker/Dockerfile'
+docker_build(
+  'worker',
+  context='.',
+  dockerfile='./rust/worker/Dockerfile'
 )
 
-# Actual chroma
-k8s_yaml(helm('k8s/distributed-chroma',
-              namespace='chroma',
-              values=['k8s/distributed-chroma/values.yaml']
-              ))
+k8s_yaml(
+  helm(
+    'k8s/distributed-chroma',
+    namespace='chroma',
+    values=[
+      'k8s/distributed-chroma/values.yaml'
+    ]
+  )
+)
 
-# Extra stuff to make debugging and testing easier
-k8s_yaml('k8s/test/coordinator_service.yaml')
-k8s_yaml('k8s/test/minio.yaml')
-k8s_yaml('k8s/test/pulsar_service.yaml')
-k8s_yaml('k8s/test/segment_server_service.yaml')
-k8s_yaml('k8s/test/test_memberlist_cr.yaml')
-
-# Lots of things assume the cluster is in a basic state, so we need to get it
-# into a basic state before we can start deploying things.
+# Lots of things assume the cluster is in a basic state. Get it into a basic
+# state before deploying anything else.
 k8s_resource(
-  objects=['chroma:Namespace',
-           'worker-memberlist-readerwriter:ClusterRole',
-           'pod-watcher:Role',
-           'coordinator-serviceaccount-rolebinding:RoleBinding',
-           'segmentserver-serviceaccount-rolebinding:RoleBinding',
-           'memberlists.chroma.cluster:CustomResourceDefinition',
-           'worker-memberlist:MemberList',
-           'test-memberlist:MemberList',
-           'test-memberlist-reader:ClusterRole',
-           'test-memberlist-reader-binding:ClusterRoleBinding',
-           ],
+  objects=[
+    'chroma:Namespace',
+    'pod-watcher:Role',
+    'memberlists.chroma.cluster:CustomResourceDefinition',
+    'worker-memberlist:MemberList',
+
+    'coordinator-serviceaccount:serviceaccount',
+    'coordinator-serviceaccount-rolebinding:RoleBinding',
+    'coordinator-worker-memberlist-binding:clusterrolebinding',
+
+    'worker-serviceaccount:serviceaccount',
+    'worker-serviceaccount-rolebinding:RoleBinding',
+    'worker-memberlist-readerwriter:ClusterRole',
+    'worker-worker-memberlist-binding:clusterrolebinding',
+    'worker-memberlist-readerwriter-binding:clusterrolebinding',
+
+    'test-memberlist:MemberList',
+    'test-memberlist-reader:ClusterRole',
+    'test-memberlist-reader-binding:ClusterRoleBinding',
+  ],
   new_name='k8s_setup',
   labels=["infrastructure"]
 )
 
+# Production Chroma
 k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"])
 k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080'])
 k8s_resource('migration', resource_deps=['postgres'], labels=["chroma"])
 k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"])
 k8s_resource('frontend-server', resource_deps=['pulsar'],labels=["chroma"], port_forwards=8000 )
 k8s_resource('coordinator', resource_deps=['pulsar'], labels=["chroma"], port_forwards=50051)
 k8s_resource('worker', resource_deps=['coordinator'],labels=["chroma"])
+
+# Extra stuff to make debugging and testing easier
+k8s_yaml([
+  'k8s/test/coordinator_service.yaml',
+  'k8s/test/minio.yaml',
+  'k8s/test/pulsar_service.yaml',
+  'k8s/test/worker_service.yaml',
+  'k8s/test/test_memberlist_cr.yaml',
+])
+k8s_resource(
+  objects=[
+    # I don't know why but Tilt denies the existence of 'coordinator:service' et al
+    # when you try to add them here.
+    'worker:service',
+  ],
+  new_name='debug_setup',
+  resource_deps=['worker'],
+  labels=["debug"],
+)
+
+# Local S3
+k8s_resource('minio-deployment', resource_deps=['k8s_setup'], labels=["debug"], port_forwards=9000)

k8s/distributed-chroma/templates/worker.yaml

@@ -1,10 +1,29 @@
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: worker
+  namespace: chroma
+spec:
+  ports:
+    - name: worker-server-port
+      port: 50051
+      targetPort: 50051
+  selector:
+    app: worker-server
+  type: ClusterIP
+
+---
+
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: worker
-  namespace: {{ .Values.namespace }}
+  namespace: chroma
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: worker
@@ -14,6 +33,7 @@ spec:
         app: worker
         member-type: worker
     spec:
+      serviceAccountName: worker-serviceaccount
       containers:
         - name: worker
           image: worker
@@ -27,14 +47,43 @@ spec:
           env:
             - name: CHROMA_WORKER__PULSAR_URL
               value: pulsar://pulsar.chroma:6650
-            - name: CHROMA_WORKER__PULSAR_NAMESPACE
-              value: default
-            - name: CHROMA_WORKER__PULSAR_TENANT
-              value: default
             - name: CHROMA_WORKER__MY_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: "kubernetes.io/hostname"
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              member-type: worker
       volumes:
         - name: chroma
-          emptyDir: {}
+          emptyDir: {}
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: worker-serviceaccount
+  namespace: chroma
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: worker-serviceaccount-rolebinding
+  namespace: chroma
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-watcher
+subjects:
+- kind: ServiceAccount
+  name: worker-serviceaccount
+  namespace: chroma
+
+---

k8s/distributed-chroma/templates/worker_memberlist_cr.yaml

@@ -2,7 +2,6 @@
 # They will be used for the upcoming distributed version of chroma. They are not even ready
 # for testing yet. Please do not use them unless you are working on the distributed version of chroma.
 
-# Create a memberlist called worker-memberlist
 apiVersion: chroma.cluster/v1
 kind: MemberList
 metadata:
@@ -49,6 +48,23 @@ subjects:
 
 ---
 
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  # Awkward name, but this lets the worker-serviceaccount read
+  # the worker-memberlist.
+  name: worker-worker-memberlist-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: worker-memberlist-readerwriter
+subjects:
+- kind: ServiceAccount
+  name: worker-serviceaccount
+  namespace: chroma
+
+---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:

k8s/test/minio.yaml

@@ -18,25 +18,25 @@ spec:
         - name: minio
           emptyDir: {}
       containers:
-      - name: minio
-        image: minio/minio:latest
-        args:
-        - server
-        - /storage
-        env:
-        - name: MINIO_ACCESS_KEY
-          value: "minio"
-        - name: MINIO_SECRET_KEY
-          value: "minio123"
-        ports:
-        - containerPort: 9000
-          hostPort: 9000
-        volumeMounts:
         - name: minio
-          mountPath: /storage
+          image: minio/minio:latest
+          args:
+            - server
+            - /storage
+          env:
+            - name: MINIO_ACCESS_KEY
+              value: "minio"
+            - name: MINIO_SECRET_KEY
+              value: "minio123"
+          ports:
+            - containerPort: 9000
+              hostPort: 9000
+              name: http
+          volumeMounts:
+            - name: minio
+              mountPath: /storage
 
 ---
-
 apiVersion: v1
 kind: Service
 metadata: