update kube docs (#4749)

.bumpversion.cfg

@@ -19,3 +19,7 @@ serialize =
 [bumpversion:file:kube/overlays/stable/.env]
 
 [bumpversion:file:kube/overlays/stable/kustomization.yaml]
+
+[bumpversion:file:kube/overlays/stable-with-resource-limits/.env]
+
+[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml]

docs/deploying-airbyte/on-kubernetes.md

@@ -10,7 +10,7 @@ Airbyte allows scaling sync workloads horizontally using Kubernetes. The core co
 For local testing we recommend following one of the following setup guides:
 * [Docker Desktop (Mac)](https://docs.docker.com/desktop/kubernetes/)
 * [Minikube](https://minikube.sigs.k8s.io/docs/start/)
-  * NOTE: Start Minikube with at least 4gb RAM to Minikube with `minikube start --memory=4000`
+  * NOTE: Start Minikube with at least 4gb RAM with `minikube start --memory=4000`
 * [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
 
 For testing on GKE you can [create a cluster with the command line or the Cloud Console UI](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster).
@@ -135,6 +135,17 @@ Now visit [http://localhost:8000](http://localhost:8000) in your browser and sta
 
 ## Production Airbyte on Kubernetes
 
+### Setting resource limits
+
+* Core container pods
+  * Instead of launching Airbyte with `kubectl apply -k kube/overlays/stable`, you can run with `kubectl apply -k kube/overlays/stable-with-resource-limits`.
+  * The `kube/overlays/stable-with-resource-limits/set-resource-limits.yaml` file can be modified to provide different resource requirements for core pods.
+* Connector pods
+  * By default, connector pods launch without resource limits.
+  * To add resource limits, configure the "Docker Resource Limits" section of the `.env` file in the overlay folder you're using.
+* Volume sizes
+  * You can modify `kube/resources/volume-*` files to specify different volume sizes for the persistent volumes backing Airbyte.
+
 ### Cloud logging
 
 Airbyte writes logs to two directories. App logs, including server and scheduler logs, are written to the `app-logging` directory.
@@ -163,10 +174,6 @@ there are any other issues blocking your adoption of Airbyte or if you would lik
 
 * The server and scheduler deployments must run on the same node. ([#4232](https://github.com/airbytehq/airbyte/issues/4232))
 * Some UI operations have higher latency on Kubernetes than Docker-Compose. ([#4233](https://github.com/airbytehq/airbyte/issues/4233))
-* Pod histories must be cleaned up manually. ([#3634](https://github.com/airbytehq/airbyte/issues/3634))
-* Specifying resource limits for pods is not supported yet. ([#3638](https://github.com/airbytehq/airbyte/issues/3638))
-* Pods Airbyte launches to run connector jobs are always launched in the `default` namespace. ([#3636](https://github.com/airbytehq/airbyte/issues/3636))
-* S3 is the only Cloud Storage currently supported. ([#4200](https://github.com/airbytehq/airbyte/issues/4200))
 * Large log files might take a while to load. ([#4201](https://github.com/airbytehq/airbyte/issues/4201))
 * UI does not include configured buckets in the displayed log path. ([#4204](https://github.com/airbytehq/airbyte/issues/4204))
 * Logs are not reset when Airbyte is re-deployed. ([#4235](https://github.com/airbytehq/airbyte/issues/4235))

kube/overlays/stable-with-resource-limits/.env

@@ -0,0 +1,55 @@
+AIRBYTE_VERSION=0.27.2-alpha
+
+# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+DATABASE_HOST=airbyte-db-svc
+DATABASE_PORT=5432
+DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
+DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+
+# When using the airbyte-db via default docker image:
+CONFIG_ROOT=/configs
+DATA_DOCKER_MOUNT=airbyte_data
+DB_DOCKER_MOUNT=airbyte_db
+
+# Temporal.io worker configuration
+TEMPORAL_HOST=airbyte-temporal-svc:7233
+TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030
+
+# Workspace storage for running jobs (logs, etc)
+WORKSPACE_ROOT=/workspace
+WORKSPACE_DOCKER_MOUNT=airbyte_workspace
+
+LOCAL_ROOT=/tmp/airbyte_local
+
+# Miscellaneous
+TRACKING_STRATEGY=segment
+WEBAPP_URL=airbyte-webapp-svc:80
+API_URL=/api/v1/
+INTERNAL_API_HOST=airbyte-server-svc:8001
+
+WORKER_ENVIRONMENT=kubernetes
+PAPERCUPS_STORYTIME=enabled
+FULLSTORY=enabled
+IS_DEMO=false
+LOG_LEVEL=INFO
+
+# S3/Minio Log Configuration
+S3_LOG_BUCKET=airbyte-dev-logs
+S3_LOG_BUCKET_REGION=us-east-1
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
+S3_PATH_STYLE_ACCESS=true
+
+# GCS Log Configuration
+GCP_STORAGE_BUCKET=
+GOOGLE_APPLICATION_CREDENTIALS=
+
+# Docker Resource Limits
+RESOURCE_CPU_REQUEST=
+RESOURCE_CPU_LIMIT=
+RESOURCE_MEMORY_REQUEST=
+RESOURCE_MEMORY_LIMIT=

kube/overlays/stable-with-resource-limits/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: default
+
+bases:
+  - ../../resources
+
+images:
+  - name: airbyte/seed
+    newTag: 0.27.2-alpha
+  - name: airbyte/db
+    newTag: 0.27.2-alpha
+  - name: airbyte/scheduler
+    newTag: 0.27.2-alpha
+  - name: airbyte/server
+    newTag: 0.27.2-alpha
+  - name: airbyte/webapp
+    newTag: 0.27.2-alpha
+  - name: temporalio/auto-setup
+    newTag: 1.7.0
+
+configMapGenerator:
+  - name: airbyte-env
+    env: .env
+
+patchesStrategicMerge:
+  - set-resource-limits.yaml

kube/overlays/stable/set-resource-limits.yaml → kube/overlays/stable-with-resource-limits/set-resource-limits.yaml

@@ -10,7 +10,7 @@ spec:
           resources:
             limits:
               cpu: 2
-              memory: 4Gi
+              memory: 2Gi
 ---
 apiVersion: apps/v1
 kind: Deployment

kube/overlays/stable/kustomization.yaml

@@ -23,6 +23,3 @@ images:
 configMapGenerator:
   - name: airbyte-env
     env: .env
-
-patchesStrategicMerge:
-  - set-resource-limits.yaml