Skip to content

Commit

Permalink
Merge branch 'master' into optimize-canal-json-encoder
Browse files Browse the repository at this point in the history
  • Loading branch information
3AceShowHand authored Oct 17, 2022
2 parents 1ed1d2c + 2b00b5a commit 84ef430
Show file tree
Hide file tree
Showing 40 changed files with 687 additions and 293 deletions.
33 changes: 32 additions & 1 deletion .github/workflows/dataflow_engine_chaos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,38 @@ jobs:
echo describe pvc
kubectl describe pvc -l app=tidb
kubectl wait --for=condition=Ready pod/tidb-0 --timeout=0s
# Set up minio and create a bucket for tests
- name: Set up minio
run: |
kubectl apply -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio.yaml
kubectl get -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio.yaml
kubectl describe -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio.yaml
- name: Wait for minio ready
run: |
kubectl wait --for=condition=Ready pod/chaos-minio-0 --timeout=10m || true
echo show pvc
kubectl get pvc -l app=minio -o wide
echo show pv
kubectl get pv -o wide
echo show svc
kubectl get svc -l app=minio -o wide
echo show sts
kubectl get sts -l app=minio -o wide
echo show po
kubectl get po -l app=minio -o wide
echo describe po
kubectl describe po -l app=minio
echo describe pvc
kubectl describe pvc -l app=minio
kubectl wait --for=condition=Ready pod/chaos-minio-0 --timeout=0s
- name: Set up minio-create-bucket job
run: |
kubectl apply -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio-create-bucket.yaml
kubectl get -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio-create-bucket.yaml
kubectl describe -f $GITHUB_WORKSPACE/deployments/engine/helm/minio/minio-create-bucket.yaml
kubectl wait --for=condition=Complete job/chaos-minio-create-bucket-job --timeout=2m
# Set up metastore and basic services
- name: Set up metastore and basic services
run: |
Expand Down
105 changes: 0 additions & 105 deletions .github/workflows/dataflow_engine_e2e.yaml

This file was deleted.

6 changes: 3 additions & 3 deletions deployments/engine/docker-compose/1minio.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '2.3'

services:
minio:
minio-standalone:
image: minio/minio
container_name: minio-standalone
command: server --console-address ":9001" /data
Expand All @@ -21,10 +21,10 @@ services:
createbuckets:
image: minio/mc
depends_on:
- minio
- minio-standalone
entrypoint: >
/bin/sh -c "
/usr/bin/mc alias set myminio http://minio:9000 engine engineSecret;
/usr/bin/mc alias set myminio http://minio-standalone:9000 engine engineSecret;
/usr/bin/mc mb myminio/engine-ut;
/usr/bin/mc version enable myminio/engine-ut;
exit 0;
Expand Down
194 changes: 194 additions & 0 deletions deployments/engine/docker-compose/3m3e_with_s3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
version: "2.3"
services:
server-master-0:
image: dataflow:test
container_name: server-master-0
ports:
- "10245:10240"
volumes:
- ./config/master_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "master"
- "--name=server-master-0"
- "--addr=0.0.0.0:10240"
- "--advertise-addr=server-master-0:10240"
- "--config=/config.toml"
- "--log-file=/log/server-master-0.log"
depends_on:
"etcd-standalone":
condition: service_started
"mysql-standalone":
condition: service_healthy
"minio-standalone":
condition: service_healthy
server-master-1:
image: dataflow:test
container_name: server-master-1
ports:
- "10246:10240"
volumes:
- ./config/master_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "master"
- "--name=server-master-1"
- "--addr=0.0.0.0:10240"
- "--advertise-addr=server-master-1:10240"
- "--config=/config.toml"
- "--log-file=/log/server-master-1.log"
depends_on:
"etcd-standalone":
condition: service_started
"mysql-standalone":
condition: service_healthy
"minio-standalone":
condition: service_healthy
server-master-2:
image: dataflow:test
container_name: server-master-2
ports:
- "10247:10240"
volumes:
- ./config/master_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "master"
- "--name=server-master-2"
- "--addr=0.0.0.0:10240"
- "--advertise-addr=server-master-2:10240"
- "--config=/config.toml"
- "--log-file=/log/server-master-2.log"
depends_on:
"etcd-standalone":
condition: service_started
"mysql-standalone":
condition: service_healthy
"minio-standalone":
condition: service_healthy
server-executor-0:
image: dataflow:test
container_name: server-executor-0
ports:
- "11241:10241"
volumes:
- ./config/executor_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "executor"
- "--name=server-executor-0"
- "--addr=0.0.0.0:10241"
- "--advertise-addr=server-executor-0:10241"
- "--join=server-master-0:10240,server-master-1:10240,server-master-2:10240"
- "--config=/config.toml"
- "--log-file=/log/server-executor-0.log"
- "--labels=\"name=exec-0\""
depends_on:
- "server-master-0"
- "server-master-2"
- "server-master-1"
extra_hosts:
- "host.docker.internal:host-gateway"
server-executor-1:
image: dataflow:test
container_name: server-executor-1
ports:
- "11242:10241"
volumes:
- ./config/executor_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "executor"
- "--name=server-executor-1"
- "--addr=0.0.0.0:10241"
- "--advertise-addr=server-executor-1:10241"
- "--join=server-master-0:10240,server-master-1:10240,server-master-2:10240"
- "--config=/config.toml"
- "--log-file=/log/server-executor-1.log"
- "--labels=\"name=exec-1\""
depends_on:
- "server-master-0"
- "server-master-2"
- "server-master-1"
extra_hosts:
- "host.docker.internal:host-gateway"
server-executor-2:
image: dataflow:test
container_name: server-executor-2
ports:
- "11243:10241"
volumes:
- ./config/executor_with_s3.toml:/config.toml
- /tmp/tiflow_engine_test:/log
command:
- "/tiflow"
- "executor"
- "--name=server-executor-2"
- "--addr=0.0.0.0:10241"
- "--advertise-addr=server-executor-2:10241"
- "--join=server-master-0:10240,server-master-1:10240,server-master-2:10240"
- "--config=/config.toml"
- "--log-file=/log/server-executor-2.log"
- "--labels=\"name=exec-2\""
depends_on:
- "server-master-0"
- "server-master-2"
- "server-master-1"
extra_hosts:
- "host.docker.internal:host-gateway"
etcd-standalone:
image: quay.io/coreos/etcd
container_name: etcd-standalone
command:
- "etcd"
- "--listen-client-urls=http://0.0.0.0:2379"
- "--advertise-client-urls=http://etcd-standalone:2379"
ports:
- "12479:2379"
mysql-standalone:
image: mysql:8.0
container_name: mysql-standalone
command: --default-authentication-plugin=mysql_native_password
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
volumes:
- './config/mysql_meta.cnf:/etc/my.cnf'
ports:
- "3336:3306"
healthcheck:
test: mysql -h127.0.0.1 -P3306 -e "show databases"
interval: 10s
timeout: 600s
retries: 60
minio-standalone:
image: minio/minio
container_name: minio-standalone
command: server --console-address ":9001" /data
ports:
- "9000:9000"
- "9001:9001"
environment:
MINIO_ROOT_USER: engine
MINIO_ROOT_PASSWORD: engineSecret
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 10s
timeout: 60s
retries: 6
minio-createbuckets:
image: minio/mc
container_name: minio-createbuckets
depends_on:
- minio-standalone
entrypoint: >
/bin/sh -c "
/usr/bin/mc alias set myminio http://minio-standalone:9000 engine engineSecret || exit 1;
/usr/bin/mc mb myminio/engine-it || exit 1;
/usr/bin/mc version enable myminio/engine-ut;
exit 0;
"
Loading

0 comments on commit 84ef430

Please sign in to comment.