Add Load from s3, kafka-json & docker-compose deploy tutorial (#22)

* Bump mysql-connector-java from 8.0.16 to 8.0.28 in /SparkDemo Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.16 to 8.0.28. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](mysql/mysql-connector-j@8.0.16...8.0.28) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> * author Sida Shen <[email protected]> 1677748393 +0800 committer Sida Shen <[email protected]> 1679490760 +0800 Add Load from s3, deploy and kafka load tutorial Signed-off-by: Sida Shen <[email protected]> --------- Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: Sida Shen <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
StarRocks · Apr 7, 2023 · d2e1388 · d2e1388
1 parent 5677718
commit d2e1388
Show file tree

Hide file tree

Showing 10 changed files with 748 additions and 0 deletions.
diff --git a/deploy/docker-compose/docker-compose-3BE.yml b/deploy/docker-compose/docker-compose-3BE.yml
@@ -0,0 +1,71 @@
+version: "3.3"
+services:
+  starrocks-fe:
+    image: starrocks/fe-ubuntu:2.5.4
+    hostname: starrocks-fe
+    container_name: starrocks-fe
+    user: root
+    command: /opt/starrocks/fe/bin/start_fe.sh --host_type FQDN
+    ports:
+      - 1030:8030
+      - 2020:9020
+      - 3030:9030
+    volumes:
+#      - ../../conf/fe.conf:/opt/starrocks/fe/conf/fe.conf
+      - ./starrocks/fe/meta:/opt/starrocks/fe/meta
+      - ./starrocks/fe/log:/opt/starrocks/fe/log
+
+  starrocks-be1:
+    image: starrocks/be-ubuntu:2.5.4
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be1:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be1
+    container_name: starrocks-be1
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be1/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be1/log:/opt/starrocks/be/log
+  starrocks-be2:
+    image: starrocks/be-ubuntu:2.5.4
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s;  mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be2:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be2
+    container_name: starrocks-be2
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be2/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be2/log:/opt/starrocks/be/log
+  starrocks-be3:
+    image: starrocks/be-ubuntu:2.5.4
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be3:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be3
+    container_name: starrocks-be3
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be3/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be3/log:/opt/starrocks/be/log
diff --git a/deploy/docker-compose/docker-compose-shared-data.yml b/deploy/docker-compose/docker-compose-shared-data.yml
@@ -0,0 +1,99 @@
+version: "3"
+services:
+  minio:
+    container_name: starrocks-minio
+    image: minio/minio:latest
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - starrocks-miniodata:/minio_data
+    entrypoint: sh
+    command: '-c ''mkdir -p /minio_data/starrocks && minio server /minio_data --console-address ":9001"'''
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      network:
+        ipv4_address: 10.5.0.6
+
+  starrocks-fe:
+    image: starrocks/fe-ubuntu:3.0.0-rc01
+    hostname: starrocks-fe
+    container_name: starrocks-fe
+    user: root
+    command: >
+      bash -c "echo run_mode=shared_data >> /opt/starrocks/fe/conf/fe.conf &&
+      echo cloud_native_meta_port=6090 >> /opt/starrocks/fe/conf/fe.conf &&
+      echo aws_s3_path=starrocks >> /opt/starrocks/fe/conf/fe.conf &&
+      echo aws_s3_endpoint=minio:9000 >> /opt/starrocks/fe/conf/fe.conf &&
+      echo aws_s3_use_instance_profile=false >> /opt/starrocks/fe/conf/fe.conf &&
+      echo aws_s3_access_key=minioadmin >> /opt/starrocks/fe/conf/fe.conf &&
+      echo aws_s3_secret_key=minioadmin >> /opt/starrocks/fe/conf/fe.conf &&
+      sh /opt/starrocks/fe/bin/start_fe.sh"
+    ports:
+      - 8030:8030
+      - 9020:9020
+      - 9030:9030
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    depends_on:
+      - minio
+    volumes:
+      # - ./starrocks/starrocks-fe/conf:/opt/starrocks/fe/conf
+      - starrocks-fe-meta:/opt/starrocks/fe/meta
+      - starrocks-fe-log:/opt/starrocks/fe/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.2
+
+  starrocks-be:
+    image: starrocks/be-ubuntu:3.0.0-rc01
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; echo starlet_port = 8167 >> /opt/starrocks/be/conf/be.conf;
+        mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";"
+        /opt/starrocks/be/bin/start_be.sh
+    ports:
+      - 8040:8040
+    hostname: starrocks-be
+    container_name: starrocks-be
+    user: root
+    depends_on:
+      - starrocks-fe
+      - minio
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+#      - be.conf:/opt/starrocks/be/conf/be.conf
+#      - starrocks-be-storage:/opt/starrocks/be/storage
+      - starrocks-be-log:/opt/starrocks/be/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.3
+volumes:
+#  starrocks-be-storage:
+  starrocks-be-log:
+  starrocks-fe-meta:
+  starrocks-fe-log:
+  starrocks-miniodata:
+networks:
+  network:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.5.0.0/16
+          gateway: 10.5.0.1
diff --git a/deploy/docker-compose/docker-compose.yml b/deploy/docker-compose/docker-compose.yml
@@ -0,0 +1,60 @@
+version: "3"
+services:
+  starrocks-fe:
+    image: starrocks/fe-ubuntu:2.5.4
+    hostname: starrocks-fe
+    container_name: starrocks-fe
+    user: root
+    command:
+      /opt/starrocks/fe/bin/start_fe.sh
+    ports:
+      - 8030:8030
+      - 9020:9020
+      - 9030:9030
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+#      - fe.conf:/opt/starrocks/fe/conf/fe.conf
+      - ./starrocks/starrocks-fe/meta:/opt/starrocks/fe/meta
+      - ./starrocks/starrocks-fe/log:/opt/starrocks/fe/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.2
+
+  starrocks-be:
+    image: starrocks/be-ubuntu:2.5.4
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";"
+        /opt/starrocks/be/bin/start_be.sh
+    ports:
+      - 8040:8040
+    hostname: starrocks-be
+    container_name: starrocks-be
+    user: root
+    depends_on:
+      - starrocks-fe
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+#      - be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be/log:/opt/starrocks/be/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.3
+networks:
+  network:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.5.0.0/16
+          gateway: 10.5.0.1
diff --git a/loadFromObjectStoreDemo/broker_load_ssb.sql b/loadFromObjectStoreDemo/broker_load_ssb.sql
@@ -0,0 +1,69 @@
+-- BrokerLoad scripts to SSB tables from an AWS S3 bucket to StarRocks
+
+-- This script uses instance profile authentication method. View this documentation for more supported authentication
+-- methods: https://docs.starrocks.io/en-us/latest/integrations/authenticate_to_aws_resources
+
+-- Please edit the S3 path, table name and s3 endpoint.
+-- See broker load here: https://docs.starrocks.io/en-us/latest/loading/BrokerLoad
+
+USE load_broker;
+
+LOAD LABEL load_broker.lineorder
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/lineorder/*")
+    INTO TABLE lineorder
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.customer
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/customer/*")
+    INTO TABLE customer
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.dates
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/dates/*")
+    INTO TABLE dates
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.part
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/part/*")
+    INTO TABLE part
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.supplier
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/supplier/*")
+    INTO TABLE supplier
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);