parent 076b70f

author Sida Shen <[email protected]> 1677748393 +0800 committer Sida Shen <[email protected]> 1679490760 +0800 Add Load from s3, deploy and kafka load tutorial Signed-off-by: Sida Shen <[email protected]>
StarRocks · Mar 22, 2023 · 0885a8c · 0885a8c
1 parent 076b70f
commit 0885a8c
Show file tree

Hide file tree

Showing 9 changed files with 649 additions and 0 deletions.
diff --git a/deploy/docker-compose/docker-compose-3BE.yml b/deploy/docker-compose/docker-compose-3BE.yml
@@ -0,0 +1,71 @@
+version: "3.3"
+services:
+  starrocks-fe:
+    image: starrocks/fe-ubuntu:2.5.2
+    hostname: starrocks-fe
+    container_name: starrocks-fe
+    user: root
+    command: /opt/starrocks/fe/bin/start_fe.sh --host_type FQDN
+    ports:
+      - 1030:8030
+      - 2020:9020
+      - 3030:9030
+    volumes:
+#      - ../../conf/fe.conf:/opt/starrocks/fe/conf/fe.conf
+      - ./starrocks/fe/meta:/opt/starrocks/fe/meta
+      - ./starrocks/fe/log:/opt/starrocks/fe/log
+
+  starrocks-be1:
+    image: starrocks/be-ubuntu:2.5.2
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be1:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be1
+    container_name: starrocks-be1
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be1/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be1/log:/opt/starrocks/be/log
+  starrocks-be2:
+    image: starrocks/be-ubuntu:2.5.2
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s;  mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be2:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be2
+    container_name: starrocks-be2
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be2/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be2/log:/opt/starrocks/be/log
+  starrocks-be3:
+    image: starrocks/be-ubuntu:2.5.2
+    user: root
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be3:9050\";"
+        /opt/starrocks/be/bin/start_be.sh 
+
+    hostname: starrocks-be3
+    container_name: starrocks-be3
+    depends_on:
+      - "starrocks-fe"
+    volumes:
+#      - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be3/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be3/log:/opt/starrocks/be/log
diff --git a/deploy/docker-compose/docker-compose.yml b/deploy/docker-compose/docker-compose.yml
@@ -0,0 +1,60 @@
+version: "3.3"
+services:
+  starrocks-fe:
+    image: starrocks/fe-ubuntu:2.5.2
+    hostname: starrocks-fe
+    container_name: starrocks-fe
+    user: root
+    command:
+      /opt/starrocks/fe/bin/start_fe.sh
+    ports:
+      - 8030:8030
+      - 9020:9020
+      - 9030:9030
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+#      - fe.conf:/opt/starrocks/fe/conf/fe.conf
+      - ./starrocks/starrocks-fe/meta:/opt/starrocks/fe/meta
+      - ./starrocks/starrocks-fe/log:/opt/starrocks/fe/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.2
+
+  starrocks-be:
+    image: starrocks/be-ubuntu:2.5.2
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";"
+        /opt/starrocks/be/bin/start_be.sh
+    ports:
+      - 8040:8040
+    hostname: starrocks-be
+    container_name: starrocks-be
+    user: root
+    depends_on:
+      - starrocks-fe
+    healthcheck:
+      test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"'
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+#      - be.conf:/opt/starrocks/be/conf/be.conf
+      - ./starrocks/starrocks-be/storage:/opt/starrocks/be/storage
+      - ./starrocks/starrocks-be/log:/opt/starrocks/be/log
+    networks:
+      network:
+        ipv4_address: 10.5.0.3
+networks:
+  network:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.5.0.0/16
+          gateway: 10.5.0.1
diff --git a/loadFromObjectStoreDemo/broker_load_ssb.sql b/loadFromObjectStoreDemo/broker_load_ssb.sql
@@ -0,0 +1,69 @@
+-- BrokerLoad scripts to SSB tables from an AWS S3 bucket to StarRocks
+
+-- This script uses instance profile authentication method. View this documentation for more supported authentication
+-- methods: https://docs.starrocks.io/en-us/latest/integrations/authenticate_to_aws_resources
+
+-- Please edit the S3 path, table name and s3 endpoint.
+-- See broker load here: https://docs.starrocks.io/en-us/latest/loading/BrokerLoad
+
+USE load_broker;
+
+LOAD LABEL load_broker.lineorder
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/lineorder/*")
+    INTO TABLE lineorder
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.customer
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/customer/*")
+    INTO TABLE customer
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.dates
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/dates/*")
+    INTO TABLE dates
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.part
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/part/*")
+    INTO TABLE part
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
+
+LOAD LABEL load_broker.supplier
+(
+    DATA INFILE("s3a://<your-bucket>/ssb_50mb_parquet/supplier/*")
+    INTO TABLE supplier
+    FORMAT AS "parquet"
+)
+WITH BROKER
+(
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com" -- change to your region
+);
diff --git a/loadFromObjectStoreDemo/create_table_sr_file_external.sql b/loadFromObjectStoreDemo/create_table_sr_file_external.sql
@@ -0,0 +1,117 @@
+-- these SQL statements creates external file tables for ssb_50mb_parquet dataset.
+-- This script uses instance profile authentication method. View this documentation for more supported authentication
+-- methods: https://docs.starrocks.io/en-us/latest/integrations/authenticate_to_aws_resources
+
+-- Please edit the S3 path, table name and s3 endpoint. Replace <your-bucket> and <your-region>
+-- See file external table here: https://docs.starrocks.io/en-us/latest/data_source/file_external_table
+
+CREATE DATABASE file_external_tables;
+
+USE file_external_tables;
+
+CREATE EXTERNAL TABLE lineorder (
+    `lo_orderkey` bigint(20) NOT NULL COMMENT "",
+    `lo_linenumber` int(11) NOT NULL COMMENT "",
+    `lo_custkey` int(11) NOT NULL COMMENT "",
+    `lo_partkey` int(11) NOT NULL COMMENT "",
+    `lo_suppkey` int(11) NOT NULL COMMENT "",
+    `lo_orderdate` int(11) NOT NULL COMMENT "",
+    `lo_orderpriority` varchar(16) NOT NULL COMMENT "",
+    `lo_shippriority` int(11) NOT NULL COMMENT "",
+    `lo_quantity` int(11) NOT NULL COMMENT "",
+    `lo_extendedprice` int(11) NOT NULL COMMENT "",
+    `lo_ordtotalprice` int(11) NOT NULL COMMENT "",
+    `lo_discount` int(11) NOT NULL COMMENT "",
+    `lo_revenue` int(11) NOT NULL COMMENT "",
+    `lo_supplycost` int(11) NOT NULL COMMENT "",
+    `lo_tax` int(11) NOT NULL COMMENT "",
+    `lo_commitdate` int(11) NOT NULL COMMENT "",
+    `lo_shipmode` varchar(11) NOT NULL COMMENT ""
+)
+ENGINE = FILE
+PROPERTIES(
+    "path" = "s3://<your-bucket>/ssb_50mb_parquet/lineorder/",
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com", -- change to your region
+    "format" = "parquet"
+);
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS `customer` (
+ `c_custkey` int(11) NOT NULL COMMENT "",
+ `c_name` varchar(26) NOT NULL COMMENT "",
+ `c_address` varchar(41) NOT NULL COMMENT "",
+ `c_city` varchar(11) NOT NULL COMMENT "",
+ `c_nation` varchar(16) NOT NULL COMMENT "",
+ `c_region` varchar(13) NOT NULL COMMENT "",
+ `c_phone` varchar(16) NOT NULL COMMENT "",
+ `c_mktsegment` varchar(11) NOT NULL COMMENT ""
+) ENGINE = FILE
+PROPERTIES(
+    "path" = "s3://<your-bucket>/ssb_50mb_parquet/customer/",
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com",
+    "format" = "parquet"
+);
+;
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS `dates` (
+ `d_datekey` int(11) NOT NULL COMMENT "",
+ `d_date` varchar(20) NOT NULL COMMENT "",
+ `d_dayofweek` varchar(10) NOT NULL COMMENT "",
+ `d_month` varchar(11) NOT NULL COMMENT "",
+ `d_year` int(11) NOT NULL COMMENT "",
+ `d_yearmonthnum` int(11) NOT NULL COMMENT "",
+ `d_yearmonth` varchar(9) NOT NULL COMMENT "",
+ `d_daynuminweek` int(11) NOT NULL COMMENT "",
+ `d_daynuminmonth` int(11) NOT NULL COMMENT "",
+ `d_daynuminyear` int(11) NOT NULL COMMENT "",
+ `d_monthnuminyear` int(11) NOT NULL COMMENT "",
+ `d_weeknuminyear` int(11) NOT NULL COMMENT "",
+ `d_sellingseason` varchar(14) NOT NULL COMMENT "",
+ `d_lastdayinweekfl` int(11) NOT NULL COMMENT "",
+ `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "",
+ `d_holidayfl` int(11) NOT NULL COMMENT "",
+ `d_weekdayfl` int(11) NOT NULL COMMENT ""
+) ENGINE = FILE
+PROPERTIES(
+    "path" = "s3://<your-bucket>/ssb_50mb_parquet/dates/",
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com",
+    "format" = "parquet"
+);
+
+ CREATE EXTERNAL TABLE IF NOT EXISTS `supplier` (
+ `s_suppkey` int(11) NOT NULL COMMENT "",
+ `s_name` varchar(26) NOT NULL COMMENT "",
+ `s_address` varchar(26) NOT NULL COMMENT "",
+ `s_city` varchar(11) NOT NULL COMMENT "",
+ `s_nation` varchar(16) NOT NULL COMMENT "",
+ `s_region` varchar(13) NOT NULL COMMENT "",
+ `s_phone` varchar(16) NOT NULL COMMENT ""
+) ENGINE = FILE
+PROPERTIES(
+    "path" = "s3://<your-bucket>/ssb_50mb_parquet/supplier/",
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com",
+    "format" = "parquet"
+);
+
+CREATE EXTERNAL TABLE IF NOT EXISTS `part` (
+ `p_partkey` int(11) NOT NULL COMMENT "",
+ `p_name` varchar(23) NOT NULL COMMENT "",
+ `p_mfgr` varchar(7) NOT NULL COMMENT "",
+ `p_category` varchar(8) NOT NULL COMMENT "",
+ `p_brand` varchar(10) NOT NULL COMMENT "",
+ `p_color` varchar(12) NOT NULL COMMENT "",
+ `p_type` varchar(26) NOT NULL COMMENT "",
+ `p_size` int(11) NOT NULL COMMENT "",
+ `p_container` varchar(11) NOT NULL COMMENT ""
+) ENGINE = FILE
+PROPERTIES(
+    "path" = "s3://<your-bucket>/ssb_50mb_parquet/part/",
+    "aws.s3.use_instance_profile" = "True",
+    "aws.s3.endpoint" = "s3.<your-region>.amazonaws.com",
+    "format" = "parquet"
+);