diff --git a/deploy/docker-compose/docker-compose-3BE.yml b/deploy/docker-compose/docker-compose-3BE.yml new file mode 100644 index 0000000..11f6447 --- /dev/null +++ b/deploy/docker-compose/docker-compose-3BE.yml @@ -0,0 +1,71 @@ +version: "3.3" +services: + starrocks-fe: + image: starrocks/fe-ubuntu:2.5.4 + hostname: starrocks-fe + container_name: starrocks-fe + user: root + command: /opt/starrocks/fe/bin/start_fe.sh --host_type FQDN + ports: + - 1030:8030 + - 2020:9020 + - 3030:9030 + volumes: +# - ../../conf/fe.conf:/opt/starrocks/fe/conf/fe.conf + - ./starrocks/fe/meta:/opt/starrocks/fe/meta + - ./starrocks/fe/log:/opt/starrocks/fe/log + + starrocks-be1: + image: starrocks/be-ubuntu:2.5.4 + user: root + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be1:9050\";" + /opt/starrocks/be/bin/start_be.sh + + hostname: starrocks-be1 + container_name: starrocks-be1 + depends_on: + - "starrocks-fe" + volumes: +# - - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf + - ./starrocks/starrocks-be1/storage:/opt/starrocks/be/storage + - ./starrocks/starrocks-be1/log:/opt/starrocks/be/log + starrocks-be2: + image: starrocks/be-ubuntu:2.5.4 + user: root + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be2:9050\";" + /opt/starrocks/be/bin/start_be.sh + + hostname: starrocks-be2 + container_name: starrocks-be2 + depends_on: + - "starrocks-fe" + volumes: +# - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf + - ./starrocks/starrocks-be2/storage:/opt/starrocks/be/storage + - ./starrocks/starrocks-be2/log:/opt/starrocks/be/log + starrocks-be3: + image: starrocks/be-ubuntu:2.5.4 + user: root + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be3:9050\";" + /opt/starrocks/be/bin/start_be.sh + + hostname: starrocks-be3 + container_name: starrocks-be3 + depends_on: + - "starrocks-fe" + volumes: +# - ../../conf/be.conf:/opt/starrocks/be/conf/be.conf + - ./starrocks/starrocks-be3/storage:/opt/starrocks/be/storage + - ./starrocks/starrocks-be3/log:/opt/starrocks/be/log \ No newline at end of file diff --git a/deploy/docker-compose/docker-compose-shared-data.yml b/deploy/docker-compose/docker-compose-shared-data.yml new file mode 100644 index 0000000..603209f --- /dev/null +++ b/deploy/docker-compose/docker-compose-shared-data.yml @@ -0,0 +1,99 @@ +version: "3" +services: + minio: + container_name: starrocks-minio + image: minio/minio:latest + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - starrocks-miniodata:/minio_data + entrypoint: sh + command: '-c ''mkdir -p /minio_data/starrocks && minio server /minio_data --console-address ":9001"''' + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + network: + ipv4_address: 10.5.0.6 + + starrocks-fe: + image: starrocks/fe-ubuntu:3.0.0-rc01 + hostname: starrocks-fe + container_name: starrocks-fe + user: root + command: > + bash -c "echo run_mode=shared_data >> /opt/starrocks/fe/conf/fe.conf && + echo cloud_native_meta_port=6090 >> /opt/starrocks/fe/conf/fe.conf && + echo aws_s3_path=starrocks >> /opt/starrocks/fe/conf/fe.conf && + echo aws_s3_endpoint=minio:9000 >> /opt/starrocks/fe/conf/fe.conf && + echo aws_s3_use_instance_profile=false >> /opt/starrocks/fe/conf/fe.conf && + echo aws_s3_access_key=minioadmin >> /opt/starrocks/fe/conf/fe.conf && + echo aws_s3_secret_key=minioadmin >> /opt/starrocks/fe/conf/fe.conf && + sh /opt/starrocks/fe/bin/start_fe.sh" + ports: + - 8030:8030 + - 9020:9020 + - 9030:9030 + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + depends_on: + - minio + volumes: + # - ./starrocks/starrocks-fe/conf:/opt/starrocks/fe/conf + - starrocks-fe-meta:/opt/starrocks/fe/meta + - starrocks-fe-log:/opt/starrocks/fe/log + networks: + network: + ipv4_address: 10.5.0.2 + + starrocks-be: + image: starrocks/be-ubuntu:3.0.0-rc01 + command: + - /bin/bash + - -c + - | + sleep 15s; echo starlet_port = 8167 >> /opt/starrocks/be/conf/be.conf; + mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";" + /opt/starrocks/be/bin/start_be.sh + ports: + - 8040:8040 + hostname: starrocks-be + container_name: starrocks-be + user: root + depends_on: + - starrocks-fe + - minio + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + volumes: +# - be.conf:/opt/starrocks/be/conf/be.conf +# - starrocks-be-storage:/opt/starrocks/be/storage + - starrocks-be-log:/opt/starrocks/be/log + networks: + network: + ipv4_address: 10.5.0.3 +volumes: +# starrocks-be-storage: + starrocks-be-log: + starrocks-fe-meta: + starrocks-fe-log: + starrocks-miniodata: +networks: + network: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 diff --git a/deploy/docker-compose/docker-compose.yml b/deploy/docker-compose/docker-compose.yml new file mode 100644 index 0000000..638a114 --- /dev/null +++ b/deploy/docker-compose/docker-compose.yml @@ -0,0 +1,60 @@ +version: "3" +services: + starrocks-fe: + image: starrocks/fe-ubuntu:2.5.4 + hostname: starrocks-fe + container_name: starrocks-fe + user: root + command: + /opt/starrocks/fe/bin/start_fe.sh + ports: + - 8030:8030 + - 9020:9020 + - 9030:9030 + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + volumes: +# - fe.conf:/opt/starrocks/fe/conf/fe.conf + - ./starrocks/starrocks-fe/meta:/opt/starrocks/fe/meta + - ./starrocks/starrocks-fe/log:/opt/starrocks/fe/log + networks: + network: + ipv4_address: 10.5.0.2 + + starrocks-be: + image: starrocks/be-ubuntu:2.5.4 + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";" + /opt/starrocks/be/bin/start_be.sh + ports: + - 8040:8040 + hostname: starrocks-be + container_name: starrocks-be + user: root + depends_on: + - starrocks-fe + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + volumes: +# - be.conf:/opt/starrocks/be/conf/be.conf + - ./starrocks/starrocks-be/storage:/opt/starrocks/be/storage + - ./starrocks/starrocks-be/log:/opt/starrocks/be/log + networks: + network: + ipv4_address: 10.5.0.3 +networks: + network: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 diff --git a/loadFromObjectStoreDemo/broker_load_ssb.sql b/loadFromObjectStoreDemo/broker_load_ssb.sql new file mode 100644 index 0000000..5caa28a --- /dev/null +++ b/loadFromObjectStoreDemo/broker_load_ssb.sql @@ -0,0 +1,69 @@ +-- BrokerLoad scripts to SSB tables from an AWS S3 bucket to StarRocks + +-- This script uses instance profile authentication method. View this documentation for more supported authentication +-- methods: https://docs.starrocks.io/en-us/latest/integrations/authenticate_to_aws_resources + +-- Please edit the S3 path, table name and s3 endpoint. +-- See broker load here: https://docs.starrocks.io/en-us/latest/loading/BrokerLoad + +USE load_broker; + +LOAD LABEL load_broker.lineorder +( + DATA INFILE("s3a:///ssb_50mb_parquet/lineorder/*") + INTO TABLE lineorder + FORMAT AS "parquet" +) +WITH BROKER +( + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com" -- change to your region +); + +LOAD LABEL load_broker.customer +( + DATA INFILE("s3a:///ssb_50mb_parquet/customer/*") + INTO TABLE customer + FORMAT AS "parquet" +) +WITH BROKER +( + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com" -- change to your region +); + +LOAD LABEL load_broker.dates +( + DATA INFILE("s3a:///ssb_50mb_parquet/dates/*") + INTO TABLE dates + FORMAT AS "parquet" +) +WITH BROKER +( + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com" -- change to your region +); + +LOAD LABEL load_broker.part +( + DATA INFILE("s3a:///ssb_50mb_parquet/part/*") + INTO TABLE part + FORMAT AS "parquet" +) +WITH BROKER +( + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com" -- change to your region +); + +LOAD LABEL load_broker.supplier +( + DATA INFILE("s3a:///ssb_50mb_parquet/supplier/*") + INTO TABLE supplier + FORMAT AS "parquet" +) +WITH BROKER +( + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com" -- change to your region +); \ No newline at end of file diff --git a/loadFromObjectStoreDemo/create_table_sr_file_external.sql b/loadFromObjectStoreDemo/create_table_sr_file_external.sql new file mode 100644 index 0000000..924954b --- /dev/null +++ b/loadFromObjectStoreDemo/create_table_sr_file_external.sql @@ -0,0 +1,117 @@ +-- these SQL statements creates external file tables for ssb_50mb_parquet dataset. +-- This script uses instance profile authentication method. View this documentation for more supported authentication +-- methods: https://docs.starrocks.io/en-us/latest/integrations/authenticate_to_aws_resources + +-- Please edit the S3 path, table name and s3 endpoint. Replace and +-- See file external table here: https://docs.starrocks.io/en-us/latest/data_source/file_external_table + +CREATE DATABASE file_external_tables; + +USE file_external_tables; + +CREATE EXTERNAL TABLE lineorder ( + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` int(11) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` int(11) NOT NULL COMMENT "", + `lo_extendedprice` int(11) NOT NULL COMMENT "", + `lo_ordtotalprice` int(11) NOT NULL COMMENT "", + `lo_discount` int(11) NOT NULL COMMENT "", + `lo_revenue` int(11) NOT NULL COMMENT "", + `lo_supplycost` int(11) NOT NULL COMMENT "", + `lo_tax` int(11) NOT NULL COMMENT "", + `lo_commitdate` int(11) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +ENGINE = FILE +PROPERTIES( + "path" = "s3:///ssb_50mb_parquet/lineorder/", + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com", -- change to your region + "format" = "parquet" +); + + +CREATE EXTERNAL TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) ENGINE = FILE +PROPERTIES( + "path" = "s3:///ssb_50mb_parquet/customer/", + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com", + "format" = "parquet" +); +; + + +CREATE EXTERNAL TABLE IF NOT EXISTS `dates` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) ENGINE = FILE +PROPERTIES( + "path" = "s3:///ssb_50mb_parquet/dates/", + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com", + "format" = "parquet" +); + + CREATE EXTERNAL TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) ENGINE = FILE +PROPERTIES( + "path" = "s3:///ssb_50mb_parquet/supplier/", + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com", + "format" = "parquet" +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) ENGINE = FILE +PROPERTIES( + "path" = "s3:///ssb_50mb_parquet/part/", + "aws.s3.use_instance_profile" = "True", + "aws.s3.endpoint" = "s3..amazonaws.com", + "format" = "parquet" +); \ No newline at end of file diff --git a/loadFromObjectStoreDemo/create_table_ssb_hive.sql b/loadFromObjectStoreDemo/create_table_ssb_hive.sql new file mode 100644 index 0000000..3d07c61 --- /dev/null +++ b/loadFromObjectStoreDemo/create_table_ssb_hive.sql @@ -0,0 +1,87 @@ +-- Please edit the S3 path, +-- Replace + +CREATE DATABASE IF NOT EXISTS ssb_50mb_parquet; + +USE ssb_50mb_parquet; + +CREATE EXTERNAL TABLE `customer`( + `c_custkey` int, + `c_name` varchar(26), + `c_address` varchar(41), + `c_city` varchar(11), + `c_nation` varchar(16), + `c_region` varchar(13), + `c_phone` varchar(16), + `c_mktsegment` varchar(11)) +stored as parquet +location 's3:///ssb_50mb_parquet/customer/'; + + + + CREATE EXTERNAL TABLE `dates`( + `d_datekey` int, + `d_date` varchar(20), + `d_dayofweek` varchar(10), + `d_month` varchar(11), + `d_year` int, + `d_yearmonthnum` int, + `d_yearmonth` varchar(9), + `d_daynuminweek` int, + `d_daynuminmonth` int, + `d_daynuminyear` int, + `d_monthnuminyear` int, + `d_weeknuminyear` int, + `d_sellingseason` varchar(14), + `d_lastdayinweekfl` int, + `d_lastdayinmonthfl` int, + `d_holidayfl` int, + `d_weekdayfl` int) +stored as parquet +location 's3:///ssb_50mb_parquet/dates/'; + + CREATE EXTERNAL TABLE `lineorder`( + `lo_orderkey` bigint, + `lo_linenumber` int, + `lo_custkey` int, + `lo_partkey` int, + `lo_suppkey` int, + `lo_orderdate` int, + `lo_orderpriority` varchar(16), + `lo_shippriority` int, + `lo_quantity` int, + `lo_extendedprice` int, + `lo_ordtotalprice` int, + `lo_discount` int, + `lo_revenue` int, + `lo_supplycost` int, + `lo_tax` int, + `lo_commitdate` int, + `lo_shipmode` varchar(11)) +stored as parquet +location 's3:///ssb_50mb_parquet/lineorder/'; + + + CREATE EXTERNAL TABLE `part`( + `p_partkey` int, + `p_name` varchar(23), + `p_mfgr` varchar(7), + `p_category` varchar(8), + `p_brand` varchar(10), + `p_color` varchar(12), + `p_type` varchar(26), + `p_size` int, + `p_container` varchar(11)) +stored as parquet +location 's3:///ssb_50mb_parquet/part/'; + + CREATE EXTERNAL TABLE `supplier`( + `s_suppkey` int, + `s_name` varchar(26), + `s_address` varchar(26), + `s_city` varchar(11), + `s_nation` varchar(16), + `s_region` varchar(13), + `s_phone` varchar(16)) +stored as parquet +location 's3:///ssb_50mb_parquet/supplier/'; \ No newline at end of file diff --git a/loadFromObjectStoreDemo/create_table_ssb_sr.sql b/loadFromObjectStoreDemo/create_table_ssb_sr.sql new file mode 100644 index 0000000..4513f6f --- /dev/null +++ b/loadFromObjectStoreDemo/create_table_ssb_sr.sql @@ -0,0 +1,95 @@ +-- SSB 50MB CREATE TABLE for StarRocks internal table (OLAP table) + + +CREATE TABLE `lineorder` ( + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` int(11) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` int(11) NOT NULL COMMENT "", + `lo_extendedprice` int(11) NOT NULL COMMENT "", + `lo_ordtotalprice` int(11) NOT NULL COMMENT "", + `lo_discount` int(11) NOT NULL COMMENT "", + `lo_revenue` int(11) NOT NULL COMMENT "", + `lo_supplycost` int(11) NOT NULL COMMENT "", + `lo_tax` int(11) NOT NULL COMMENT "", + `lo_commitdate` int(11) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`lo_orderkey`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 3; + +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`c_custkey`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 3 +; + + +CREATE TABLE IF NOT EXISTS `dates` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`d_datekey`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +; + + CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`s_suppkey`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 3 +; + +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`p_partkey`) +COMMENT "OLAP" +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 1 +; \ No newline at end of file diff --git a/loadJsonFromKafka/gen.py b/loadJsonFromKafka/gen.py new file mode 100644 index 0000000..a5f8440 --- /dev/null +++ b/loadJsonFromKafka/gen.py @@ -0,0 +1,60 @@ +#!/bin/python +# Copyright (c) 2021 Beijing Dingshi Zongheng Technology Co., Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import random +import time +from kafka import KafkaProducer + + +def genUid(s=10000): + return random.randint(1, s) + + +def getSite(): + site_scope = ['https://www.starrocks.io/'] * 100 + ['https://www.starrocks.io/blog'] * 34 + \ + ['https://www.starrocks.io/product/community'] * 12 + ['https://docs.starrocks.io/'] * 55 + idx = random.randint(0, len(site_scope) - 1) + return site_scope[idx] + + +def getTm(): + delay_jitter = random.randint(-1800, 0) + chance = random.randint(0, 3) + return long(time.time() + delay_jitter * chance) + + +""" +{uid:1, site: https://www.starrocks.com/, vtime: 1621410635} +""" + + +def gen(): + data = """{ "uid": %d, "site": "%s", "vtime": %s } """ % (genUid(), getSite(), getTm()) + return data + + +def main(): + lines = random.randint(1, long(sys.argv[1])) + producer = KafkaProducer(bootstrap_servers='127.0.0.1:9092') + + for x in range(lines): + data = gen() + print(data) + producer.send('quickstart', data) + time.sleep(1) + + +if __name__ == '__main__': + main() diff --git a/loadJsonFromKafka/requirements.txt b/loadJsonFromKafka/requirements.txt new file mode 100644 index 0000000..34aa892 --- /dev/null +++ b/loadJsonFromKafka/requirements.txt @@ -0,0 +1 @@ +kafka-python \ No newline at end of file diff --git a/loadJsonFromKafka/stack-docker-compose.yml b/loadJsonFromKafka/stack-docker-compose.yml new file mode 100644 index 0000000..1b75ae4 --- /dev/null +++ b/loadJsonFromKafka/stack-docker-compose.yml @@ -0,0 +1,89 @@ +version: "3" +services: + starrocks-fe: + image: starrocks/fe-ubuntu:2.5.4 + hostname: starrocks-fe + container_name: starrocks-fe + user: root + command: + /opt/starrocks/fe/bin/start_fe.sh + ports: + - 8030:8030 + - 9020:9020 + - 9030:9030 + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show frontends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + volumes: +# - fe.conf:/opt/starrocks/fe/conf/fe.conf + - ./starrocks/starrocks-fe/meta:/opt/starrocks/fe/meta + - ./starrocks/fe/log:/opt/starrocks/fe/log + networks: + network: + ipv4_address: 10.5.0.2 + + starrocks-be: + image: starrocks/be-ubuntu:2.5.4 + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";" + /opt/starrocks/be/bin/start_be.sh + ports: + - 8040:8040 + hostname: starrocks-be + container_name: starrocks-be + user: root + depends_on: + - starrocks-fe + healthcheck: + test: 'mysql -uroot -h10.5.0.2 -P 9030 -e "show backends\G" |grep "Alive: true"' + interval: 10s + timeout: 5s + retries: 3 + volumes: +# - be.conf:/opt/starrocks/be/conf/be.conf + - ./starrocks/starrocks-be/storage:/opt/starrocks/be/storage + - ./starrocks/starrocks-be/log:/opt/starrocks/be/log + networks: + network: + ipv4_address: 10.5.0.3 + + zookeeper: + image: confluentinc/cp-zookeeper:7.3.2 + container_name: zookeeper + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + networks: + network: + ipv4_address: 10.5.0.4 + + + kafka: + image: confluentinc/cp-kafka:7.3.2 + container_name: broker + ports: + - 9092:9092 + depends_on: + - zookeeper + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: INTERNAL://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:9094,OUTSIDE://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + networks: + network: + ipv4_address: 10.5.0.5 + +networks: + network: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1