From e09b8318b3674b6231e26ab3f05d876d64363814 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 23 Jan 2021 12:53:50 +0800 Subject: [PATCH 1/3] Benchmark TPC-DS with 1GB scale factor --- .github/workflows/build_and_test.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 2011104a19b8a..c209c9c57b0f5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -430,3 +430,29 @@ jobs: - name: Build with SBT run: | ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Phadoop-2.7 compile test:compile + + tpcds1g: + name: Benchmark TPC-DS with 1GB scale factor + runs-on: ubuntu-20.04 + continue-on-error: true + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + - name: Checkout TPC-DS 1G repository + uses: actions/checkout@v2 + with: + repository: wangyum/tpcds1g + path: ./tpcds1g + - name: Install Java 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Run TPCDSQueryBenchmark + run: | + SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark --data-location `pwd`/tpcds1g --query-filter --cbo" + - name: Upload benchmark result to report + if: always() + uses: actions/upload-artifact@v2 + with: + name: TPC-DS 1GB benchmark result + path: "sql/core/benchmarks/TPCDSQueryBenchmark-results.txt" From e8faad34eca393cc79a5995f950ff1c7120a2e72 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 23 Jan 2021 12:56:32 +0800 Subject: [PATCH 2/3] Benchmark TPC-DS with 1GB scale factor --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c209c9c57b0f5..91ec1c63ec241 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -449,7 +449,7 @@ jobs: java-version: 8 - name: Run TPCDSQueryBenchmark run: | - SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark --data-location `pwd`/tpcds1g --query-filter --cbo" + SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark --data-location `pwd`/tpcds1g --cbo" - name: Upload benchmark result to report if: always() uses: actions/upload-artifact@v2 From 9da5102bc18b0a7c0bad124839bee034390d26e8 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 23 Jan 2021 15:09:17 +0800 Subject: [PATCH 3/3] fix --- .github/workflows/build_and_test.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 91ec1c63ec241..67ab3dddd9136 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -438,15 +438,24 @@ jobs: steps: - name: Checkout Spark repository uses: actions/checkout@v2 - - name: Checkout TPC-DS 1G repository + - name: Checkout tpcds-kit repository uses: actions/checkout@v2 with: - repository: wangyum/tpcds1g - path: ./tpcds1g + repository: databricks/tpcds-kit + path: ./tpcds-kit + - name: Checkout spark-sql-perf repository + uses: actions/checkout@v2 + with: + repository: wangyum/spark-sql-perf + path: ./spark-sql-perf - name: Install Java 8 uses: actions/setup-java@v1 with: java-version: 8 + - name: Build tpcds-kit + run: cd tpcds-kit/tools && make OS=LINUX + - name: Gen TPCDS Data + run: cd spark-sql-perf && build/sbt "test:runMain com.databricks.spark.sql.GenTPCDSData `pwd`/../tpcds-kit/tools 1 `pwd`/../tpcds1g parquet" - name: Run TPCDSQueryBenchmark run: | SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark --data-location `pwd`/tpcds1g --cbo"