Fix

maropu · Apr 30, 2021 · be9f62f · be9f62f
1 parent e8bf8fe
commit be9f62f
Show file tree

Hide file tree

Showing 136 changed files with 17,734 additions and 17,112 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -493,19 +493,6 @@ jobs:
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
-    - name: Cache TPC-DS generated data
-      id: cache-tpcds-sf-1
-      uses: actions/cache@v2
-      with:
-        path: ./tpcds-sf-1
-        key: tpcds-556111e35d400f56cb0625dc16e9063d54628320
-    - name: Checkout TPC-DS (SF=1) generated data repository
-      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      uses: actions/checkout@v2
-      with:
-        repository: maropu/spark-tpcds-sf-1
-        ref: 556111e35d400f56cb0625dc16e9063d54628320
-        path: ./tpcds-sf-1
     - name: Cache Scala, SBT and Maven
       uses: actions/cache@v2
       with:
@@ -528,6 +515,24 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 8
+    - name: Cache TPC-DS generated data
+      id: cache-tpcds-sf-1
+      uses: actions/cache@v2
+      with:
+        path: ./tpcds-sf-1
+        key: tpcds-${{ hashFiles('sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
+    - name: Checkout tpcds-kit repository
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: databricks/tpcds-kit
+        path: ./tpcds-kit
+    - name: Build tpcds-kit
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: cd tpcds-kit/tools && make OS=LINUX
+    - name: Generate TPC-DS (SF=1) table data
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
     - name: Run TPC-DS queries
       run: |
         SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSQueryTestSuite"

diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out
@@ -3,103 +3,103 @@
 -- !query schema
 struct<c_customer_id:string>
 -- !query output
-AAAAAAAAAAABBAAA
-AAAAAAAAAAADBAAA
-AAAAAAAAAAADBAAA
-AAAAAAAAAAAKAAAA
-AAAAAAAAAABDAAAA
-AAAAAAAAAABHBAAA
-AAAAAAAAAABLAAAA
-AAAAAAAAAABMAAAA
-AAAAAAAAAACHAAAA
-AAAAAAAAAACMAAAA
-AAAAAAAAAADDAAAA
-AAAAAAAAAADGAAAA
-AAAAAAAAAADGBAAA
-AAAAAAAAAADGBAAA
+AAAAAAAAAAABAAAA
+AAAAAAAAAAAHBAAA
+AAAAAAAAAAAMAAAA
+AAAAAAAAAAAOAAAA
+AAAAAAAAAABCBAAA
+AAAAAAAAAABEAAAA
+AAAAAAAAAABFAAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAADBBAAA
+AAAAAAAAAADOAAAA
 AAAAAAAAAADPAAAA
-AAAAAAAAAAEBAAAA
-AAAAAAAAAAEFBAAA
+AAAAAAAAAAEABAAA
+AAAAAAAAAAEEAAAA
 AAAAAAAAAAEGBAAA
-AAAAAAAAAAEIAAAA
-AAAAAAAAAAEMAAAA
-AAAAAAAAAAFAAAAA
+AAAAAAAAAAENAAAA
+AAAAAAAAAAFCBAAA
+AAAAAAAAAAFEBAAA
+AAAAAAAAAAFGAAAA
+AAAAAAAAAAFLAAAA
 AAAAAAAAAAFPAAAA
-AAAAAAAAAAGGBAAA
-AAAAAAAAAAGHBAAA
-AAAAAAAAAAGJAAAA
-AAAAAAAAAAGMAAAA
-AAAAAAAAAAHEBAAA
-AAAAAAAAAAHFBAAA
-AAAAAAAAAAIEBAAA
-AAAAAAAAAAJGBAAA
-AAAAAAAAAAJHBAAA
-AAAAAAAAAAKCAAAA
-AAAAAAAAAAKCAAAA
-AAAAAAAAAAKJAAAA
-AAAAAAAAAAKMAAAA
-AAAAAAAAAAKMAAAA
-AAAAAAAAAALAAAAA
-AAAAAAAAAALABAAA
-AAAAAAAAAALGAAAA
-AAAAAAAAAALHBAAA
+AAAAAAAAAAGCAAAA
+AAAAAAAAAAGEAAAA
+AAAAAAAAAAGIBAAA
+AAAAAAAAAAGOAAAA
+AAAAAAAAAAHABAAA
+AAAAAAAAAAHGBAAA
+AAAAAAAAAAHHAAAA
+AAAAAAAAAAHMAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAJAAAAA
+AAAAAAAAAAJEBAAA
+AAAAAAAAAAJMAAAA
+AAAAAAAAAAJPAAAA
+AAAAAAAAAAKBBAAA
+AAAAAAAAAAKGBAAA
+AAAAAAAAAAKHBAAA
+AAAAAAAAAAKLAAAA
+AAAAAAAAAALCAAAA
 AAAAAAAAAALJAAAA
-AAAAAAAAAANHAAAA
+AAAAAAAAAALJAAAA
+AAAAAAAAAAMABAAA
+AAAAAAAAAAMGAAAA
+AAAAAAAAAAMLAAAA
+AAAAAAAAAAMMAAAA
 AAAAAAAAAANHBAAA
-AAAAAAAAAANJAAAA
-AAAAAAAAAANMAAAA
 AAAAAAAAAANMAAAA
-AAAAAAAAAANNAAAA
 AAAAAAAAAAOBBAAA
-AAAAAAAAAAODBAAA
-AAAAAAAAAAOLAAAA
-AAAAAAAAAAPGBAAA
-AAAAAAAAABAAAAAA
-AAAAAAAAABAEAAAA
-AAAAAAAAABAEBAAA
-AAAAAAAAABAFBAAA
-AAAAAAAAABAIAAAA
-AAAAAAAAABAOAAAA
-AAAAAAAAABBDBAAA
-AAAAAAAAABCFAAAA
-AAAAAAAAABCHBAAA
-AAAAAAAAABDHAAAA
-AAAAAAAAABENAAAA
-AAAAAAAAABFEBAAA
-AAAAAAAAABFGAAAA
-AAAAAAAAABFMAAAA
-AAAAAAAAABFPAAAA
-AAAAAAAAABGFAAAA
-AAAAAAAAABGFBAAA
-AAAAAAAAABGJAAAA
-AAAAAAAAABIBBAAA
-AAAAAAAAABICBAAA
-AAAAAAAAABIIAAAA
-AAAAAAAAABJNAAAA
-AAAAAAAAABKGBAAA
-AAAAAAAAABLOAAAA
-AAAAAAAAABLPAAAA
-AAAAAAAAABMABAAA
+AAAAAAAAAAPDAAAA
+AAAAAAAAAAPKAAAA
+AAAAAAAAAAPLAAAA
+AAAAAAAAABANAAAA
+AAAAAAAAABCCBAAA
+AAAAAAAAABCGAAAA
+AAAAAAAAABDABAAA
+AAAAAAAAABDBAAAA
+AAAAAAAAABDEAAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDFBAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABEBBAAA
+AAAAAAAAABEDAAAA
+AAAAAAAAABEEAAAA
+AAAAAAAAABEEBAAA
+AAAAAAAAABEIBAAA
+AAAAAAAAABEOAAAA
+AAAAAAAAABFFBAAA
+AAAAAAAAABFHAAAA
+AAAAAAAAABFNAAAA
+AAAAAAAAABFOAAAA
+AAAAAAAAABGAAAAA
+AAAAAAAAABHDBAAA
+AAAAAAAAABHGAAAA
+AAAAAAAAABHGBAAA
+AAAAAAAAABHLAAAA
+AAAAAAAAABIAAAAA
+AAAAAAAAABIBAAAA
+AAAAAAAAABIDBAAA
+AAAAAAAAABIEBAAA
+AAAAAAAAABKLAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABLJAAAA
+AAAAAAAAABLNAAAA
+AAAAAAAAABMAAAAA
+AAAAAAAAABMEBAAA
 AAAAAAAAABMPAAAA
-AAAAAAAAABNAAAAA
-AAAAAAAAABNCBAAA
-AAAAAAAAABNEBAAA
-AAAAAAAAABNLAAAA
-AAAAAAAAABNOAAAA
-AAAAAAAAABNPAAAA
-AAAAAAAAABOAAAAA
-AAAAAAAAABOFBAAA
-AAAAAAAAABOOAAAA
-AAAAAAAAABOPAAAA
-AAAAAAAAABPEAAAA
-AAAAAAAAACADAAAA
-AAAAAAAAACAFAAAA
-AAAAAAAAACAFAAAA
-AAAAAAAAACAHBAAA
-AAAAAAAAACAJAAAA
-AAAAAAAAACBDAAAA
-AAAAAAAAACBDAAAA
-AAAAAAAAACBEBAAA
-AAAAAAAAACBNAAAA
-AAAAAAAAACBPAAAA
-AAAAAAAAACCHAAAA
+AAAAAAAAABNABAAA
+AAAAAAAAABNBAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNGAAAA
+AAAAAAAAABNNAAAA
+AAAAAAAAABOEAAAA
+AAAAAAAAABOGBAAA
+AAAAAAAAABPABAAA
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out
@@ -3,8 +3,9 @@
 -- !query schema
 struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint,cd_dep_count:int,cnt4:bigint,cd_dep_employed_count:int,cnt5:bigint,cd_dep_college_count:int,cnt6:bigint>
 -- !query output
-F	D	Advanced Degree     	1	3000	1	High Risk 	1	2	1	4	1	5	1
-F	D	Unknown             	1	1500	1	Good      	1	6	1	5	1	4	1
-M	D	College             	1	8500	1	Low Risk  	1	3	1	0	1	1	1
-M	D	Primary             	1	7000	1	Unknown   	1	2	1	1	1	1	1
-M	W	Unknown             	1	4500	1	Good      	1	5	1	0	1	1	1
+F	W	4 yr Degree         	1	4000	1	Low Risk  	1	4	1	6	1	4	1
+M	D	4 yr Degree         	1	1500	1	Low Risk  	1	3	1	4	1	2	1
+M	S	College             	1	4500	1	High Risk 	1	3	1	4	1	3	1
+M	S	Primary             	1	9500	1	Low Risk  	1	3	1	0	1	6	1
+M	S	Secondary           	1	3000	1	High Risk 	1	1	1	1	1	4	1
+M	U	4 yr Degree         	1	2000	1	Low Risk  	1	3	1	1	1	3	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out
@@ -5,6 +5,9 @@ struct<customer_preferred_cust_flag:string>
 -- !query output
 NULL
 NULL
+NULL
+NULL
+N
 N
 N
 N
@@ -48,6 +51,9 @@ N
 N
 N
 N
+N
+Y
+Y
 Y
 Y
 Y