[SPARK-35293][SQL][TESTS] Use the newer dsdgen for TPCDSQueryTestSuite

### What changes were proposed in this pull request? This PR intends to replace `maropu/spark-tpcds-datagen` with `databricks/tpcds-kit` for using a newer dsdgen and update the golden files in `tpcds-query-results`. ### Why are the changes needed? For better testing. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? GA passed. Closes #32420 from maropu/UseTpcdsKit. Authored-by: Takeshi Yamamuro <[email protected]> Signed-off-by: Takeshi Yamamuro <[email protected]>
apache · May 6, 2021 · 5c67d0c · 5c67d0c
1 parent 19661f6
commit 5c67d0c
Show file tree

Hide file tree

Showing 134 changed files with 16,723 additions and 16,569 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -525,14 +525,14 @@ jobs:
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
       uses: actions/checkout@v2
       with:
-        repository: maropu/spark-tpcds-datagen
+        repository: databricks/tpcds-kit
         path: ./tpcds-kit
     - name: Build tpcds-kit
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      run: cd tpcds-kit/thirdparty/tpcds-kit/tools && make OS=LINUX
+      run: cd tpcds-kit/tools && make OS=LINUX
     - name: Generate TPC-DS (SF=1) table data
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/thirdparty/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
+      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
     - name: Run TPC-DS queries
       run: |
         SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSQueryTestSuite"

diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out
@@ -3,103 +3,103 @@
 -- !query schema
 struct<c_customer_id:string>
 -- !query output
-AAAAAAAAAAABBAAA
-AAAAAAAAAAADBAAA
-AAAAAAAAAAADBAAA
-AAAAAAAAAAAKAAAA
-AAAAAAAAAABDAAAA
-AAAAAAAAAABHBAAA
-AAAAAAAAAABLAAAA
-AAAAAAAAAABMAAAA
-AAAAAAAAAACHAAAA
-AAAAAAAAAACMAAAA
-AAAAAAAAAADDAAAA
-AAAAAAAAAADGAAAA
-AAAAAAAAAADGBAAA
-AAAAAAAAAADGBAAA
+AAAAAAAAAAABAAAA
+AAAAAAAAAAAHBAAA
+AAAAAAAAAAAMAAAA
+AAAAAAAAAAAOAAAA
+AAAAAAAAAABCBAAA
+AAAAAAAAAABEAAAA
+AAAAAAAAAABFAAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAACFBAAA
+AAAAAAAAAADBBAAA
+AAAAAAAAAADOAAAA
 AAAAAAAAAADPAAAA
-AAAAAAAAAAEBAAAA
-AAAAAAAAAAEFBAAA
+AAAAAAAAAAEABAAA
+AAAAAAAAAAEEAAAA
 AAAAAAAAAAEGBAAA
-AAAAAAAAAAEIAAAA
-AAAAAAAAAAEMAAAA
-AAAAAAAAAAFAAAAA
+AAAAAAAAAAENAAAA
+AAAAAAAAAAFCBAAA
+AAAAAAAAAAFEBAAA
+AAAAAAAAAAFGAAAA
+AAAAAAAAAAFLAAAA
 AAAAAAAAAAFPAAAA
-AAAAAAAAAAGGBAAA
-AAAAAAAAAAGHBAAA
-AAAAAAAAAAGJAAAA
-AAAAAAAAAAGMAAAA
-AAAAAAAAAAHEBAAA
-AAAAAAAAAAHFBAAA
-AAAAAAAAAAIEBAAA
-AAAAAAAAAAJGBAAA
-AAAAAAAAAAJHBAAA
-AAAAAAAAAAKCAAAA
-AAAAAAAAAAKCAAAA
-AAAAAAAAAAKJAAAA
-AAAAAAAAAAKMAAAA
-AAAAAAAAAAKMAAAA
-AAAAAAAAAALAAAAA
-AAAAAAAAAALABAAA
-AAAAAAAAAALGAAAA
-AAAAAAAAAALHBAAA
+AAAAAAAAAAGCAAAA
+AAAAAAAAAAGEAAAA
+AAAAAAAAAAGIBAAA
+AAAAAAAAAAGOAAAA
+AAAAAAAAAAHABAAA
+AAAAAAAAAAHGBAAA
+AAAAAAAAAAHHAAAA
+AAAAAAAAAAHMAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAHPAAAA
+AAAAAAAAAAJAAAAA
+AAAAAAAAAAJEBAAA
+AAAAAAAAAAJMAAAA
+AAAAAAAAAAJPAAAA
+AAAAAAAAAAKBBAAA
+AAAAAAAAAAKGBAAA
+AAAAAAAAAAKHBAAA
+AAAAAAAAAAKLAAAA
+AAAAAAAAAALCAAAA
 AAAAAAAAAALJAAAA
-AAAAAAAAAANHAAAA
+AAAAAAAAAALJAAAA
+AAAAAAAAAAMABAAA
+AAAAAAAAAAMGAAAA
+AAAAAAAAAAMLAAAA
+AAAAAAAAAAMMAAAA
 AAAAAAAAAANHBAAA
-AAAAAAAAAANJAAAA
-AAAAAAAAAANMAAAA
 AAAAAAAAAANMAAAA
-AAAAAAAAAANNAAAA
 AAAAAAAAAAOBBAAA
-AAAAAAAAAAODBAAA
-AAAAAAAAAAOLAAAA
-AAAAAAAAAAPGBAAA
-AAAAAAAAABAAAAAA
-AAAAAAAAABAEAAAA
-AAAAAAAAABAEBAAA
-AAAAAAAAABAFBAAA
-AAAAAAAAABAIAAAA
-AAAAAAAAABAOAAAA
-AAAAAAAAABBDBAAA
-AAAAAAAAABCFAAAA
-AAAAAAAAABCHBAAA
-AAAAAAAAABDHAAAA
-AAAAAAAAABENAAAA
-AAAAAAAAABFEBAAA
-AAAAAAAAABFGAAAA
-AAAAAAAAABFMAAAA
-AAAAAAAAABFPAAAA
-AAAAAAAAABGFAAAA
-AAAAAAAAABGFBAAA
-AAAAAAAAABGJAAAA
-AAAAAAAAABIBBAAA
-AAAAAAAAABICBAAA
-AAAAAAAAABIIAAAA
-AAAAAAAAABJNAAAA
-AAAAAAAAABKGBAAA
-AAAAAAAAABLOAAAA
-AAAAAAAAABLPAAAA
-AAAAAAAAABMABAAA
+AAAAAAAAAAPDAAAA
+AAAAAAAAAAPKAAAA
+AAAAAAAAAAPLAAAA
+AAAAAAAAABANAAAA
+AAAAAAAAABCCBAAA
+AAAAAAAAABCGAAAA
+AAAAAAAAABDABAAA
+AAAAAAAAABDBAAAA
+AAAAAAAAABDEAAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDEBAAA
+AAAAAAAAABDFBAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABDOAAAA
+AAAAAAAAABEBBAAA
+AAAAAAAAABEDAAAA
+AAAAAAAAABEEAAAA
+AAAAAAAAABEEBAAA
+AAAAAAAAABEIBAAA
+AAAAAAAAABEOAAAA
+AAAAAAAAABFFBAAA
+AAAAAAAAABFHAAAA
+AAAAAAAAABFNAAAA
+AAAAAAAAABFOAAAA
+AAAAAAAAABGAAAAA
+AAAAAAAAABHDBAAA
+AAAAAAAAABHGAAAA
+AAAAAAAAABHGBAAA
+AAAAAAAAABHLAAAA
+AAAAAAAAABIAAAAA
+AAAAAAAAABIBAAAA
+AAAAAAAAABIDBAAA
+AAAAAAAAABIEBAAA
+AAAAAAAAABKLAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABKNAAAA
+AAAAAAAAABLJAAAA
+AAAAAAAAABLNAAAA
+AAAAAAAAABMAAAAA
+AAAAAAAAABMEBAAA
 AAAAAAAAABMPAAAA
-AAAAAAAAABNAAAAA
-AAAAAAAAABNCBAAA
-AAAAAAAAABNEBAAA
-AAAAAAAAABNLAAAA
-AAAAAAAAABNOAAAA
-AAAAAAAAABNPAAAA
-AAAAAAAAABOAAAAA
-AAAAAAAAABOFBAAA
-AAAAAAAAABOOAAAA
-AAAAAAAAABOPAAAA
-AAAAAAAAABPEAAAA
-AAAAAAAAACADAAAA
-AAAAAAAAACAFAAAA
-AAAAAAAAACAFAAAA
-AAAAAAAAACAHBAAA
-AAAAAAAAACAJAAAA
-AAAAAAAAACBDAAAA
-AAAAAAAAACBDAAAA
-AAAAAAAAACBEBAAA
-AAAAAAAAACBNAAAA
-AAAAAAAAACBPAAAA
-AAAAAAAAACCHAAAA
+AAAAAAAAABNABAAA
+AAAAAAAAABNBAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNEAAAA
+AAAAAAAAABNGAAAA
+AAAAAAAAABNNAAAA
+AAAAAAAAABOEAAAA
+AAAAAAAAABOGBAAA
+AAAAAAAAABPABAAA
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out
@@ -3,8 +3,9 @@
 -- !query schema
 struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint,cd_dep_count:int,cnt4:bigint,cd_dep_employed_count:int,cnt5:bigint,cd_dep_college_count:int,cnt6:bigint>
 -- !query output
-F	D	Advanced Degree     	1	3000	1	High Risk 	1	2	1	4	1	5	1
-F	D	Unknown             	1	1500	1	Good      	1	6	1	5	1	4	1
-M	D	College             	1	8500	1	Low Risk  	1	3	1	0	1	1	1
-M	D	Primary             	1	7000	1	Unknown   	1	2	1	1	1	1	1
-M	W	Unknown             	1	4500	1	Good      	1	5	1	0	1	1	1
+F	W	4 yr Degree         	1	4000	1	Low Risk  	1	4	1	6	1	4	1
+M	D	4 yr Degree         	1	1500	1	Low Risk  	1	3	1	4	1	2	1
+M	S	College             	1	4500	1	High Risk 	1	3	1	4	1	3	1
+M	S	Primary             	1	9500	1	Low Risk  	1	3	1	0	1	6	1
+M	S	Secondary           	1	3000	1	High Risk 	1	1	1	1	1	4	1
+M	U	4 yr Degree         	1	2000	1	Low Risk  	1	3	1	1	1	3	1
diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out
@@ -5,6 +5,9 @@ struct<customer_preferred_cust_flag:string>
 -- !query output
 NULL
 NULL
+NULL
+NULL
+N
 N
 N
 N
@@ -48,6 +51,9 @@ N
 N
 N
 N
+N
+Y
+Y
 Y
 Y
 Y