Address comments

.github/workflows/build_and_test.yml

@@ -525,14 +525,14 @@ jobs:
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
       uses: actions/checkout@v2
       with:
-        repository: databricks/tpcds-kit
+        repository: maropu/spark-tpcds-datagen
         path: ./tpcds-kit
     - name: Build tpcds-kit
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      run: cd tpcds-kit/tools && make OS=LINUX
+      run: cd tpcds-kit/thirdparty/tpcds-kit/tools && make OS=LINUX
     - name: Generate TPC-DS (SF=1) table data
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
+      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/thirdparty/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
     - name: Run TPC-DS queries
       run: |
         SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSQueryTestSuite"

sql/core/src/test/resources/tpcds-query-results/v1_4/q1.sql.out

@@ -3,103 +3,103 @@
 -- !query schema
 struct<c_customer_id:string>
 -- !query output
-AAAAAAAAAAABAAAA
-AAAAAAAAAAAHBAAA
-AAAAAAAAAAAMAAAA
-AAAAAAAAAAAOAAAA
-AAAAAAAAAABCBAAA
-AAAAAAAAAABEAAAA
-AAAAAAAAAABFAAAA
-AAAAAAAAAACFBAAA
-AAAAAAAAAACFBAAA
-AAAAAAAAAADBBAAA
-AAAAAAAAAADOAAAA
+AAAAAAAAAAABBAAA
+AAAAAAAAAAADBAAA
+AAAAAAAAAAADBAAA
+AAAAAAAAAAAKAAAA
+AAAAAAAAAABDAAAA
+AAAAAAAAAABHBAAA
+AAAAAAAAAABLAAAA
+AAAAAAAAAABMAAAA
+AAAAAAAAAACHAAAA
+AAAAAAAAAACMAAAA
+AAAAAAAAAADDAAAA
+AAAAAAAAAADGAAAA
+AAAAAAAAAADGBAAA
+AAAAAAAAAADGBAAA
 AAAAAAAAAADPAAAA
-AAAAAAAAAAEABAAA
-AAAAAAAAAAEEAAAA
+AAAAAAAAAAEBAAAA
+AAAAAAAAAAEFBAAA
 AAAAAAAAAAEGBAAA
-AAAAAAAAAAENAAAA
-AAAAAAAAAAFCBAAA
-AAAAAAAAAAFEBAAA
-AAAAAAAAAAFGAAAA
-AAAAAAAAAAFLAAAA
+AAAAAAAAAAEIAAAA
+AAAAAAAAAAEMAAAA
+AAAAAAAAAAFAAAAA
 AAAAAAAAAAFPAAAA
-AAAAAAAAAAGCAAAA
-AAAAAAAAAAGEAAAA
-AAAAAAAAAAGIBAAA
-AAAAAAAAAAGOAAAA
-AAAAAAAAAAHABAAA
-AAAAAAAAAAHGBAAA
-AAAAAAAAAAHHAAAA
-AAAAAAAAAAHMAAAA
-AAAAAAAAAAHPAAAA
-AAAAAAAAAAHPAAAA
-AAAAAAAAAAHPAAAA
-AAAAAAAAAAJAAAAA
-AAAAAAAAAAJEBAAA
-AAAAAAAAAAJMAAAA
-AAAAAAAAAAJPAAAA
-AAAAAAAAAAKBBAAA
-AAAAAAAAAAKGBAAA
-AAAAAAAAAAKHBAAA
-AAAAAAAAAAKLAAAA
-AAAAAAAAAALCAAAA
+AAAAAAAAAAGGBAAA
+AAAAAAAAAAGHBAAA
+AAAAAAAAAAGJAAAA
+AAAAAAAAAAGMAAAA
+AAAAAAAAAAHEBAAA
+AAAAAAAAAAHFBAAA
+AAAAAAAAAAIEBAAA
+AAAAAAAAAAJGBAAA
+AAAAAAAAAAJHBAAA
+AAAAAAAAAAKCAAAA
+AAAAAAAAAAKCAAAA
+AAAAAAAAAAKJAAAA
+AAAAAAAAAAKMAAAA
+AAAAAAAAAAKMAAAA
+AAAAAAAAAALAAAAA
+AAAAAAAAAALABAAA
+AAAAAAAAAALGAAAA
+AAAAAAAAAALHBAAA
 AAAAAAAAAALJAAAA
-AAAAAAAAAALJAAAA
-AAAAAAAAAAMABAAA
-AAAAAAAAAAMGAAAA
-AAAAAAAAAAMLAAAA
-AAAAAAAAAAMMAAAA
+AAAAAAAAAANHAAAA
 AAAAAAAAAANHBAAA
+AAAAAAAAAANJAAAA
+AAAAAAAAAANMAAAA
 AAAAAAAAAANMAAAA
+AAAAAAAAAANNAAAA
 AAAAAAAAAAOBBAAA
-AAAAAAAAAAPDAAAA
-AAAAAAAAAAPKAAAA
-AAAAAAAAAAPLAAAA
-AAAAAAAAABANAAAA
-AAAAAAAAABCCBAAA
-AAAAAAAAABCGAAAA
-AAAAAAAAABDABAAA
-AAAAAAAAABDBAAAA
-AAAAAAAAABDEAAAA
-AAAAAAAAABDEBAAA
-AAAAAAAAABDEBAAA
-AAAAAAAAABDFBAAA
-AAAAAAAAABDOAAAA
-AAAAAAAAABDOAAAA
-AAAAAAAAABEBBAAA
-AAAAAAAAABEDAAAA
-AAAAAAAAABEEAAAA
-AAAAAAAAABEEBAAA
-AAAAAAAAABEIBAAA
-AAAAAAAAABEOAAAA
-AAAAAAAAABFFBAAA
-AAAAAAAAABFHAAAA
-AAAAAAAAABFNAAAA
-AAAAAAAAABFOAAAA
-AAAAAAAAABGAAAAA
-AAAAAAAAABHDBAAA
-AAAAAAAAABHGAAAA
-AAAAAAAAABHGBAAA
-AAAAAAAAABHLAAAA
-AAAAAAAAABIAAAAA
-AAAAAAAAABIBAAAA
-AAAAAAAAABIDBAAA
-AAAAAAAAABIEBAAA
-AAAAAAAAABKLAAAA
-AAAAAAAAABKNAAAA
-AAAAAAAAABKNAAAA
-AAAAAAAAABLJAAAA
-AAAAAAAAABLNAAAA
-AAAAAAAAABMAAAAA
-AAAAAAAAABMEBAAA
+AAAAAAAAAAODBAAA
+AAAAAAAAAAOLAAAA
+AAAAAAAAAAPGBAAA
+AAAAAAAAABAAAAAA
+AAAAAAAAABAEAAAA
+AAAAAAAAABAEBAAA
+AAAAAAAAABAFBAAA
+AAAAAAAAABAIAAAA
+AAAAAAAAABAOAAAA
+AAAAAAAAABBDBAAA
+AAAAAAAAABCFAAAA
+AAAAAAAAABCHBAAA
+AAAAAAAAABDHAAAA
+AAAAAAAAABENAAAA
+AAAAAAAAABFEBAAA
+AAAAAAAAABFGAAAA
+AAAAAAAAABFMAAAA
+AAAAAAAAABFPAAAA
+AAAAAAAAABGFAAAA
+AAAAAAAAABGFBAAA
+AAAAAAAAABGJAAAA
+AAAAAAAAABIBBAAA
+AAAAAAAAABICBAAA
+AAAAAAAAABIIAAAA
+AAAAAAAAABJNAAAA
+AAAAAAAAABKGBAAA
+AAAAAAAAABLOAAAA
+AAAAAAAAABLPAAAA
+AAAAAAAAABMABAAA
 AAAAAAAAABMPAAAA
-AAAAAAAAABNABAAA
-AAAAAAAAABNBAAAA
-AAAAAAAAABNEAAAA
-AAAAAAAAABNEAAAA
-AAAAAAAAABNGAAAA
-AAAAAAAAABNNAAAA
-AAAAAAAAABOEAAAA
-AAAAAAAAABOGBAAA
-AAAAAAAAABPABAAA
+AAAAAAAAABNAAAAA
+AAAAAAAAABNCBAAA
+AAAAAAAAABNEBAAA
+AAAAAAAAABNLAAAA
+AAAAAAAAABNOAAAA
+AAAAAAAAABNPAAAA
+AAAAAAAAABOAAAAA
+AAAAAAAAABOFBAAA
+AAAAAAAAABOOAAAA
+AAAAAAAAABOPAAAA
+AAAAAAAAABPEAAAA
+AAAAAAAAACADAAAA
+AAAAAAAAACAFAAAA
+AAAAAAAAACAFAAAA
+AAAAAAAAACAHBAAA
+AAAAAAAAACAJAAAA
+AAAAAAAAACBDAAAA
+AAAAAAAAACBDAAAA
+AAAAAAAAACBEBAAA
+AAAAAAAAACBNAAAA
+AAAAAAAAACBPAAAA
+AAAAAAAAACCHAAAA

sql/core/src/test/resources/tpcds-query-results/v1_4/q10.sql.out

@@ -3,9 +3,8 @@
 -- !query schema
 struct<cd_gender:string,cd_marital_status:string,cd_education_status:string,cnt1:bigint,cd_purchase_estimate:int,cnt2:bigint,cd_credit_rating:string,cnt3:bigint,cd_dep_count:int,cnt4:bigint,cd_dep_employed_count:int,cnt5:bigint,cd_dep_college_count:int,cnt6:bigint>
 -- !query output
-F	W	4 yr Degree         	1	4000	1	Low Risk  	1	4	1	6	1	4	1
-M	D	4 yr Degree         	1	1500	1	Low Risk  	1	3	1	4	1	2	1
-M	S	College             	1	4500	1	High Risk 	1	3	1	4	1	3	1
-M	S	Primary             	1	9500	1	Low Risk  	1	3	1	0	1	6	1
-M	S	Secondary           	1	3000	1	High Risk 	1	1	1	1	1	4	1
-M	U	4 yr Degree         	1	2000	1	Low Risk  	1	3	1	1	1	3	1
+F	D	Advanced Degree     	1	3000	1	High Risk 	1	2	1	4	1	5	1
+F	D	Unknown             	1	1500	1	Good      	1	6	1	5	1	4	1
+M	D	College             	1	8500	1	Low Risk  	1	3	1	0	1	1	1
+M	D	Primary             	1	7000	1	Unknown   	1	2	1	1	1	1	1
+M	W	Unknown             	1	4500	1	Good      	1	5	1	0	1	1	1

sql/core/src/test/resources/tpcds-query-results/v1_4/q11.sql.out

@@ -5,9 +5,6 @@ struct<customer_preferred_cust_flag:string>
 -- !query output
 NULL
 NULL
-NULL
-NULL
-N
 N
 N
 N
@@ -51,9 +48,6 @@ N
 N
 N
 N
-N
-Y
-Y
 Y
 Y
 Y