From 6e652a12d373f2dcf8e515ba93766b8416b1d338 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Sun, 25 Apr 2021 17:25:14 +0800 Subject: [PATCH] [SPARK-33985][SQL] Add query test of combine usage of TRANSFORM and CLUSTER BY/ORDER BY --- .../resources/sql-tests/inputs/transform.sql | 50 ++++++++ .../sql-tests/results/transform.sql.out | 112 +++++++++++++++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/transform.sql b/sql/core/src/test/resources/sql-tests/inputs/transform.sql index 7419ca1bd0a80..a24ee6841f25b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/transform.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/transform.sql @@ -11,6 +11,18 @@ CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES (7, 8, 9) AS script_trans(a, b, c); +CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES +(1, 2, 4), +(1, 1, 7), +(2, 1, 9), +(2, 3, 2), +(3, 1, 1), +(2, 2, 5), +(3, 2, 8), +(1, 3, 5), +(3, 3, 10) +as complex_trans(a, b, c); + SELECT TRANSFORM(a) USING 'cat' AS (a) FROM t; @@ -342,3 +354,41 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING)) FROM script_trans WHERE a <= 2 GROUP BY b; + +-- SPARK-33985 TRANSFORM with CLUSTER BY/ORDER BY/SORT BY +FROM ( + FROM complex_trans + MAP a, b, c + USING 'cat' AS (a, b, c) + CLUSTER BY a +) map_output +REDUCE a, b, c + USING 'cat' AS (a, b, b); + +FROM ( + SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, c) + FROM complex_trans + CLUSTER BY a +) map_output +SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, b); + +FROM ( + FROM complex_trans + MAP a, b, c + USING 'cat' AS (a, b, c) + ORDER BY b +) map_output +REDUCE a, b, c + USING 'cat' AS (a, b, b); + + +FROM ( + SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, c) + FROM complex_trans + ORDER BY b +) map_output +SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, b); \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index 1d7e9cdb430e0..c32ae4efed3b9 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 44 +-- Number of queries: 49 -- !query @@ -26,6 +26,24 @@ struct<> +-- !query +CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES +(1, 2, 4), +(1, 1, 7), +(2, 1, 9), +(2, 3, 2), +(3, 1, 1), +(2, 2, 5), +(3, 2, 8), +(1, 3, 5), +(3, 3, 10) +as complex_trans(a, b, c) +-- !query schema +struct<> +-- !query output + + + -- !query SELECT TRANSFORM(a) USING 'cat' AS (a) @@ -717,3 +735,95 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING)) FROM script_trans WHERE a <= 2 GROUP BY b + + +-- !query +FROM ( + FROM complex_trans + MAP a, b, c + USING 'cat' AS (a, b, c) + CLUSTER BY a +) map_output +REDUCE a, b, c + USING 'cat' AS (a, b, b) +-- !query schema +struct +-- !query output +1 1 7 +1 2 4 +1 3 5 +2 1 9 +2 2 5 +2 3 2 +3 1 1 +3 2 8 +3 3 10 + + +-- !query +FROM ( + SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, c) + FROM complex_trans + CLUSTER BY a +) map_output +SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, b) +-- !query schema +struct +-- !query output +1 1 7 +1 2 4 +1 3 5 +2 1 9 +2 2 5 +2 3 2 +3 1 1 +3 2 8 +3 3 10 + + +-- !query +FROM ( + FROM complex_trans + MAP a, b, c + USING 'cat' AS (a, b, c) + ORDER BY b +) map_output +REDUCE a, b, c + USING 'cat' AS (a, b, b) +-- !query schema +struct +-- !query output +1 1 7 +2 1 9 +3 1 1 +1 2 4 +2 2 5 +3 2 8 +2 3 2 +1 3 5 +3 3 10 + + +-- !query +FROM ( + SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, c) + FROM complex_trans + ORDER BY b +) map_output +SELECT TRANSFORM(a, b, c) + USING 'cat' AS (a, b, b) +-- !query schema +struct +-- !query output +1 1 7 +2 1 9 +3 1 1 +1 2 4 +2 2 5 +3 2 8 +2 3 2 +1 3 5 +3 3 10