Skip to content

Commit

Permalink
[SPARK-33985][SQL] Add query test of combine usage of TRANSFORM and C…
Browse files Browse the repository at this point in the history
…LUSTER BY/ORDER BY
  • Loading branch information
AngersZhuuuu committed Apr 25, 2021
1 parent bcac733 commit 6e652a1
Show file tree
Hide file tree
Showing 2 changed files with 161 additions and 1 deletion.
50 changes: 50 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/transform.sql
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@ CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES
(7, 8, 9)
AS script_trans(a, b, c);

CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
(1, 2, 4),
(1, 1, 7),
(2, 1, 9),
(2, 3, 2),
(3, 1, 1),
(2, 2, 5),
(3, 2, 8),
(1, 3, 5),
(3, 3, 10)
as complex_trans(a, b, c);

SELECT TRANSFORM(a)
USING 'cat' AS (a)
FROM t;
Expand Down Expand Up @@ -342,3 +354,41 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
FROM script_trans
WHERE a <= 2
GROUP BY b;

-- SPARK-33985 TRANSFORM with CLUSTER BY/ORDER BY/SORT BY
FROM (
FROM complex_trans
MAP a, b, c
USING 'cat' AS (a, b, c)
CLUSTER BY a
) map_output
REDUCE a, b, c
USING 'cat' AS (a, b, b);

FROM (
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, c)
FROM complex_trans
CLUSTER BY a
) map_output
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, b);

FROM (
FROM complex_trans
MAP a, b, c
USING 'cat' AS (a, b, c)
ORDER BY b
) map_output
REDUCE a, b, c
USING 'cat' AS (a, b, b);


FROM (
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, c)
FROM complex_trans
ORDER BY b
) map_output
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, b);
112 changes: 111 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/transform.sql.out
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 44
-- Number of queries: 49


-- !query
Expand All @@ -26,6 +26,24 @@ struct<>



-- !query
CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
(1, 2, 4),
(1, 1, 7),
(2, 1, 9),
(2, 3, 2),
(3, 1, 1),
(2, 2, 5),
(3, 2, 8),
(1, 3, 5),
(3, 3, 10)
as complex_trans(a, b, c)
-- !query schema
struct<>
-- !query output



-- !query
SELECT TRANSFORM(a)
USING 'cat' AS (a)
Expand Down Expand Up @@ -717,3 +735,95 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
FROM script_trans
WHERE a <= 2
GROUP BY b


-- !query
FROM (
FROM complex_trans
MAP a, b, c
USING 'cat' AS (a, b, c)
CLUSTER BY a
) map_output
REDUCE a, b, c
USING 'cat' AS (a, b, b)
-- !query schema
struct<a:string,b:string,b:string>
-- !query output
1 1 7
1 2 4
1 3 5
2 1 9
2 2 5
2 3 2
3 1 1
3 2 8
3 3 10


-- !query
FROM (
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, c)
FROM complex_trans
CLUSTER BY a
) map_output
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, b)
-- !query schema
struct<a:string,b:string,b:string>
-- !query output
1 1 7
1 2 4
1 3 5
2 1 9
2 2 5
2 3 2
3 1 1
3 2 8
3 3 10


-- !query
FROM (
FROM complex_trans
MAP a, b, c
USING 'cat' AS (a, b, c)
ORDER BY b
) map_output
REDUCE a, b, c
USING 'cat' AS (a, b, b)
-- !query schema
struct<a:string,b:string,b:string>
-- !query output
1 1 7
2 1 9
3 1 1
1 2 4
2 2 5
3 2 8
2 3 2
1 3 5
3 3 10


-- !query
FROM (
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, c)
FROM complex_trans
ORDER BY b
) map_output
SELECT TRANSFORM(a, b, c)
USING 'cat' AS (a, b, b)
-- !query schema
struct<a:string,b:string,b:string>
-- !query output
1 1 7
2 1 9
3 1 1
1 2 4
2 2 5
3 2 8
2 3 2
1 3 5
3 3 10

0 comments on commit 6e652a1

Please sign in to comment.