From ea37717f7c709a86985e006a192bf040f8958da3 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 13:50:05 +0900 Subject: [PATCH] [SPARK-32106][SQL][FOLLOWUP] Fix flaky tests in transform.sql ### What changes were proposed in this pull request? This PR intends to fix flaky GitHub Actions (GA) tests below in `transform.sql` (this flakiness does not seem to happen in the Jenkins tests): - https://github.com/apache/spark/runs/1592987501 - https://github.com/apache/spark/runs/1593196242 - https://github.com/apache/spark/runs/1595496305 - https://github.com/apache/spark/runs/1596309555 This is because the error message is different between test runs in GA (the error message seems to be truncated indeterministically) ,e.g., ``` # https://github.com/apache/spark/runs/1592987501 Expected "...h status 127. Error:[ /bin/bash: some_non_existent_command: command not found]", but got "...h status 127. Error:[]" Result did not match for query #2 # https://github.com/apache/spark/runs/1593196242 Expected "...istent_command: comm[and not found]", but got "...istent_command: comm[]" Result did not match for query #2 ``` The root cause of this indeterministic behaviour happening only in GA is not clear though, this test throws SparkException consistently even in GA. So, this PR proposes to make the test just check if it will be thrown when running it. This PR comes from the dongjoon-hyun comment: https://github.com/apache/spark/pull/29414/files#r547414513 ### Why are the changes needed? Bugfix. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Added tests. Closes #30896 from maropu/SPARK-32106-FOLLOWUP. Authored-by: Takeshi Yamamuro Signed-off-by: HyukjinKwon --- .../resources/sql-tests/inputs/transform.sql | 10 -------- .../sql-tests/results/transform.sql.out | 24 +------------------ .../BaseScriptTransformationSuite.scala | 20 ++++++++++++++++ 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/transform.sql b/sql/core/src/test/resources/sql-tests/inputs/transform.sql index 65b060eca3a62..3f39700a95913 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/transform.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/transform.sql @@ -9,16 +9,6 @@ SELECT TRANSFORM(a) USING 'cat' AS (a) FROM t; --- with non-exist command -SELECT TRANSFORM(a) -USING 'some_non_existent_command' AS (a) -FROM t; - --- with non-exist file -SELECT TRANSFORM(a) -USING 'python some_non_existent_file' AS (a) -FROM t; - -- common supported data types between no serde and serde transform SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM ( SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l) diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index 83ab5cb729c24..3267a7625a7d9 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 18 +-- Number of queries: 16 -- !query @@ -26,28 +26,6 @@ struct 3 --- !query -SELECT TRANSFORM(a) -USING 'some_non_existent_command' AS (a) -FROM t --- !query schema -struct<> --- !query output -org.apache.spark.SparkException -Subprocess exited with status 127. Error: /bin/bash: some_non_existent_command: command not found - - --- !query -SELECT TRANSFORM(a) -USING 'python some_non_existent_file' AS (a) -FROM t --- !query schema -struct<> --- !query output -org.apache.spark.SparkException -Subprocess exited with status 2. Error: python: can't open file 'some_non_existent_file': [Errno 2] No such file or directory - - -- !query SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM ( SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala index 81f292809df4a..863657a7862a6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala @@ -420,6 +420,26 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU 'b.cast("string").as("b"), lit(null), lit(null)).collect()) } + + test("SPARK-32106: TRANSFORM with non-existent command/file") { + Seq( + s""" + |SELECT TRANSFORM(a) + |USING 'some_non_existent_command' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin, + s""" + |SELECT TRANSFORM(a) + |USING 'python some_non_existent_file' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin).foreach { query => + intercept[SparkException] { + // Since an error message is shell-dependent, this test just checks + // if the expected exception will be thrown. + sql(query).collect() + } + } + } } case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {