Skip to content

Commit

Permalink
[SPARK-49438][SQL] Fix the pretty name of the FromAvro & ToAvro e…
Browse files Browse the repository at this point in the history
…xpression

### What changes were proposed in this pull request?
In the PR, I propose to override the `prettyName` method of the `FromAvro` & `ToAvro` expression and set to `from_avro` & `to_avro` by default as in `FunctionRegistry`:
https://github.com/apache/spark/blob/6d8235f3b2bbaa88b10c35d6eecddffa4d1b04a4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala#L873-L874

### Why are the changes needed?
To don't confuse users by non-existent function name, and print correct name in errors.

### Does this PR introduce _any_ user-facing change?
Yes.

### How was this patch tested?
Update existed UT.

### Was this patch authored or co-authored using generative AI tooling?
No.

Closes #47900 from panbingkun/SPARK-49438.

Authored-by: panbingkun <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
  • Loading branch information
panbingkun authored and MaxGekk committed Aug 28, 2024
1 parent 0781631 commit df3d4fd
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ class AvroFunctionsSuite extends QueryTest with SharedSparkSession {
|select to_avro(s, 42) as result from t
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map("sqlExpr" -> "\"toavro(s, 42)\"",
parameters = Map("sqlExpr" -> "\"to_avro(s, 42)\"",
"msg" -> ("The second argument of the TO_AVRO SQL function must be a constant string " +
"containing the JSON representation of the schema to use for converting the value to " +
"AVRO format"),
Expand All @@ -343,7 +343,7 @@ class AvroFunctionsSuite extends QueryTest with SharedSparkSession {
|select from_avro(s, 42, '') as result from t
|""".stripMargin)),
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map("sqlExpr" -> "\"fromavro(s, 42, )\"",
parameters = Map("sqlExpr" -> "\"from_avro(s, 42, )\"",
"msg" -> ("The second argument of the FROM_AVRO SQL function must be a constant string " +
"containing the JSON representation of the schema to use for converting the value " +
"from AVRO format"),
Expand All @@ -360,7 +360,7 @@ class AvroFunctionsSuite extends QueryTest with SharedSparkSession {
errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
parameters = Map(
"sqlExpr" ->
s"\"fromavro(s, $jsonFormatSchema, 42)\"".stripMargin,
s"\"from_avro(s, $jsonFormatSchema, 42)\"".stripMargin,
"msg" -> ("The third argument of the FROM_AVRO SQL function must be a constant map of " +
"strings to strings containing the options to use for converting the value " +
"from AVRO format"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.spark.sql.catalyst.expressions

import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
import org.apache.spark.sql.errors.QueryCompilationErrors
Expand Down Expand Up @@ -117,6 +118,9 @@ case class FromAvro(child: Expression, jsonFormatSchema: Expression, options: Ex
val expr = constructor.newInstance(child, schemaValue, optionsValue)
expr.asInstanceOf[Expression]
}

override def prettyName: String =
getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("from_avro")
}

/**
Expand Down Expand Up @@ -189,4 +193,7 @@ case class ToAvro(child: Expression, jsonFormatSchema: Expression)
val expr = constructor.newInstance(child, schemaValue)
expr.asInstanceOf[Expression]
}

override def prettyName: String =
getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("to_avro")
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The second argument of the FROM_AVRO SQL function must be a constant string containing the JSON representation of the schema to use for converting the value from AVRO format",
"sqlExpr" : "\"fromavro(s, 42, map())\""
"sqlExpr" : "\"from_avro(s, 42, map())\""
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -58,7 +58,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The third argument of the FROM_AVRO SQL function must be a constant map of strings to strings containing the options to use for converting the value from AVRO format",
"sqlExpr" : "\"fromavro(s, variablereference(system.session.avro_schema='{ \"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }] }'), 42)\""
"sqlExpr" : "\"from_avro(s, variablereference(system.session.avro_schema='{ \"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }] }'), 42)\""
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -80,7 +80,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The second argument of the TO_AVRO SQL function must be a constant string containing the JSON representation of the schema to use for converting the value to AVRO format",
"sqlExpr" : "\"toavro(s, 42)\""
"sqlExpr" : "\"to_avro(s, 42)\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The second argument of the FROM_AVRO SQL function must be a constant string containing the JSON representation of the schema to use for converting the value from AVRO format",
"sqlExpr" : "\"fromavro(s, 42, map())\""
"sqlExpr" : "\"from_avro(s, 42, map())\""
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -62,7 +62,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The third argument of the FROM_AVRO SQL function must be a constant map of strings to strings containing the options to use for converting the value from AVRO format",
"sqlExpr" : "\"fromavro(s, variablereference(system.session.avro_schema='{ \"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }] }'), 42)\""
"sqlExpr" : "\"from_avro(s, variablereference(system.session.avro_schema='{ \"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }] }'), 42)\""
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -86,7 +86,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"messageParameters" : {
"hint" : "",
"msg" : "The second argument of the TO_AVRO SQL function must be a constant string containing the JSON representation of the schema to use for converting the value to AVRO format",
"sqlExpr" : "\"toavro(s, 42)\""
"sqlExpr" : "\"to_avro(s, 42)\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down

0 comments on commit df3d4fd

Please sign in to comment.