From 578d167bfccdc2d1d5ce9ca06cab7b7b753bb3eb Mon Sep 17 00:00:00 2001 From: Jacky Li Date: Wed, 3 Dec 2014 01:33:44 +0800 Subject: [PATCH 1/2] make caseSensitive configurable --- .../src/main/scala/org/apache/spark/sql/SQLConf.scala | 8 ++++++++ .../src/main/scala/org/apache/spark/sql/SQLContext.scala | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 9697beb132fbb..caf5fa5803948 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -46,6 +46,8 @@ private[spark] object SQLConf { // This is only used for the thriftserver val THRIFTSERVER_POOL = "spark.sql.thriftserver.scheduler.pool" + val CASE_SENSITIVE = "spark.sql.caseSensitive" + object Deprecated { val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks" } @@ -148,6 +150,12 @@ private[sql] trait SQLConf { private[spark] def columnNameOfCorruptRecord: String = getConf(COLUMN_NAME_OF_CORRUPT_RECORD, "_corrupt_record") + /** + * When set to true, analyzer is case sensitive + */ + private[spark] def caseSensitive: Boolean = + getConf(CASE_SENSITIVE, "true").toBoolean + /** ********************** SQLConf functionality methods ************ */ /** Set Spark SQL configuration properties. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 31cc4170aa867..d6c48fb387336 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -65,7 +65,7 @@ class SQLContext(@transient val sparkContext: SparkContext) @transient protected[sql] lazy val analyzer: Analyzer = - new Analyzer(catalog, functionRegistry, caseSensitive = true) + new Analyzer(catalog, functionRegistry, caseSensitive) @transient protected[sql] lazy val optimizer: Optimizer = DefaultOptimizer From f57f15ce72652b3a04229a860a2aba22297368b8 Mon Sep 17 00:00:00 2001 From: Jacky Li Date: Thu, 4 Dec 2014 01:48:04 +0800 Subject: [PATCH 2/2] add testcase --- .../org/apache/spark/sql/SQLContext.scala | 2 +- .../test/TestCaseInsensitiveSQLContext.scala | 35 ++++++++++++ .../sql/SQLQueryCaseInsensitiveSuite.scala | 53 +++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/test/TestCaseInsensitiveSQLContext.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/SQLQueryCaseInsensitiveSuite.scala diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index d6c48fb387336..a2d1f60ac7c0a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -58,7 +58,7 @@ class SQLContext(@transient val sparkContext: SparkContext) self => @transient - protected[sql] lazy val catalog: Catalog = new SimpleCatalog(true) + protected[sql] lazy val catalog: Catalog = new SimpleCatalog(caseSensitive) @transient protected[sql] lazy val functionRegistry: FunctionRegistry = new SimpleFunctionRegistry diff --git a/sql/core/src/main/scala/org/apache/spark/sql/test/TestCaseInsensitiveSQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/test/TestCaseInsensitiveSQLContext.scala new file mode 100644 index 0000000000000..60077ebdd87c5 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/test/TestCaseInsensitiveSQLContext.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.test + +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.sql.{SQLConf, SQLContext} + +/** A case insensitive SQLContext that can be used for local testing. */ +object TestCaseInsensitiveSQLContext + extends SQLContext( + new SparkContext( + "local[2]", + "CaseInsensitiveSQLContext", + new SparkConf().set(SQLConf.CASE_SENSITIVE, "false"))) { + + /** Fewer partitions to speed up testing. */ + override private[spark] def numShufflePartitions: Int = + getConf(SQLConf.SHUFFLE_PARTITIONS, "5").toInt +} + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryCaseInsensitiveSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryCaseInsensitiveSuite.scala new file mode 100644 index 0000000000000..270d36fd13a23 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryCaseInsensitiveSuite.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import java.util.TimeZone + +import org.apache.spark.sql.test.TestCaseInsensitiveSQLContext +import org.scalatest.BeforeAndAfterAll + +/* Implicits */ + +import org.apache.spark.sql.test.TestCaseInsensitiveSQLContext._ + +object CaseInsensitiveTestData{ + case class StringData(s: String) + val table = TestCaseInsensitiveSQLContext.sparkContext.parallelize(StringData("test") :: Nil) + table.registerTempTable("caseInsensitiveTable") +} + +class SQLQueryCaseInsensitiveSuite extends QueryTest with BeforeAndAfterAll { + CaseInsensitiveTestData + + var origZone: TimeZone = _ + + override protected def beforeAll() { + origZone = TimeZone.getDefault + TimeZone.setDefault(TimeZone.getTimeZone("UTC")) + } + + override protected def afterAll() { + TimeZone.setDefault(origZone) + } + + test("SPARK-4699 case sensitivity SQL query") { + checkAnswer(sql("SELECT S FROM CASEINSENSITIVETABLE"), "test") + } + +}