Skip to content

Commit

Permalink
SPARK-1757 Add failing test for saving SparkSQL Schemas with Option[?…
Browse files Browse the repository at this point in the history
…] fields as parquet
  • Loading branch information
ash211 committed May 8, 2014
1 parent 19c8fb0 commit 8f3f281
Showing 1 changed file with 44 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,20 @@ import org.apache.spark.sql.test.TestSQLContext._

case class TestRDDEntry(key: Int, value: String)

case class NullReflectData(
intField: java.lang.Integer,
longField: java.lang.Long,
floatField: java.lang.Float,
doubleField: java.lang.Double,
booleanField: java.lang.Boolean)

case class OptionalReflectData(
intField: Option[Int],
longField: Option[Long],
floatField: Option[Float],
doubleField: Option[Double],
booleanField: Option[Boolean])

class ParquetQuerySuite extends QueryTest with FunSuite with BeforeAndAfterAll {
import TestData._
TestData // Load test data tables.
Expand Down Expand Up @@ -195,5 +209,35 @@ class ParquetQuerySuite extends QueryTest with FunSuite with BeforeAndAfterAll {
Utils.deleteRecursively(ParquetTestData.testDir)
ParquetTestData.writeFile()
}

test("save and load case class RDD with nulls as parquet") {
val data = NullReflectData(null, null, null, null, null)
val rdd = sparkContext.parallelize(data :: Nil)

val file = getTempFilePath("parquet")
val path = file.toString
rdd.saveAsParquetFile(path)
val readFile = parquetFile(path)

val rdd_saved = readFile.collect()
assert(rdd_saved(0) === Seq.fill(5)(null))
Utils.deleteRecursively(file)
assert(true)
}

test("save and load case class RDD with Nones as parquet") {
val data = OptionalReflectData(null, null, null, null, null)
val rdd = sparkContext.parallelize(data :: Nil)

val file = getTempFilePath("parquet")
val path = file.toString
rdd.saveAsParquetFile(path)
val readFile = parquetFile(path)

val rdd_saved = readFile.collect()
assert(rdd_saved(0) === Seq.fill(5)(null))
Utils.deleteRecursively(file)
assert(true)
}
}

0 comments on commit 8f3f281

Please sign in to comment.