From 70a35447b05296b0a69e2fdccb6e83957e67f4f4 Mon Sep 17 00:00:00 2001 From: yantangzhai Date: Thu, 18 Dec 2014 16:37:08 +0800 Subject: [PATCH] [SPARK-4693] [SQL] PruningPredicates may be wrong if predicates contains an empty AttributeSet() references --- .../spark/sql/catalyst/expressions/AttributeSet.scala | 2 ++ .../scala/org/apache/spark/sql/hive/HiveStrategies.scala | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala index 2b4969b7cfec0..4b1185d7cb75f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeSet.scala @@ -112,4 +112,6 @@ class AttributeSet private (val baseSet: Set[AttributeEquals]) override def toSeq: Seq[Attribute] = baseSet.map(_.a).toArray.toSeq override def toString = "{" + baseSet.map(_.a).mkString(", ") + "}" + + def isEmpty: Boolean = baseSet.isEmpty } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 07d82f6fd5db5..489f1e1617d7b 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -194,10 +194,9 @@ private[hive] trait HiveStrategies { // Filter out all predicates that only deal with partition keys, these are given to the // hive table scan operator to be used for partition pruning. val partitionKeyIds = AttributeSet(relation.partitionKeys) - val (pruningPredicates, otherPredicates) = predicates.partition { x => - x.references.baseSet != null && - !x.references.baseSet.isEmpty && - x.references.subsetOf(partitionKeyIds) + val (pruningPredicates, otherPredicates) = predicates.partition { predicate => + !predicate.references.isEmpty && + predicate.references.subsetOf(partitionKeyIds) } pruneFilterProject(