Skip to content

Commit f0ad3cf

Browse files
Undoing changes not needed for this PR
1 parent 210e9cb commit f0ad3cf

File tree

3 files changed

+7
-12
lines changed

3 files changed

+7
-12
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,10 @@ package org.apache.spark.sql.catalyst.types
1919

2020
import java.sql.Timestamp
2121

22-
import scala.reflect.ClassTag
23-
import scala.reflect.runtime.universe.{typeTag, TypeTag, runtimeMirror}
22+
import scala.reflect.runtime.universe.{typeTag, TypeTag}
2423

2524
import org.apache.spark.sql.catalyst.expressions.Expression
2625

27-
import org.apache.spark.util.Utils
28-
2926
abstract class DataType {
3027
/** Matches any expression that evaluates to this DataType */
3128
def unapply(a: Expression): Boolean = a match {
@@ -40,11 +37,6 @@ abstract class NativeType extends DataType {
4037
type JvmType
4138
@transient val tag: TypeTag[JvmType]
4239
val ordering: Ordering[JvmType]
43-
44-
@transient val classTag = {
45-
val mirror = runtimeMirror(Utils.getSparkClassLoader)
46-
ClassTag[JvmType](mirror.runtimeClass(tag.tpe))
47-
}
4840
}
4941

5042
case object StringType extends NativeType {

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,11 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
141141
case logical.InsertIntoTable(table: ParquetRelation, partition, child, overwrite) =>
142142
InsertIntoParquetTable(table, planLater(child), overwrite)(sparkContext) :: Nil
143143
case PhysicalOperation(projectList, filters, relation: ParquetRelation) =>
144+
// Note: we do not actually remove the filters that were pushed down to Parquet from
145+
// the plan, in case that some of the predicates cannot be evaluated there because
146+
// they contain complex operations, such as CASTs.
147+
// TODO: rethink whether conjuntions that are handed down to Parquet should be removed
148+
// from the list of higher-level filters.
144149
pruneFilterProject(
145150
projectList,
146151
filters,

sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,9 @@ import parquet.schema.PrimitiveType.{PrimitiveTypeName => ParquetPrimitiveTypeNa
3333
import parquet.schema.Type.Repetition
3434

3535
import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, UnresolvedException}
36-
import org.apache.spark.sql.catalyst.expressions._
36+
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Row}
3737
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, LeafNode}
3838
import org.apache.spark.sql.catalyst.types._
39-
import org.apache.spark.sql.catalyst.types.ArrayType
40-
import org.apache.spark.sql.catalyst.expressions.AttributeReference
4139

4240
// Implicits
4341
import scala.collection.JavaConversions._

0 commit comments

Comments
 (0)