Skip to content

Commit 8223ce6

Browse files
committed
[SPARK-6114][SQL] Avoid metastore conversions before plan is resolved
Author: Michael Armbrust <[email protected]> Closes apache#4855 from marmbrus/explodeBug and squashes the following commits: a712249 [Michael Armbrust] [SPARK-6114][SQL] Avoid metastore conversions before plan is resolved
1 parent 26c1c56 commit 8223ce6

File tree

2 files changed

+14
-0
lines changed

2 files changed

+14
-0
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,10 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
444444
*/
445445
object ParquetConversions extends Rule[LogicalPlan] {
446446
override def apply(plan: LogicalPlan): LogicalPlan = {
447+
if (!plan.resolved) {
448+
return plan
449+
}
450+
447451
// Collects all `MetastoreRelation`s which should be replaced
448452
val toBeReplaced = plan.collect {
449453
// Write path

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,23 @@ case class Nested1(f1: Nested2)
3131
case class Nested2(f2: Nested3)
3232
case class Nested3(f3: Int)
3333

34+
case class NestedArray2(b: Seq[Int])
35+
case class NestedArray1(a: NestedArray2)
36+
3437
/**
3538
* A collection of hive query tests where we generate the answers ourselves instead of depending on
3639
* Hive to generate them (in contrast to HiveQuerySuite). Often this is because the query is
3740
* valid, but Hive currently cannot execute it.
3841
*/
3942
class SQLQuerySuite extends QueryTest {
4043

44+
test("explode nested Field") {
45+
Seq(NestedArray1(NestedArray2(Seq(1,2,3)))).toDF.registerTempTable("nestedArray")
46+
checkAnswer(
47+
sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
48+
Row(1) :: Row(2) :: Row(3) :: Nil)
49+
}
50+
4151
test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") {
4252
checkAnswer(
4353
sql("SELECT * FROM (SELECT key + key AS a FROM src SORT BY value) t ORDER BY t.a"),

0 commit comments

Comments
 (0)