Skip to content

Commit 624765c

Browse files
committed
Tests for applySchema.
1 parent aa92e84 commit 624765c

File tree

3 files changed

+41
-0
lines changed

3 files changed

+41
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
9797
*/
9898
@DeveloperApi
9999
def applySchema(rowRDD: RDD[Row], schema: StructType): SchemaRDD = {
100+
// TODO: use MutableProjection when rowRDD is another SchemaRDD and the applied
101+
// schema differs from the existing schema on any field data type.
100102
val logicalPlan = SparkLogicalPlan(ExistingRdd(schema.toAttributes, rowRDD))
101103
new SchemaRDD(this, logicalPlan)
102104
}

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,4 +431,36 @@ class SQLQuerySuite extends QueryTest {
431431
)
432432
clear()
433433
}
434+
435+
test("apply schema") {
436+
val schema = StructType(
437+
StructField("f1", IntegerType, false) ::
438+
StructField("f2", StringType, false) ::
439+
StructField("f3", BooleanType, false) ::
440+
StructField("f4", IntegerType, true) :: Nil)
441+
442+
val rowRDD = unparsedStrings.map { r =>
443+
val values = r.split(",").map(_.trim)
444+
val v4 = try values(3).toInt catch {
445+
case _: NumberFormatException => null
446+
}
447+
Row(values(0).toInt, values(1), values(2).toBoolean, v4)
448+
}
449+
450+
val schemaRDD = applySchema(rowRDD, schema)
451+
schemaRDD.registerAsTable("applySchema")
452+
checkAnswer(
453+
sql("SELECT * FROM applySchema"),
454+
(1, "A1", true, null) ::
455+
(2, "B2", false, null) ::
456+
(3, "C3", true, null) ::
457+
(4, "D4", true, 2147483644) :: Nil)
458+
459+
checkAnswer(
460+
sql("SELECT f1, f4 FROM applySchema"),
461+
(1, null) ::
462+
(2, null) ::
463+
(3, null) ::
464+
(4, 2147483644) :: Nil)
465+
}
434466
}

sql/core/src/test/scala/org/apache/spark/sql/TestData.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,11 @@ object TestData {
128128

129129
case class TableName(tableName: String)
130130
TestSQLContext.sparkContext.parallelize(TableName("test") :: Nil).registerAsTable("tableName")
131+
132+
val unparsedStrings =
133+
TestSQLContext.sparkContext.parallelize(
134+
"1, A1, true, null" ::
135+
"2, B2, false, null" ::
136+
"3, C3, true, null" ::
137+
"4, D4, true, 2147483644" :: Nil)
131138
}

0 commit comments

Comments
 (0)