Skip to content

Commit 837785a

Browse files
author
Yanbo Liang
committed
Use mutable row arrays for inner arrays
1 parent 2f001ba commit 837785a

File tree

1 file changed

+9
-2
lines changed

1 file changed

+9
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -418,8 +418,15 @@ private[sql] object JsonRDD extends Logging {
418418
case DecimalType() => toDecimal(value)
419419
case BooleanType => value.asInstanceOf[BooleanType.JvmType]
420420
case NullType => null
421-
case ArrayType(elementType, _) =>
422-
value.asInstanceOf[Seq[Any]].map(enforceCorrectType(_, elementType))
421+
case ArrayType(elementType, _) => {
422+
val arrayLength = value.asInstanceOf[Seq[Any]].length
423+
val arraySlot = if (slot == null) {
424+
(new Array[Any](arrayLength)).toSeq
425+
} else {
426+
slot.asInstanceOf[Seq[Any]]
427+
}
428+
value.asInstanceOf[Seq[Any]].zip(arraySlot).map{case (v, s) => enforceCorrectType(v, elementType,s)}
429+
}
423430
case struct: StructType =>
424431
asRow(value.asInstanceOf[Map[String, Any]], struct, slot.asInstanceOf[GenericMutableRow])
425432
case DateType => toDate(value)

0 commit comments

Comments
 (0)