Skip to content

Commit 27e815c

Browse files
clockflycloud-fan
authored andcommitted
[SPARK-16888][SQL] Implements eval method for expression AssertNotNull
## What changes were proposed in this pull request? Implements `eval()` method for expression `AssertNotNull` so that we can convert local projection on LocalRelation to another LocalRelation. ### Before change: ``` scala> import org.apache.spark.sql.catalyst.dsl.expressions._ scala> import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull scala> import org.apache.spark.sql.Column scala> case class A(a: Int) scala> Seq((A(1),2)).toDS().select(new Column(AssertNotNull("_1".attr, Nil))).explain java.lang.UnsupportedOperationException: Only code-generated evaluation is supported. at org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull.eval(objects.scala:850) ... ``` ### After the change: ``` scala> Seq((A(1),2)).toDS().select(new Column(AssertNotNull("_1".attr, Nil))).explain(true) == Parsed Logical Plan == 'Project [assertnotnull('_1) AS assertnotnull(_1)#5] +- LocalRelation [_1#2, _2#3] == Analyzed Logical Plan == assertnotnull(_1): struct<a:int> Project [assertnotnull(_1#2) AS assertnotnull(_1)#5] +- LocalRelation [_1#2, _2#3] == Optimized Logical Plan == LocalRelation [assertnotnull(_1)#5] == Physical Plan == LocalTableScan [assertnotnull(_1)#5] ``` ## How was this patch tested? Unit test. Author: Sean Zhong <[email protected]> Closes #14486 from clockfly/assertnotnull_eval.
1 parent 780c722 commit 27e815c

File tree

2 files changed

+21
-7
lines changed

2 files changed

+21
-7
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -859,17 +859,23 @@ case class AssertNotNull(child: Expression, walkedTypePath: Seq[String])
859859
override def foldable: Boolean = false
860860
override def nullable: Boolean = false
861861

862-
override def eval(input: InternalRow): Any =
863-
throw new UnsupportedOperationException("Only code-generated evaluation is supported.")
862+
private val errMsg = "Null value appeared in non-nullable field:" +
863+
walkedTypePath.mkString("\n", "\n", "\n") +
864+
"If the schema is inferred from a Scala tuple/case class, or a Java bean, " +
865+
"please try to use scala.Option[_] or other nullable types " +
866+
"(e.g. java.lang.Integer instead of int/scala.Int)."
867+
868+
override def eval(input: InternalRow): Any = {
869+
val result = child.eval(input)
870+
if (result == null) {
871+
throw new RuntimeException(errMsg);
872+
}
873+
result
874+
}
864875

865876
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
866877
val childGen = child.genCode(ctx)
867878

868-
val errMsg = "Null value appeared in non-nullable field:" +
869-
walkedTypePath.mkString("\n", "\n", "\n") +
870-
"If the schema is inferred from a Scala tuple/case class, or a Java bean, " +
871-
"please try to use scala.Option[_] or other nullable types " +
872-
"(e.g. java.lang.Integer instead of int/scala.Int)."
873879
val errMsgField = ctx.addReferenceObj("errMsg", errMsg)
874880

875881
val code = s"""

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.spark.sql.catalyst.expressions
1919

2020
import org.apache.spark.SparkFunSuite
21+
import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull
2122
import org.apache.spark.sql.types._
2223

2324
class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -45,6 +46,13 @@ class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
4546
}
4647
}
4748

49+
test("AssertNotNUll") {
50+
val ex = intercept[RuntimeException] {
51+
evaluate(AssertNotNull(Literal(null), Seq.empty[String]))
52+
}.getMessage
53+
assert(ex.contains("Null value appeared in non-nullable field"))
54+
}
55+
4856
test("IsNaN") {
4957
checkEvaluation(IsNaN(Literal(Double.NaN)), true)
5058
checkEvaluation(IsNaN(Literal(Float.NaN)), true)

0 commit comments

Comments
 (0)