Skip to content

[SPARK-40315][SQL] Add hashCode() for Literal of ArrayBasedMapData #37807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,9 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
val valueHashCode = value match {
case null => 0
case binary: Array[Byte] => util.Arrays.hashCode(binary)
// SPARK-40315: Literals of ArrayBasedMapData should have deterministic hashCode.
case arrayBasedMapData: ArrayBasedMapData =>
arrayBasedMapData.keyArray.hashCode() * 37 + arrayBasedMapData.valueArray.hashCode()
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is simply re-adding the old hashCode function that was removed in: #13847

case other => other.hashCode()
}
31 * Objects.hashCode(dataType) + valueHashCode
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -517,4 +517,30 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {

assert(m1.semanticEquals(m2))
}

test("SPARK-40315: Literals of ArrayBasedMapData should have deterministic hashCode.") {
val keys = new Array[UTF8String](1)
val values1 = new Array[UTF8String](1)
val values2 = new Array[UTF8String](1)

keys(0) = UTF8String.fromString("key")
values1(0) = UTF8String.fromString("value1")
values2(0) = UTF8String.fromString("value2")

val d1 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values1))
val d2 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values1))
val d3 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values2))
val m1 = Literal.create(d1, MapType(StringType, StringType))
val m2 = Literal.create(d2, MapType(StringType, StringType))
val m3 = Literal.create(d3, MapType(StringType, StringType))

// If two Literals of ArrayBasedMapData have the same elements, we expect them to be equal and
// to have the same hashCode().
assert(m1 == m2)
assert(m1.hashCode() == m2.hashCode())
// If two Literals of ArrayBasedMapData have different elements, we expect them not to be equal
// and to have different hashCode().
assert(m1 != m3)
assert(m1.hashCode() != m3.hashCode())
}
}