Skip to content

Commit 8693288

Browse files
replace multiplier with ByteUnit methods
1 parent 9bf93e9 commit 8693288

File tree

1 file changed

+6
-5
lines changed

1 file changed

+6
-5
lines changed

core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import org.roaringbitmap.{ArrayContainer, BitmapContainer, RoaringArray, Roaring
3131
import org.apache.spark._
3232
import org.apache.spark.api.python.PythonBroadcast
3333
import org.apache.spark.broadcast.HttpBroadcast
34+
import org.apache.spark.network.util.ByteUnit
3435
import org.apache.spark.network.nio.{GetBlock, GotBlock, PutBlock}
3536
import org.apache.spark.scheduler.{CompressedMapStatus, HighlyCompressedMapStatus}
3637
import org.apache.spark.storage._
@@ -51,18 +52,18 @@ class KryoSerializer(conf: SparkConf)
5152

5253
private val bufferSizeKb = conf.getSizeAsKb("spark.kryoserializer.buffer", "64k")
5354

54-
if (bufferSizeKb >= 2048 * 1024) {
55+
if (bufferSizeKb >= ByteUnit.GiB.toKiB(2)) {
5556
throw new IllegalArgumentException("spark.kryoserializer.buffer must be less than " +
56-
s"2048 mb, got: + ${bufferSizeKb/1024} mb.")
57+
s"2048 mb, got: + ${ByteUnit.KiB.toMiB(bufferSizeKb)} mb.")
5758
}
58-
private val bufferSize = (bufferSizeKb * 1024).toInt
59+
private val bufferSize = ByteUnit.KiB.toBytes(bufferSizeKb).toInt
5960

6061
val maxBufferSizeMb = conf.getSizeAsMb("spark.kryoserializer.buffer.max", "64m").toInt
61-
if (maxBufferSizeMb >= 2048) {
62+
if (maxBufferSizeMb >= ByteUnit.GiB.toMiB(2)) {
6263
throw new IllegalArgumentException("spark.kryoserializer.buffer.max must be less than " +
6364
s"2048 mb, got: + $maxBufferSizeMb mb.")
6465
}
65-
private val maxBufferSize = maxBufferSizeMb * 1024 * 1024
66+
private val maxBufferSize = ByteUnit.MiB.toBytes(maxBufferSizeMb).toInt
6667

6768
private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true)
6869
private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false)

0 commit comments

Comments
 (0)