Skip to content

Commit 407b2c9

Browse files
committed
Refine according comments
1 parent 39960d0 commit 407b2c9

File tree

1 file changed

+11
-1
lines changed

1 file changed

+11
-1
lines changed

core/src/main/scala/org/apache/spark/storage/MemoryStore.scala

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,8 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long)
395395
}
396396

397397
// Take into account the amount of memory currently occupied by unrolling blocks
398+
// and minus the pending unroll memory for that block on current thread.
399+
val threadId = Thread.currentThread().getId
398400
val actualFreeMemory = freeMemory - currentUnrollMemory
399401

400402
if (actualFreeMemory < space) {
@@ -482,13 +484,20 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long)
482484
}
483485
}
484486

487+
/**
488+
* Reserve the unroll memory of current unroll successful block used by this thread
489+
* until actually put the block into memory entry.
490+
*/
485491
def reservePendingUnrollMemoryForThisThread(memory: Long): Unit = {
486492
val threadId = Thread.currentThread().getId
487493
accountingLock.synchronized {
488494
pendingUnrollMemoryMap(threadId) = pendingUnrollMemoryMap.getOrElse(threadId, 0L) + memory
489495
}
490496
}
491497

498+
/**
499+
* Release pending unroll memory of current unroll successful block used by this thread
500+
*/
492501
def releasePendingUnrollMemoryForThisThread(): Unit = {
493502
val threadId = Thread.currentThread().getId
494503
accountingLock.synchronized {
@@ -500,7 +509,8 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long)
500509
* Return the amount of memory currently occupied for unrolling blocks across all threads.
501510
*/
502511
def currentUnrollMemory: Long = accountingLock.synchronized {
503-
unrollMemoryMap.values.sum + pendingUnrollMemoryMap.values.sum
512+
unrollMemoryMap.values.sum + pendingUnrollMemoryMap.
513+
filter(_._1 != Thread.currentThread().getId).values.sum
504514
}
505515

506516
/**

0 commit comments

Comments
 (0)