Skip to content

Commit db8647e

Browse files
author
Ilya Ganelin
committed
Added update for shuffleWriteTime around spilled file cleanup in ExternalSorter
1 parent 4a17eed commit db8647e

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,12 @@ private[spark] class SortShuffleWriter[K, V, C](
8888
} finally {
8989
// Clean up our sorter, which may have its own intermediate files
9090
if (sorter != null) {
91+
val startTime = System.nanoTime()
9192
sorter.stop()
93+
context.taskMetrics().shuffleWriteMetrics.getOrElse {
94+
case Some(metrics : ShuffleWriteMetrics) =>
95+
metrics.incShuffleWriteTime(System.nanoTime()-startTime)
96+
}
9297
sorter = null
9398
}
9499
}

0 commit comments

Comments
 (0)