Skip to content

Commit aeed4b1

Browse files
sarutaksarutak
committed
Removed stage timeline
1 parent fc1696c commit aeed4b1

File tree

3 files changed

+4
-279
lines changed

3 files changed

+4
-279
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/timeline-view.css

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -23,63 +23,10 @@ div#application-timeline, div#job-timeline {
2323
margin-top: 20px;
2424
}
2525

26-
#task-assignment-timeline div.legend-area {
27-
float: right;
28-
border: 1px solid #000000;
29-
}
30-
31-
#task-assignment-timeline .legend-area>svg {
32-
width: 600px;
33-
height: 80px;
34-
}
35-
36-
#task-assignment-timeline>div.control-panel {
37-
float: left;
38-
}
39-
40-
.control-panel input+span {
41-
cursor: pointer;
42-
}
43-
44-
#task-assignment-timeline>.timeline-header:after {
45-
content: "";
46-
clear: both;
47-
height: 0;
48-
display: block;
49-
visibility: hidden;
50-
}
51-
52-
#task-assignment-timeline div.item.range {
53-
padding: 0px;
54-
height: 40px;
55-
}
56-
57-
.task-assignment-timeline-content {
58-
width: 100%;
59-
height: 19px;
60-
}
61-
62-
.task-assignment-timeline-duration-bar {
63-
width: 100%;
64-
height: 19px;
65-
}
66-
6726
.vis.timeline div.content {
6827
width: 100%;
6928
}
7029

71-
.vis.timeline .item.task.succeeded {
72-
background-color: #D5DDF6;
73-
}
74-
75-
.vis.timeline .item.task.failed {
76-
background-color: #FF5475;
77-
}
78-
79-
.vis.timeline .item.task.running {
80-
background-color: #FDFFCA;
81-
}
82-
8330
.vis.timeline .item.stage.succeeded {
8431
background-color: #D5DDF6;
8532
}

‎core/src/main/resources/org/apache/spark/ui/static/timeline-view.js

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,34 +57,6 @@ function drawJobTimeline(groupArray, eventObjArray) {
5757
setupZoomable("#job-timeline-zoom-lock", jobTimeline);
5858
}
5959

60-
function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, zoomMax) {
61-
var groups = new vis.DataSet(groupArray);
62-
var items = new vis.DataSet(eventObjArray);
63-
var container = $("#task-assignment-timeline")[0]
64-
var options = {
65-
groupOrder: function(a, b) {
66-
return a.value - b.value
67-
},
68-
editable: false,
69-
align: 'left',
70-
selectable: false,
71-
showCurrentTime: false,
72-
zoomable: false,
73-
zoomMax: zoomMax
74-
};
75-
76-
var taskTimeline = new vis.Timeline(container)
77-
taskTimeline.setOptions(options);
78-
taskTimeline.setGroups(groups);
79-
taskTimeline.setItems(items);
80-
var curEnd = taskTimeline.getWindow()["end"].getTime();
81-
if (curEnd - minLaunchTime > zoomMax) {
82-
curEnd = minLaunchTime + zoomMax;
83-
}
84-
taskTimeline.setWindow(minLaunchTime, curEnd);
85-
setupZoomable('#task-assignment-timeline-zoom-lock', taskTimeline);
86-
}
87-
8860
function setupZoomable(id, timeline) {
8961
$(id + '>input[type="checkbox"]').click(function() {
9062
if (this.checked) {

‎core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala

Lines changed: 4 additions & 198 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ package org.apache.spark.ui.jobs
2020
import java.util.Date
2121
import javax.servlet.http.HttpServletRequest
2222

23-
import scala.collection.mutable.HashSet
2423
import scala.xml.{Elem, Node, Unparsed}
2524

2625
import org.apache.commons.lang3.StringEscapeUtils
@@ -209,12 +208,10 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
209208

210209
val unzipped = taskHeadersAndCssClasses.unzip
211210

212-
val currentTime = System.currentTimeMillis()
213211
val taskTable = UIUtils.listingTable(
214212
unzipped._1,
215213
taskRow(hasAccumulators, stageData.hasInput, stageData.hasOutput,
216-
stageData.hasShuffleRead, stageData.hasShuffleWrite,
217-
stageData.hasBytesSpilled, currentTime),
214+
stageData.hasShuffleRead, stageData.hasShuffleWrite, stageData.hasBytesSpilled),
218215
tasks,
219216
headerClasses = unzipped._2)
220217
// Excludes tasks which failed and have incomplete metrics
@@ -434,219 +431,28 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
434431
val maybeAccumulableTable: Seq[Node] =
435432
if (accumulables.size > 0) { <h4>Accumulators</h4> ++ accumulableTable } else Seq()
436433

437-
val executorsSet = new HashSet[(String, String)]
438-
439-
var minLaunchTime = Long.MaxValue
440-
var maxFinishTime = Long.MinValue
441-
var numEffectiveTasks = 0
442-
val executorsArrayStr = stageData.taskData.flatMap {
443-
case (_, taskUIData) =>
444-
val taskInfo = taskUIData.taskInfo
445-
446-
val executorId = taskInfo.executorId
447-
val host = taskInfo.host
448-
executorsSet += ((executorId, host))
449-
450-
val taskId = taskInfo.taskId
451-
val taskIdWithIndexAndAttempt = s"Task ${taskId}(${taskInfo.id})"
452-
453-
val isSucceeded = taskInfo.successful
454-
val isFailed = taskInfo.failed
455-
val isRunning = taskInfo.running
456-
val classNameByStatus = {
457-
if (isSucceeded) {
458-
"succeeded"
459-
} else if (isFailed) {
460-
"failed"
461-
} else if (isRunning) {
462-
"running"
463-
}
464-
}
465-
466-
if (isSucceeded || isRunning || isFailed) {
467-
val launchTime = taskInfo.launchTime
468-
val finishTime = if (!isRunning) taskInfo.finishTime else currentTime
469-
val totalExecutionTime = finishTime - launchTime
470-
minLaunchTime = launchTime.min(minLaunchTime)
471-
maxFinishTime = launchTime.max(maxFinishTime)
472-
numEffectiveTasks += 1
473-
474-
val metricsOpt = taskUIData.taskMetrics
475-
val shuffleReadTime =
476-
metricsOpt.flatMap(_.shuffleReadMetrics.map(_.fetchWaitTime)).getOrElse(0L).toDouble
477-
val shuffleReadTimeProportion =
478-
(shuffleReadTime / totalExecutionTime * 100).toLong
479-
val shuffleWriteTime =
480-
metricsOpt.flatMap(_.shuffleWriteMetrics.map(_.shuffleWriteTime)).getOrElse(0L) / 1e6
481-
val shuffleWriteTimeProportion =
482-
(shuffleWriteTime / totalExecutionTime * 100).toLong
483-
val executorRuntimeProportion =
484-
((metricsOpt.map(_.executorRunTime).getOrElse(0L) -
485-
shuffleReadTime - shuffleWriteTime) / totalExecutionTime * 100).toLong
486-
val serializationTimeProportion =
487-
(metricsOpt.map(_.resultSerializationTime).getOrElse(0L).toDouble /
488-
totalExecutionTime * 100).toLong
489-
val deserializationTimeProportion =
490-
(metricsOpt.map(_.executorDeserializeTime).getOrElse(0L).toDouble /
491-
totalExecutionTime * 100).toLong
492-
val gettingResultTimeProportion =
493-
(getGettingResultTime(taskUIData.taskInfo).toDouble / totalExecutionTime * 100).toLong
494-
val schedulerDelayProportion =
495-
100 - executorRuntimeProportion - shuffleReadTimeProportion -
496-
shuffleWriteTimeProportion - serializationTimeProportion -
497-
deserializationTimeProportion - gettingResultTimeProportion
498-
499-
val schedulerDelayProportionPos = 0
500-
val deserializationTimeProportionPos =
501-
schedulerDelayProportionPos + schedulerDelayProportion
502-
val shuffleReadTimeProportionPos =
503-
deserializationTimeProportionPos + deserializationTimeProportion
504-
val executorRuntimeProportionPos =
505-
shuffleReadTimeProportionPos + shuffleReadTimeProportion
506-
val shuffleWriteTimeProportionPos =
507-
executorRuntimeProportionPos + executorRuntimeProportion
508-
val serializationTimeProportionPos =
509-
shuffleWriteTimeProportionPos + shuffleWriteTimeProportion
510-
val gettingResultTimeProportionPos =
511-
serializationTimeProportionPos + serializationTimeProportion
512-
513-
val timelineObject =
514-
s"""
515-
|{
516-
| 'className': 'task task-assignment-timeline-object ${classNameByStatus}',
517-
| 'group': '${executorId}',
518-
| 'content': '<div class="task-assignment-timeline-content">' +
519-
| '${taskIdWithIndexAndAttempt}</div>' +
520-
| '<svg class="task-assignment-timeline-duration-bar">' +
521-
| '<rect x="${schedulerDelayProportionPos}%" y="0" height="100%"' +
522-
| 'width="${schedulerDelayProportion}%" fill="#F6D76B"></rect>' +
523-
| '<rect x="${deserializationTimeProportionPos}%" y="0" height="100%"' +
524-
| 'width="${deserializationTimeProportion}%" fill="#FFBDD8"></rect>' +
525-
| '<rect x="${shuffleReadTimeProportionPos}%" y="0" height="100%"' +
526-
| 'width="${shuffleReadTimeProportion}%" fill="#8AC7DE"></rect>' +
527-
| '<rect x="${executorRuntimeProportionPos}%" y="0" height="100%"' +
528-
| 'width="${executorRuntimeProportion}%" fill="#D9EB52"></rect>' +
529-
| '<rect x="${shuffleWriteTimeProportionPos}%" y="0" height="100%"' +
530-
| 'width="${shuffleWriteTimeProportion}%" fill="#87796F"></rect>' +
531-
| '<rect x="${serializationTimeProportionPos}%" y="0" height="100%"' +
532-
| 'width="${serializationTimeProportion}%" fill="#93DFB8"></rect>' +
533-
| '<rect x="${gettingResultTimeProportionPos}%" y="0" height="100%"' +
534-
| 'width="${gettingResultTimeProportion}%" fill="#FF9036"></rect></svg>',
535-
| 'start': new Date(${launchTime}),
536-
| 'end': new Date(${finishTime}),
537-
| 'title': '${taskIdWithIndexAndAttempt}\\nStatus: ${taskInfo.status}\\n' +
538-
| 'Launch Time: ${UIUtils.formatDate(new Date(launchTime))}' +
539-
| '${
540-
if (!isRunning) {
541-
s"""\\nFinish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
542-
} else {
543-
""
544-
}
545-
}'
546-
|}
547-
""".stripMargin
548-
Option(timelineObject)
549-
} else {
550-
None
551-
}
552-
}.mkString("[", ",", "]")
553-
554-
val groupArrayStr = executorsSet.map {
555-
case (executorId, host) =>
556-
s"""
557-
|{
558-
| 'id': '${executorId}',
559-
| 'content': '${executorId} / ${host}',
560-
|}
561-
""".stripMargin
562-
}.mkString("[", ",", "]")
563-
564-
var maxWindowInSec = ((maxFinishTime - minLaunchTime) / 1000.0).round
565-
if (maxWindowInSec <= 0) maxWindowInSec = 1
566-
val tasksPerSecond = numEffectiveTasks / maxWindowInSec
567-
var maxZoom = {
568-
if (tasksPerSecond > 100) {
569-
1000L / (tasksPerSecond / 100)
570-
}
571-
else {
572-
24L * 60 * 60 * 1000
573-
}
574-
}
575-
576-
if (maxZoom < 0) maxZoom = 1
577-
578434
val content =
579435
summary ++
580436
showAdditionalMetrics ++
581437
<h4>Summary Metrics for {numCompleted} Completed Tasks</h4> ++
582438
<div>{summaryTable.getOrElse("No tasks have reported metrics yet.")}</div> ++
583439
<h4>Aggregated Metrics by Executor</h4> ++ executorTable.toNodeSeq ++
584440
maybeAccumulableTable ++
585-
<h4>Tasks</h4> ++ taskTable ++
586-
<h4>Task Assignment Timeline</h4> ++
587-
<div id="task-assignment-timeline">
588-
<div class="timeline-header">
589-
{taskAssignmentTimelineControlPanel ++ taskAssignmentTimelineLegend}
590-
</div>
591-
</div> ++
592-
<script type="text/javascript">
593-
{Unparsed(s"drawTaskAssignmentTimeline(" +
594-
s"${groupArrayStr}, ${executorsArrayStr}, ${minLaunchTime}, ${maxZoom})")}
595-
</script>
441+
<h4>Tasks</h4> ++ taskTable
596442

597443
UIUtils.headerSparkPage("Details for Stage %d".format(stageId), content, parent)
598444
}
599445
}
600446

601-
private val taskAssignmentTimelineControlPanel: Seq[Node] = {
602-
<div class="control-panel">
603-
<div id="task-assignment-timeline-zoom-lock">
604-
<input type="checkbox" checked="checked"></input>
605-
<span>Zoom Lock</span>
606-
</div>
607-
</div>
608-
}
609-
610-
private val taskAssignmentTimelineLegend: Seq[Node] = {
611-
<div class="legend-area">
612-
<svg>
613-
<rect x="5px" y="5px" width="20px"
614-
height="15px" rx="2px" fill="#D5DDF6" stroke="#97B0F8"></rect>
615-
<text x="35px" y="17px">Succeeded Task</text>
616-
<rect x="215px" y="5px" width="20px"
617-
height="15px" rx="2px" fill="#FF5475" stroke="#97B0F8"></rect>
618-
<text x="245px" y="17px">Failed Task</text>
619-
<rect x="425px" y="5px" width="20px"
620-
height="15px" rx="2px" fill="#FDFFCA" stroke="#97B0F8"></rect>
621-
<text x="455px" y="17px">Running Task</text>
622-
{
623-
val legendPairs = List(("#FFBDD8", "Task Deserialization Time"),
624-
("#8AC7DE", "Shuffle Read Time"), ("#D9EB52", "Executor Computing Time"),
625-
("#87796F", "Shuffle Write Time"), ("#93DFB8", "Result Serialization TIme"),
626-
("#FF9036", "Getting Result Time"), ("#F6D76B", "Scheduler Delay"))
627-
628-
legendPairs.zipWithIndex.map {
629-
case ((color, name), index) =>
630-
<rect x={5 + (index / 3) * 210 + "px"} y={35 + (index % 3) * 15 + "px"}
631-
width="10px" height="10px" fill={color}></rect>
632-
<text x={25 + (index / 3) * 210 + "px"}
633-
y={45 + (index % 3) * 15 + "px"}>{name}</text>
634-
}
635-
}
636-
</svg>
637-
</div>
638-
}
639-
640447
def taskRow(
641448
hasAccumulators: Boolean,
642449
hasInput: Boolean,
643450
hasOutput: Boolean,
644451
hasShuffleRead: Boolean,
645452
hasShuffleWrite: Boolean,
646-
hasBytesSpilled: Boolean,
647-
currentTime: Long)(taskData: TaskUIData): Seq[Node] = {
453+
hasBytesSpilled: Boolean)(taskData: TaskUIData): Seq[Node] = {
648454
taskData match { case TaskUIData(info, metrics, errorMessage) =>
649-
val duration = if (info.status == "RUNNING") info.timeRunning(currentTime)
455+
val duration = if (info.status == "RUNNING") info.timeRunning(System.currentTimeMillis())
650456
else metrics.map(_.executorRunTime).getOrElse(1L)
651457
val formatDuration = if (info.status == "RUNNING") UIUtils.formatDuration(duration)
652458
else metrics.map(m => UIUtils.formatDuration(m.executorRunTime)).getOrElse("")

0 commit comments

Comments
 (0)