Skip to content

Commit 6a91872

Browse files
sarutaksarutak
committed
Temporary commit
1 parent 6693f34 commit 6a91872

File tree

5 files changed

+240
-181
lines changed

5 files changed

+240
-181
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/timeline-view.js

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
function drawApplicationTimeline(groupArray, eventObjArray) {
18+
function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
1919
var groups = new vis.DataSet(groupArray);
2020
var items = new vis.DataSet(eventObjArray);
2121
var container = $("#application-timeline")[0];
@@ -25,6 +25,7 @@ function drawApplicationTimeline(groupArray, eventObjArray) {
2525
},
2626
editable: false,
2727
showCurrentTime: false,
28+
showCustomTime: true,
2829
zoomable: false
2930
};
3031

@@ -33,6 +34,10 @@ function drawApplicationTimeline(groupArray, eventObjArray) {
3334
applicationTimeline.setGroups(groups);
3435
applicationTimeline.setItems(items);
3536

37+
if (startTime != -1) {
38+
applicationTimeline.setCustomTime(startTime);
39+
}
40+
3641
setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
3742

3843
$(".item.range.job.application-timeline-object").each(function() {
@@ -44,7 +49,7 @@ function drawApplicationTimeline(groupArray, eventObjArray) {
4449
});
4550
}
4651

47-
function drawJobTimeline(groupArray, eventObjArray) {
52+
function drawJobTimeline(groupArray, eventObjArray, startTime) {
4853
var groups = new vis.DataSet(groupArray);
4954
var items = new vis.DataSet(eventObjArray);
5055
var container = $('#job-timeline')[0];
@@ -54,6 +59,7 @@ function drawJobTimeline(groupArray, eventObjArray) {
5459
},
5560
editable: false,
5661
showCurrentTime: false,
62+
showCustomTime: true,
5763
zoomable: false,
5864
};
5965

@@ -62,15 +68,19 @@ function drawJobTimeline(groupArray, eventObjArray) {
6268
jobTimeline.setGroups(groups);
6369
jobTimeline.setItems(items);
6470

71+
if (startTime != -1) {
72+
jobTimeline.setCustomTime(startTime)
73+
}
74+
6575
setupZoomable("#job-timeline-zoom-lock", jobTimeline);
6676

67-
$(".item.range.stage.job-timeline-object").each(function() {
68-
$(this).click(function() {
69-
var stageIdText = $($(this).find(".job-timeline-content")[0]).text();
70-
var stageId = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)")[1].replace(".", "-");
71-
window.location.href = "#stage-" + stageId;
72-
});
77+
$(".item.range.stage.job-timeline-object").each(function() {
78+
$(this).click(function() {
79+
var stageIdText = $($(this).find(".job-timeline-content")[0]).text();
80+
var stageId = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)")[1].replace(".", "-");
81+
window.location.href = "#stage-" + stageId;
7382
});
83+
});
7484
}
7585

7686
function setupJobEntryLink(id, timeline) {

‎core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala

Lines changed: 96 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,32 @@
1717

1818
package org.apache.spark.ui.jobs
1919

20+
import scala.collection.mutable.ListBuffer
2021
import scala.xml.{Node, NodeSeq, Unparsed}
2122

2223
import java.util.Date
2324
import javax.servlet.http.HttpServletRequest
2425

2526
import org.apache.spark.ui.{UIUtils, WebUIPage}
26-
import org.apache.spark.ui.jobs.UIData.JobUIData
27+
import org.apache.spark.ui.jobs.UIData.{ExecutorUIData, JobUIData}
2728
import org.apache.spark.JobExecutionStatus
2829

2930
/** Page showing list of all ongoing and recently finished jobs */
3031
private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
3132
private val startTime: Option[Long] = parent.sc.map(_.startTime)
3233
private val listener = parent.listener
34+
private val JOBS_LEGEND =
35+
<div class="legend-area"><svg width="200px" height="85px">
36+
<rect x="5px" y="5px" width="20px" height="15px"
37+
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
38+
<text x="35px" y="17px">Succeeded Job</text>
39+
<rect x="5px" y="35px" width="20px" height="15px"
40+
rx="2px" ry="2px" stroke="#97B0F8" fill="#FF5475"></rect>
41+
<text x="35px" y="47px">Failed Job</text>
42+
<rect x="5px" y="65px" width="20px" height="15px"
43+
rx="2px" ry="2px" stroke="#97B0F8" fill="#FDFFCA"></rect>
44+
<text x="35px" y="77px">Running Job</text>
45+
</svg></div>.toString.filter(_ != '\n')
3346

3447
private def getlastStageDescription(job: JobUIData) = {
3548
val lastStageInfo = Option(job.stageIds)
@@ -41,71 +54,97 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
4154
lastStageData.flatMap(_.description).getOrElse("")
4255
}
4356

44-
private def applicationTimelineView(jobs: Seq[JobUIData], now: Long): Seq[Node] = {
45-
val jobEventJsonAsStrSeq = jobs.flatMap { jobUIData =>
46-
val jobId = jobUIData.jobId
47-
val status = jobUIData.status
48-
val jobDescription = getlastStageDescription(jobUIData)
49-
val submissionTimeOpt = jobUIData.submissionTime
50-
val completionTimeOpt = jobUIData.completionTime
51-
52-
if (status == JobExecutionStatus.UNKNOWN || submissionTimeOpt.isEmpty ||
53-
completionTimeOpt.isEmpty && status != JobExecutionStatus.RUNNING) {
54-
None
55-
}
57+
private def makeTimeline(jobs: Seq[JobUIData], executors: Seq[ExecutorUIData]): Seq[Node] = {
5658

57-
val submissionTime = submissionTimeOpt.get
58-
val completionTime = completionTimeOpt.getOrElse(now)
59-
val classNameByStatus = status match {
60-
case JobExecutionStatus.SUCCEEDED => "succeeded"
61-
case JobExecutionStatus.FAILED => "failed"
62-
case JobExecutionStatus.RUNNING => "running"
63-
}
59+
def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = {
60+
jobUIDatas.flatMap { jobUIData =>
61+
val jobId = jobUIData.jobId
62+
val status = jobUIData.status
63+
val jobDescription = getlastStageDescription(jobUIData)
64+
val submissionTimeOpt = jobUIData.submissionTime
65+
val completionTimeOpt = jobUIData.completionTime
66+
67+
if (status == JobExecutionStatus.UNKNOWN || submissionTimeOpt.isEmpty ||
68+
completionTimeOpt.isEmpty && status != JobExecutionStatus.RUNNING) {
69+
None
70+
}
71+
72+
val submissionTime = submissionTimeOpt.get
73+
val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis())
74+
val classNameByStatus = status match {
75+
case JobExecutionStatus.SUCCEEDED => "succeeded"
76+
case JobExecutionStatus.FAILED => "failed"
77+
case JobExecutionStatus.RUNNING => "running"
78+
}
6479

65-
val jobEventJsonAsStr =
66-
s"""
80+
val jobEventJsonAsStr =
81+
s"""
6782
|{
6883
| 'className': 'job application-timeline-object ${classNameByStatus}',
6984
| 'group': 'jobs',
7085
| 'start': new Date(${submissionTime}),
7186
| 'end': new Date(${completionTime}),
72-
| 'content': '<div class="application-timeline-content">${jobDescription} (Job ${jobId})</div>',
87+
| 'content': '<div class="application-timeline-content">' +
88+
| '${jobDescription} (Job ${jobId})</div>',
7389
| 'title': '${jobDescription} (Job ${jobId})\\nStatus: ${status}\\n' +
7490
| 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' +
7591
| '${
76-
if (status != JobExecutionStatus.RUNNING) {
77-
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
78-
} else {
79-
""
80-
}
92+
if (status != JobExecutionStatus.RUNNING) {
93+
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
94+
} else {
95+
""
96+
}
8197
}'
8298
|}
8399
""".stripMargin
84-
Some(jobEventJsonAsStr)
100+
Some(jobEventJsonAsStr)
101+
}
85102
}
86103

87-
val executorEventJsonAsStrSeq =
88-
(listener.executorIdToAddedTime ++
89-
listener.executorIdToRemovedTimeAndReason).map { event =>
90-
val (executorId: String, status: String, time: Long, reason: Option[String]) =
91-
event match {
92-
case (executorId, (removedTime, reason)) =>
93-
(executorId, "removed", removedTime, Some(reason))
94-
case (executorId, addedTime) =>
95-
(executorId, "added", addedTime, None)
96-
}
97-
s"""
98-
|{
99-
| 'className': 'executor ${status}',
100-
| 'group': 'executors',
101-
| 'start': new Date(${time}),
102-
| 'content': '<div>Executor ${executorId} ${status}</div>',
103-
| 'title': '${if (status == "added") "Added" else "Removed"} ' +
104-
| 'at ${UIUtils.formatDate(new Date(time))}' +
105-
| '${if (reason.isDefined) s"""\\nReason: ${reason.get}""" else ""}'
106-
|}
107-
""".stripMargin
104+
def makeExecutorEvent(executorUIDatas: Seq[ExecutorUIData]): Seq[String] = {
105+
val events = ListBuffer[String]()
106+
executorUIDatas.foreach { event =>
107+
108+
if (event.startTime.isDefined) {
109+
val addedEvent =
110+
s"""
111+
|{
112+
| 'className': 'executor added',
113+
| 'group': 'executors',
114+
| 'start': new Date(${event.startTime.get}),
115+
| 'content': '<div>Executor ${event.executorId} added</div>',
116+
| 'title': 'Added at ${UIUtils.formatDate(new Date(event.startTime.get))}'
117+
|}
118+
""".stripMargin
119+
events += addedEvent
120+
}
121+
122+
if (event.finishTime.isDefined) {
123+
val removedEvent =
124+
s"""
125+
|{
126+
| 'className': 'executor removed',
127+
| 'group': 'executors',
128+
| 'start': new Date(${event.finishTime.get}),
129+
| 'content': '<div>Executor ${event.executorId} removed</div>',
130+
| 'title': 'Removed at ${UIUtils.formatDate(new Date(event.finishTime.get))}' +
131+
| '${
132+
if (event.finishReason.isDefined) {
133+
s"""\\nReason: ${event.finishReason.get}"""
134+
} else {
135+
""
136+
}
137+
}'
138+
|}
139+
""".stripMargin
140+
events += removedEvent
141+
}
108142
}
143+
events.toSeq
144+
}
145+
146+
val jobEventJsonAsStrSeq = makeJobEvent(jobs)
147+
val executorEventJsonAsStrSeq = makeExecutorEvent(executors)
109148

110149
val executorsLegend =
111150
<div class="legend-area"><svg width="200px" height="55px">
@@ -117,19 +156,6 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
117156
<text x="35px" y="47px">Executor Removed</text>
118157
</svg></div>.toString.filter(_ != '\n')
119158

120-
val jobsLegend =
121-
<div class="legend-area"><svg width="200px" height="85px">
122-
<rect x="5px" y="5px" width="20px" height="15px"
123-
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
124-
<text x="35px" y="17px">Succeeded Job</text>
125-
<rect x="5px" y="35px" width="20px" height="15px"
126-
rx="2px" ry="2px" stroke="#97B0F8" fill="#FF5475"></rect>
127-
<text x="35px" y="47px">Failed Job</text>
128-
<rect x="5px" y="65px" width="20px" height="15px"
129-
rx="2px" ry="2px" stroke="#97B0F8" fill="#FDFFCA"></rect>
130-
<text x="35px" y="77px">Running Job</text>
131-
</svg></div>.toString.filter(_ != '\n')
132-
133159
val groupJsonArrayAsStr =
134160
s"""
135161
|[
@@ -139,7 +165,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
139165
| },
140166
| {
141167
| 'id': 'jobs',
142-
| 'content': '<div>Jobs</div>${jobsLegend}',
168+
| 'content': '<div>Jobs</div>${JOBS_LEGEND}',
143169
| }
144170
|]
145171
""".stripMargin
@@ -155,7 +181,8 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
155181
</div> ++
156182
<div id="application-timeline"></div> ++
157183
<script type="text/javascript">
158-
{Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}, ${eventArrayAsStr});")}
184+
{Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}," +
185+
s"${eventArrayAsStr}, ${startTime.getOrElse(-1L)});")}
159186
</script>
160187
}
161188

@@ -180,7 +207,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
180207
}
181208

182209
val lastStageName = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
183-
val lastStageDescription = getlastStageDescription(job)// lastStageData.flatMap(_.description).getOrElse("")
210+
val lastStageDescription = getlastStageDescription(job)
184211
val duration: Option[Long] = {
185212
job.submissionTime.map { start =>
186213
val end = job.completionTime.getOrElse(System.currentTimeMillis())
@@ -229,7 +256,6 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
229256
val activeJobs = listener.activeJobs.values.toSeq
230257
val completedJobs = listener.completedJobs.reverse.toSeq
231258
val failedJobs = listener.failedJobs.reverse.toSeq
232-
val now = System.currentTimeMillis
233259

234260
val activeJobsTable =
235261
jobsTable(activeJobs.sortBy(_.submissionTime.getOrElse(-1L)).reverse)
@@ -249,7 +275,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
249275
// Total duration is not meaningful unless the UI is live
250276
<li>
251277
<strong>Total Duration: </strong>
252-
{UIUtils.formatDuration(now - startTime.get)}
278+
{UIUtils.formatDuration(System.currentTimeMillis() - startTime.get)}
253279
</li>
254280
}}
255281
<li>
@@ -284,8 +310,9 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
284310
</div>
285311

286312
var content = summary
313+
val appStartTime =
287314
content ++= <h4>Events on Application Timeline</h4> ++
288-
applicationTimelineView(activeJobs ++ completedJobs ++ failedJobs, now)
315+
makeTimeline(activeJobs ++ completedJobs ++ failedJobs, listener.executors)
289316

290317
if (shouldShowActiveJobs) {
291318
content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++

0 commit comments

Comments
 (0)