Add table for removed executors in MasterWebUI

Author: andrewor14

core/src/main/scala/org/apache/spark/deploy/master/ApplicationInfo.scala

@@ -37,12 +37,12 @@ private[spark] class ApplicationInfo(
 
   @transient var state: ApplicationState.Value = _
   @transient var executors: mutable.HashMap[Int, ExecutorInfo] = _
+  @transient var removedExecutors: ArrayBuffer[ExecutorInfo] = _
   @transient var coresGranted: Int = _
   @transient var endTime: Long = _
   @transient var appSource: ApplicationSource = _
 
   @transient private var nextExecutorId: Int = _
-  @transient private var exitedExecutors: ArrayBuffer[ExecutorInfo] = _
 
   init()
 
@@ -53,7 +53,7 @@ private[spark] class ApplicationInfo(
     endTime = -1L
     appSource = new ApplicationSource(this)
     nextExecutorId = 0
-    exitedExecutors = new ArrayBuffer[ExecutorInfo]
+    removedExecutors = new ArrayBuffer[ExecutorInfo]
   }
 
   private def newExecutorId(useID: Option[Int] = None): Int = {
@@ -77,16 +77,15 @@ private[spark] class ApplicationInfo(
 
   def removeExecutor(exec: ExecutorInfo) {
     if (executors.contains(exec.id)) {
-      if (exec.state == ExecutorState.EXITED) {
-        exitedExecutors += executors(exec.id)
-      }
+      removedExecutors += executors(exec.id)
       executors -= exec.id
       coresGranted -= exec.cores
     }
   }
 
-  /** Return the information for all live and exited executors. */
-  def executorInfo: Seq[ExecutorInfo] = (executors.values ++ exitedExecutors).toSet.toSeq
+  def exitedExecutors: Seq[ExecutorInfo] = {
+    removedExecutors.filter(_.state == ExecutorState.EXITED)
+  }
 
   private val myMaxCores = desc.maxCores.getOrElse(defaultCores)
 

core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala

@@ -25,7 +25,7 @@ import scala.xml.Node
 import akka.pattern.ask
 import org.json4s.JValue
 
-import org.apache.spark.deploy.JsonProtocol
+import org.apache.spark.deploy.{ExecutorState, JsonProtocol}
 import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
 import org.apache.spark.deploy.master.ExecutorInfo
 import org.apache.spark.ui.{WebUIPage, UIUtils}
@@ -57,43 +57,58 @@ private[spark] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app
     })
 
     val executorHeaders = Seq("ExecutorID", "Worker", "Cores", "Memory", "State", "Logs")
-    val executors = app.executorInfo
-    val executorTable = UIUtils.listingTable(executorHeaders, executorRow, executors)
+    val allExecutors = (app.executors.values ++ app.removedExecutors).toSet.toSeq
+    val executors = allExecutors.filter { exec =>
+      !ExecutorState.isFinished(exec.state) || exec.state == ExecutorState.EXITED
+    }
+    val removedExecutors = allExecutors.diff(executors)
+    val executorsTable = UIUtils.listingTable(executorHeaders, executorRow, executors)
+    val removedExecutorsTable = UIUtils.listingTable(executorHeaders, executorRow, removedExecutors)
 
     val content =
-        <div class="row-fluid">
-          <div class="span12">
-            <ul class="unstyled">
-              <li><strong>ID:</strong> {app.id}</li>
-              <li><strong>Name:</strong> {app.desc.name}</li>
-              <li><strong>User:</strong> {app.desc.user}</li>
-              <li><strong>Cores:</strong>
-              {
-                if (app.desc.maxCores.isEmpty) {
-                  "Unlimited (%s granted)".format(app.coresGranted)
-                } else {
-                  "%s (%s granted, %s left)".format(
-                    app.desc.maxCores.get, app.coresGranted, app.coresLeft)
-                }
+      <div class="row-fluid">
+        <div class="span12">
+          <ul class="unstyled">
+            <li><strong>ID:</strong> {app.id}</li>
+            <li><strong>Name:</strong> {app.desc.name}</li>
+            <li><strong>User:</strong> {app.desc.user}</li>
+            <li><strong>Cores:</strong>
+            {
+              if (app.desc.maxCores.isEmpty) {
+                "Unlimited (%s granted)".format(app.coresGranted)
+              } else {
+                "%s (%s granted, %s left)".format(
+                  app.desc.maxCores.get, app.coresGranted, app.coresLeft)
               }
-              </li>
-              <li>
-                <strong>Executor Memory:</strong>
-                {Utils.megabytesToString(app.desc.memoryPerSlave)}
-              </li>
-              <li><strong>Submit Date:</strong> {app.submitDate}</li>
-              <li><strong>State:</strong> {app.state}</li>
-              <li><strong><a href={app.desc.appUiUrl}>Application Detail UI</a></strong></li>
-            </ul>
-          </div>
+            }
+            </li>
+            <li>
+              <strong>Executor Memory:</strong>
+              {Utils.megabytesToString(app.desc.memoryPerSlave)}
+            </li>
+            <li><strong>Submit Date:</strong> {app.submitDate}</li>
+            <li><strong>State:</strong> {app.state}</li>
+            <li><strong><a href={app.desc.appUiUrl}>Application Detail UI</a></strong></li>
+          </ul>
         </div>
+      </div>
 
-        <div class="row-fluid"> <!-- Executors -->
-          <div class="span12">
-            <h4> Executor Summary </h4>
-            {executorTable}
-          </div>
-        </div>;
+      <div class="row-fluid"> <!-- Executors -->
+        <div class="span12">
+          <h4> Executor Summary </h4>
+          {
+            executorsTable ++
+            {
+              if (removedExecutors.nonEmpty) {
+                <h4> Removed Executors </h4> ++
+                removedExecutorsTable
+              } else {
+                Seq.empty[Node]
+              }
+            }
+          }
+        </div>
+      </div>;
     UIUtils.basicSparkPage(content, "Application: " + app.desc.name)
   }