Standardize on using spark.history.* configuration.

Update documentation to mention the config options instead of the old
command line argument, and update the startup script.

Author: Marcelo Vanzin

core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala

@@ -35,7 +35,11 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
   private val UPDATE_INTERVAL_MS = conf.getInt("spark.history.fs.updateInterval",
     conf.getInt("spark.history.updateInterval", 10)) * 1000
 
-  private val logDir = conf.get("spark.history.fs.logDirectory")
+  private val logDir = conf.get("spark.history.fs.logDirectory", null)
+  if (logDir == null) {
+    throw new IllegalArgumentException("Logging directory must be specified.")
+  }
+
   private val fs = Utils.getHadoopFileSystem(logDir)
 
   // A timestamp of when the disk was last accessed to check for log updates

core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala

@@ -17,18 +17,14 @@
 
 package org.apache.spark.deploy.history
 
-import java.net.URI
-
-import org.apache.hadoop.fs.Path
-
 import org.apache.spark.SparkConf
 import org.apache.spark.util.Utils
 
 /**
  * Command-line parser for the master.
  */
 private[spark] class HistoryServerArguments(conf: SparkConf, args: Array[String]) {
-  var logDir = conf.get("spark.history.fs.logDirectory", null)
+  private var logDir: String = null
 
   parse(args.toList)
 
@@ -46,36 +42,17 @@ private[spark] class HistoryServerArguments(conf: SparkConf, args: Array[String]
       case _ =>
         printUsageAndExit(1)
     }
-    validateLogDir()
-    conf.set("spark.history.fs.logDirectory", logDir)
-  }
-
-  private def validateLogDir() {
-    if (logDir == null) {
-      System.err.println("Logging directory must be specified.")
-      printUsageAndExit(1)
-    }
-    val fileSystem = Utils.getHadoopFileSystem(new URI(logDir))
-    val path = new Path(logDir)
-    if (!fileSystem.exists(path)) {
-      System.err.println("Logging directory specified does not exist: %s".format(logDir))
-      printUsageAndExit(1)
-    }
-    if (!fileSystem.getFileStatus(path).isDir) {
-      System.err.println("Logging directory specified is not a directory: %s".format(logDir))
-      printUsageAndExit(1)
+    if (logDir != null) {
+      conf.set("spark.history.fs.logDirectory", logDir)
     }
   }
 
   private def printUsageAndExit(exitCode: Int) {
     System.err.println(
       """
-      |Usage: HistoryServer [-d logDir]
-      |
-      |The preferred way to pass options is to set the configuration below using
-      |SPARK_HISTORY_OPTS. The "-d" command line argument is avalable for backwards
-      |compatibility, and overrides "spark.history.fs.logDirectory".
+      |Usage: HistoryServer
       |
+      |Configuration options can be set by setting the corresponding JVM system property.
       |History Server options are always available; additional options depend on the provider.
       |
       |History Server options:
@@ -84,7 +61,8 @@ private[spark] class HistoryServerArguments(conf: SparkConf, args: Array[String]
       |  spark.history.acls.enable  Whether to enable view acls for all applications (default false)
       |  spark.history.provider     Name of history provider class (defaults to file system-based
       |                             provider)
-      |
+      |  spark.history.retainedApplications Max number of application UIs to keep loaded in memory
+      |                             (default 50)
       |FsHistoryProvider options:
       |
       |  spark.history.fs.logDirectory    Directory where app logs are stored (required)

docs/monitoring.md

@@ -35,11 +35,13 @@ If Spark is run on Mesos or YARN, it is still possible to reconstruct the UI of
 application through Spark's history server, provided that the application's event logs exist.
 You can start a the history server by executing:
 
-    ./sbin/start-history-server.sh <base-logging-directory>
+    ./sbin/start-history-server.sh
 
-The base logging directory must be supplied, and should contain sub-directories that each
-represents an application's event logs. This creates a web interface at
-`http://<server-url>:18080` by default. The history server can be configured as follows:
+When using the file-system provider class (see spark.history.provider below), the base logging
+directory must be supplied in the "spark.history.fs.logDirectory" configuration option, and should
+contain sub-directories that each represents an application's event logs. This creates a web
+interface at `http://<server-url>:18080` by default. The history server can be configured as
+follows:
 
 <table class="table">
   <tr><th style="width:21%">Environment Variable</th><th>Meaning</th></tr>
@@ -68,6 +70,12 @@ represents an application's event logs. This creates a web interface at
 
 <table class="table">
   <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr>
+    <td>spark.history.provider</td>
+    <td>org.apache.spark.deploy.history.FsHistoryProvider</td>
+    <td>Name of the class implementing the application history backend. Currently there is only
+    one implementation provided by Spark, which matches the default value.</td>
+  </tr>
   <tr>
     <td>spark.history.fs.updateInterval</td>
     <td>10</td>
@@ -78,7 +86,7 @@ represents an application's event logs. This creates a web interface at
   </tr>
   <tr>
     <td>spark.history.retainedApplications</td>
-    <td>250</td>
+    <td>50</td>
     <td>
       The number of application UIs to retain. If this cap is exceeded, then the oldest
       applications will be removed.

sbin/start-history-server.sh

@@ -19,19 +19,18 @@
 
 # Starts the history server on the machine this script is executed on.
 #
-# Usage: start-history-server.sh <base-log-dir> [<web-ui-port>]
-#   Example: ./start-history-server.sh --dir /tmp/spark-events --port 18080
+# Usage: start-history-server.sh
+#
+# Use the SPARK_HISTORY_OPTS environment variable to set history server configuration.
 #
 
 sbin=`dirname "$0"`
 sbin=`cd "$sbin"; pwd`
 
-if [ $# -lt 1 ]; then
-  echo "Usage: ./start-history-server.sh <base-log-dir>"
-  echo "Example: ./start-history-server.sh /tmp/spark-events"
-  exit
+if [ $# != 0 ]; then
+  echo "Using command line arguments for setting the log directory is deprecated. Please "
+  echo "set the spark.history.fs.logDirectory configuration option instead."
+  export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=$1"
 fi
 
-LOG_DIR=$1
-
-"$sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1 --dir "$LOG_DIR"
+exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1