Merge remote-tracking branch 'upstream/master' into fix_requesthandler_memory_leak

Author: viirya

.rat-excludes

@@ -30,6 +30,9 @@ log4j-defaults.properties
 bootstrap-tooltip.js
 jquery-1.11.1.min.js
 sorttable.js
+vis.min.js
+vis.min.css
+vis.map
 .*avsc
 .*txt
 .*json

LICENSE

@@ -814,6 +814,7 @@ BSD-style licenses
 The following components are provided under a BSD-style license. See project link for details.
 
      (BSD 3 Clause) core (com.github.fommil.netlib:core:1.1.2 - https://github.com/fommil/netlib-java/core)
+     (BSD 3 Clause) JPMML-Model (org.jpmml:pmml-model:1.1.15 - https://github.com/jpmml/jpmml-model)
      (BSD 3-clause style license) jblas (org.jblas:jblas:1.2.3 - http://jblas.org/)
      (BSD License) AntLR Parser Generator (antlr:antlr:2.7.7 - http://www.antlr.org/)
      (BSD License) Javolution (javolution:javolution:5.5.1 - http://javolution.org)

assembly/pom.xml

@@ -194,7 +194,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-assembly-plugin</artifactId>
-            <version>2.4</version>
             <executions>
               <execution>
                 <id>dist</id>

bin/spark-class2.cmd

@@ -61,7 +61,10 @@ if not "x%JAVA_HOME%"=="x" set RUNNER=%JAVA_HOME%\bin\java
 
 rem The launcher library prints the command to be executed in a single line suitable for being
 rem executed by the batch interpreter. So read all the output of the launcher into a variable.
-for /f "tokens=*" %%i in ('cmd /C ""%RUNNER%" -cp %LAUNCH_CLASSPATH% org.apache.spark.launcher.Main %*"') do (
+set LAUNCHER_OUTPUT=%temp%\spark-class-launcher-output-%RANDOM%.txt
+"%RUNNER%" -cp %LAUNCH_CLASSPATH% org.apache.spark.launcher.Main %* > %LAUNCHER_OUTPUT%
+for /f "tokens=*" %%i in (%LAUNCHER_OUTPUT%) do (
   set SPARK_CMD=%%i
 )
+del %LAUNCHER_OUTPUT%
 %SPARK_CMD%

conf/spark-env.sh.template

@@ -3,7 +3,7 @@
 # This file is sourced when running various Spark programs.
 # Copy it as spark-env.sh and edit that to configure Spark for your site.
 
-# Options read when launching programs locally with 
+# Options read when launching programs locally with
 # ./bin/run-example or ./bin/spark-submit
 # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
 # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
@@ -39,6 +39,7 @@
 # - SPARK_WORKER_DIR, to set the working directory of worker processes
 # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
 # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
+# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
 # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
 # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
 

core/pom.xml

@@ -95,6 +95,11 @@
       <artifactId>spark-network-shuffle_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-unsafe_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>net.java.dev.jets3t</groupId>
       <artifactId>jets3t</artifactId>
@@ -478,7 +483,6 @@
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>
-            <version>1.3.2</version>
             <executions>
               <execution>
                 <id>sparkr-pkg</id>

core/src/main/scala/org/apache/spark/storage/TachyonFileSegment.scala renamed to core/src/main/java/org/apache/spark/api/java/function/Function0.java

@@ -15,16 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.spark.storage
+package org.apache.spark.api.java.function;
 
-import tachyon.client.TachyonFile
+import java.io.Serializable;
 
 /**
- * References a particular segment of a file (potentially the entire file), based off an offset and
- * a length.
+ * A zero-argument function that returns an R.
  */
-private[spark] class TachyonFileSegment(val file: TachyonFile, val offset: Long, val length: Long) {
-  override def toString: String = {
-    "(name=%s, offset=%d, length=%d)".format(file.getPath(), offset, length)
-  }
+public interface Function0<R> extends Serializable {
+  public R call() throws Exception;
 }

core/src/main/resources/org/apache/spark/ui/static/timeline-view.css

@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+div#application-timeline, div#job-timeline {
+  margin-bottom: 30px;
+}
+
+#application-timeline div.legend-area {
+  margin-top: 5px;
+}
+
+.vis.timeline div.content {
+  width: 100%;
+}
+
+.vis.timeline .item.stage {
+  cursor: pointer;
+}
+
+.vis.timeline .item.stage.succeeded {
+  background-color: #D5DDF6;
+}
+
+.vis.timeline .item.stage.succeeded.selected {
+   background-color: #D5DDF6;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.completed-stage-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.failed {
+   background-color: #FF5475;
+}
+
+.vis.timeline .item.stage.failed.selected {
+   background-color: #FF5475;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.failed-stage-legend {
+  fill: #FF5475;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.running {
+   background-color: #FDFFCA;
+}
+
+.vis.timeline .item.stage.running.selected {
+   background-color: #FDFFCA;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.active-stage-legend {
+  fill: #FDFFCA;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job {
+  cursor: pointer;
+}
+
+.vis.timeline .item.job.succeeded {
+  background-color: #D5DDF6;
+}
+
+.vis.timeline .item.job.succeeded.selected {
+   background-color: #D5DDF6;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.succeeded-job-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.failed {
+   background-color: #FF5475;
+}
+
+.vis.timeline .item.job.failed.selected {
+   background-color: #FF5475;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.failed-job-legend {
+  fill: #FF5475;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.running {
+   background-color: #FDFFCA;
+}
+
+.vis.timeline .item.job.running.selected {
+   background-color: #FDFFCA;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.running-job-legend {
+  fill: #FDFFCA;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.added {
+  background-color: #D5DDF6;
+}
+
+.legend-area rect.executor-added-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.removed {
+  background-color: #EBCA59;
+}
+
+.legend-area rect.executor-removed-legend {
+  fill: #EBCA59;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.selected {
+  border-color: #FFC200;
+  background-color: #FFF785;
+  z-index: 2;
+}
+
+tr.corresponding-item-hover>td, tr.corresponding-item-hover>th {
+  background-color: #FFE1FA !important;
+}
+
+#application-timeline.collapsed {
+  display: none;
+}
+
+#job-timeline.collapsed {
+  display: none;
+}
+
+.control-panel {
+  margin-bottom: 5px;
+}
+
+span.expand-application-timeline, span.expand-job-timeline {
+  cursor: pointer;
+}
+
+.control-panel input+span {
+  cursor: pointer;
+}
+
+.vis.timeline .item.range .content {
+  position: unset;
+}
+
+.vis.timeline .item .tooltip-inner {
+  max-width: unset !important;
+}