Code review

Author: ScrapCodes

bin/compute-classpath.sh

@@ -20,8 +20,6 @@
 # This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
 # script and the ExecutorRunner in standalone cluster mode.
 
-SCALA_VERSION=${SCALA_VERSION:-"2.10"}
-
 # Figure out where Spark is installed
 FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 
@@ -36,6 +34,18 @@ else
   CLASSPATH="$CLASSPATH:$FWDIR/conf"
 fi
 
+if [ -z "$SCALA_VERSION" ]; then
+
+    ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11"
+    # if scala-2.11 directory for assembly exists,  we use that. Otherwise we default to 
+    # scala 2.10.
+    if [ -d "$ASSEMBLY_DIR2" ]; then
+        SCALA_VERSION="2.11"
+    else
+        SCALA_VERSION="2.10"
+    fi        
+fi
+
 ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION"
 
 if [ -n "$JAVA_HOME" ]; then

conf/spark-env.sh.template

@@ -3,9 +3,6 @@
 # This file is sourced when running various Spark programs.
 # Copy it as spark-env.sh and edit that to configure Spark for your site.
 
-# Uncomment this if you plan to use scala 2.11
-# SCALA_VERSION=2.11
-
 # Options read when launching programs locally with 
 # ./bin/run-example or ./bin/spark-submit
 # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files

core/pom.xml

@@ -296,10 +296,6 @@
       <artifactId>easymockclassextension</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.twitter</groupId>
-      <artifactId>chill-java</artifactId>
-    </dependency>
     <dependency>
       <groupId>asm</groupId>
       <artifactId>asm</artifactId>

examples/pom.xml

@@ -282,6 +282,8 @@
       </dependencies>
     </profile>
     <profile>
+      <!-- We add source directories specific to Scala 2.10 and 2.11 since some examples
+           work only in one and not the other -->
       <id>scala-2.10</id>
       <activation>
         <activeByDefault>true</activeByDefault>

project/SparkBuild.scala

@@ -100,7 +100,7 @@ object SparkBuild extends PomBuild {
           "conjunction with environment variable.")
       v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
     }
-    if(profiles.exists(_.contains("scala"))) {
+    if(profiles.exists(_.contains("scala-"))) {
       profiles
     } else {
       println("Enabled default scala profile")