diff --git a/bin/run-example b/bin/run-example index 146951ac0ee56..63622735ed1a9 100755 --- a/bin/run-example +++ b/bin/run-example @@ -21,18 +21,25 @@ SCALA_VERSION=2.10 FWDIR="$(cd `dirname $0`/..; pwd)" export SPARK_HOME="$FWDIR" -EXAMPLES_DIR="$FWDIR"/examples -if [ -f "$FWDIR/RELEASE" ]; then - export SPARK_EXAMPLES_JAR=`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar` -elif [ -e "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar ]; then - export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar` -fi +. $FWDIR/bin/load-spark-env.sh +. $FWDIR/bin/sh-funcs.sh + +# Figure out the JAR file that our examples were packaged into. This includes a bit of a hack +# to avoid the -sources and -doc packages that are built by publish-local. +EXAMPLES_DIR="$FWDIR"/examples if [[ -z $SPARK_EXAMPLES_JAR ]]; then - echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" >&2 - echo "You need to build Spark before running this program" >&2 - exit 1 + if [ -f "$FWDIR/RELEASE" ]; then + one_glob "$FWDIR/lib/spark-examples-*hadoop*.jar" + export SPARK_EXAMPLES_JAR=$_RET + fi + if [[ -z $SPARK_EXAMPLES_JAR ]]; then + jars_pat="$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar" + errmsg="You need to build Spark before running this program" + need_one_glob "$jars_pat" "$errmsg" + export SPARK_EXAMPLES_JAR=$_RET + fi fi EXAMPLE_MASTER=${MASTER:-"local[*]"} diff --git a/bin/sh-funcs.sh b/bin/sh-funcs.sh new file mode 100644 index 0000000000000..4509551c21f30 --- /dev/null +++ b/bin/sh-funcs.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +## Glob for a pattern. Return value is array in $_RET. +function glob() { + local pat="$1" + shopt -s nullglob + unset _RET + _RET=( $pat ) +} + +## Glob for a pattern, and die if there's more than one match. Return +## value is array in $_RET. +function one_glob() { + glob "$1" + + if (( ${#_RET[@]} > 1 )); then + echo "Found multiple files matching $1" >&2 + echo "Please remove all but one." >&2 + exit 1 + fi +} + +## Glob for a pattern, and die if there's not exactly one match. +## Return value is string (not array) in $_RET. +function need_one_glob() { + local pat="$1" + local errtext="$2" + one_glob "$pat" + local files=(${_RET[@]}) + unset _RET + + if (( ${#files[@]} == 0 )); then + echo "No files found matching $pat" >&2 + echo $errtext >&2 + exit 1 + fi + + _RET=${files[0]} +} diff --git a/bin/spark-class b/bin/spark-class index 6480ccb58d6aa..e3748010afa56 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -27,6 +27,8 @@ SCALA_VERSION=2.10 # Figure out where the Scala framework is installed FWDIR="$(cd `dirname $0`/..; pwd)" +. $FWDIR/bin/sh-funcs.sh + # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" @@ -93,7 +95,7 @@ else if [ `command -v java` ]; then RUNNER="java" else - echo "JAVA_HOME is not set" >&2 + echo "JAVA_HOME is not set and no 'java' executable could be found." >&2 exit 1 fi fi @@ -110,32 +112,23 @@ export JAVA_OPTS if [ ! -f "$FWDIR/RELEASE" ]; then # Exit if the user hasn't compiled Spark - num_jars=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) - jars_list=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar") - if [ "$num_jars" -eq "0" ]; then - echo "Failed to find Spark assembly in $FWDIR/assembly/target/scala-$SCALA_VERSION/" >&2 - echo "You need to build Spark before running this program." >&2 - exit 1 - fi - if [ "$num_jars" -gt "1" ]; then - echo "Found multiple Spark assembly jars in $FWDIR/assembly/target/scala-$SCALA_VERSION:" >&2 - echo "$jars_list" - echo "Please remove all but one jar." - exit 1 - fi + jars_pat="$FWDIR/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar" + errmsg="You need to build Spark before running this program." + need_one_glob "$jars_pat" "$errmsg" + jarfile=$_RET # We don't seem to actually use this value anywhere fi + +# Use the JAR from the Maven build +# TODO: this also needs to become an assembly! TOOLS_DIR="$FWDIR"/tools -SPARK_TOOLS_JAR="" -if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar ]; then - # Use the JAR from the SBT build - export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar` -fi -if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then - # Use the JAR from the Maven build - # TODO: this also needs to become an assembly! - export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar` +one_glob "$TOOLS_DIR/target/spark-tools*[0-9Tg].jar" +SPARK_TOOLS_JAR="$_RET" +if [[ -z $SPARK_TOOLS_JAR ]]; then + one_glob "$TOOLS_DIR/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar" + SPARK_TOOLS_JAR="$_RET" fi +export SPARK_TOOLS_JAR # Compute classpath using external script classpath_output=$($FWDIR/bin/compute-classpath.sh)