Skip to content

Commit 39e369c

Browse files
author
Jacky Li
committed
fix confilct after DataFrame PR
2 parents dee56e9 + bebf4c4 commit 39e369c

File tree

492 files changed

+31248
-8030
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

492 files changed

+31248
-8030
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ To build Spark and its example programs, run:
2626

2727
(You do not need to do this if you downloaded a pre-built package.)
2828
More detailed documentation is available from the project site, at
29-
["Building Spark with Maven"](http://spark.apache.org/docs/latest/building-spark.html).
29+
["Building Spark"](http://spark.apache.org/docs/latest/building-spark.html).
3030

3131
## Interactive Scala Shell
3232

‎assembly/pom.xml

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,6 @@
4343
</properties>
4444

4545
<dependencies>
46-
<!-- Promote Guava to compile scope in this module so it's included while shading. -->
47-
<dependency>
48-
<groupId>com.google.guava</groupId>
49-
<artifactId>guava</artifactId>
50-
<scope>compile</scope>
51-
</dependency>
5246
<dependency>
5347
<groupId>org.apache.spark</groupId>
5448
<artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -133,20 +127,6 @@
133127
<goal>shade</goal>
134128
</goals>
135129
<configuration>
136-
<relocations>
137-
<relocation>
138-
<pattern>com.google</pattern>
139-
<shadedPattern>org.spark-project.guava</shadedPattern>
140-
<includes>
141-
<include>com.google.common.**</include>
142-
</includes>
143-
<excludes>
144-
<exclude>com/google/common/base/Absent*</exclude>
145-
<exclude>com/google/common/base/Optional*</exclude>
146-
<exclude>com/google/common/base/Present*</exclude>
147-
</excludes>
148-
</relocation>
149-
</relocations>
150130
<transformers>
151131
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
152132
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

‎bin/compute-classpath.sh

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ fi
5050
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
5151
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
5252
"classes ahead of assembly." >&2
53+
# Spark classes
5354
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
54-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
5555
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
5656
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
5757
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
@@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then
6363
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
6464
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
6565
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
66+
# Jars for shaded deps in their original form (copied here during build)
67+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
6668
fi
6769

6870
# Use spark-assembly jar from either RELEASE or assembly directory
@@ -72,22 +74,25 @@ else
7274
assembly_folder="$ASSEMBLY_DIR"
7375
fi
7476

75-
num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar$" | wc -l)"
76-
if [ "$num_jars" -eq "0" ]; then
77-
echo "Failed to find Spark assembly in $assembly_folder"
78-
echo "You need to build Spark before running this program."
79-
exit 1
80-
fi
77+
num_jars=0
78+
79+
for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do
80+
if [[ ! -e "$f" ]]; then
81+
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
82+
echo "You need to build Spark before running this program." 1>&2
83+
exit 1
84+
fi
85+
ASSEMBLY_JAR="$f"
86+
num_jars=$((num_jars+1))
87+
done
88+
8189
if [ "$num_jars" -gt "1" ]; then
82-
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar$")
83-
echo "Found multiple Spark assembly jars in $assembly_folder:"
84-
echo "$jars_list"
85-
echo "Please remove all but one jar."
90+
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
91+
ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2
92+
echo "Please remove all but one jar." 1>&2
8693
exit 1
8794
fi
8895

89-
ASSEMBLY_JAR="$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)"
90-
9196
# Verify that versions of java used to build the jars and run Spark are compatible
9297
jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1)
9398
if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then

‎bin/run-example

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,32 @@ else
3535
fi
3636

3737
if [ -f "$FWDIR/RELEASE" ]; then
38-
export SPARK_EXAMPLES_JAR="`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`"
39-
elif [ -e "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
40-
export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar`"
38+
JAR_PATH="${FWDIR}/lib"
39+
else
40+
JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}"
4141
fi
4242

43-
if [[ -z "$SPARK_EXAMPLES_JAR" ]]; then
44-
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
45-
echo "You need to build Spark before running this program" 1>&2
43+
JAR_COUNT=0
44+
45+
for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do
46+
if [[ ! -e "$f" ]]; then
47+
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
48+
echo "You need to build Spark before running this program" 1>&2
49+
exit 1
50+
fi
51+
SPARK_EXAMPLES_JAR="$f"
52+
JAR_COUNT=$((JAR_COUNT+1))
53+
done
54+
55+
if [ "$JAR_COUNT" -gt "1" ]; then
56+
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
57+
ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2
58+
echo "Please remove all but one jar." 1>&2
4659
exit 1
4760
fi
4861

62+
export SPARK_EXAMPLES_JAR
63+
4964
EXAMPLE_MASTER=${MASTER:-"local[*]"}
5065

5166
if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then

‎bin/spark-class

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2929

3030
# Export this as SPARK_HOME
3131
export SPARK_HOME="$FWDIR"
32+
export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}"
3233

3334
. "$FWDIR"/bin/load-spark-env.sh
3435

@@ -71,6 +72,8 @@ case "$1" in
7172
'org.apache.spark.executor.MesosExecutorBackend')
7273
OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_EXECUTOR_OPTS"
7374
OUR_JAVA_MEM=${SPARK_EXECUTOR_MEMORY:-$DEFAULT_MEM}
75+
export PYTHONPATH="$FWDIR/python:$PYTHONPATH"
76+
export PYTHONPATH="$FWDIR/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH"
7477
;;
7578

7679
# Spark submit uses SPARK_JAVA_OPTS + SPARK_SUBMIT_OPTS +
@@ -118,8 +121,8 @@ fi
118121
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
119122

120123
# Load extra JAVA_OPTS from conf/java-opts, if it exists
121-
if [ -e "$FWDIR/conf/java-opts" ] ; then
122-
JAVA_OPTS="$JAVA_OPTS `cat "$FWDIR"/conf/java-opts`"
124+
if [ -e "$SPARK_CONF_DIR/java-opts" ] ; then
125+
JAVA_OPTS="$JAVA_OPTS `cat "$SPARK_CONF_DIR"/java-opts`"
123126
fi
124127

125128
# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!

‎build/mvn

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ install_app() {
4848
# check if we already have the tarball
4949
# check if we have curl installed
5050
# download application
51-
[ ! -f "${local_tarball}" ] && [ -n "`which curl 2>/dev/null`" ] && \
51+
[ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
5252
echo "exec: curl ${curl_opts} ${remote_tarball}" && \
5353
curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
5454
# if the file still doesn't exist, lets try `wget` and cross our fingers
55-
[ ! -f "${local_tarball}" ] && [ -n "`which wget 2>/dev/null`" ] && \
55+
[ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
5656
echo "exec: wget ${wget_opts} ${remote_tarball}" && \
5757
wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
5858
# if both were unsuccessful, exit
@@ -68,10 +68,10 @@ install_app() {
6868
# Install maven under the build/ folder
6969
install_mvn() {
7070
install_app \
71-
"http://apache.claz.org/maven/maven-3/3.2.3/binaries" \
72-
"apache-maven-3.2.3-bin.tar.gz" \
73-
"apache-maven-3.2.3/bin/mvn"
74-
MVN_BIN="${_DIR}/apache-maven-3.2.3/bin/mvn"
71+
"http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries" \
72+
"apache-maven-3.2.5-bin.tar.gz" \
73+
"apache-maven-3.2.5/bin/mvn"
74+
MVN_BIN="${_DIR}/apache-maven-3.2.5/bin/mvn"
7575
}
7676

7777
# Install zinc under the build/ folder

‎build/sbt-launch-lib.bash

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ acquire_sbt_jar () {
5050
# Download
5151
printf "Attempting to fetch sbt\n"
5252
JAR_DL="${JAR}.part"
53-
if hash curl 2>/dev/null; then
53+
if [ $(command -v curl) ]; then
5454
(curl --silent ${URL1} > "${JAR_DL}" || curl --silent ${URL2} > "${JAR_DL}") && mv "${JAR_DL}" "${JAR}"
55-
elif hash wget 2>/dev/null; then
55+
elif [ $(command -v wget) ]; then
5656
(wget --quiet ${URL1} -O "${JAR_DL}" || wget --quiet ${URL2} -O "${JAR_DL}") && mv "${JAR_DL}" "${JAR}"
5757
else
5858
printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"

‎conf/metrics.properties.template

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
# period 10 Poll period
8888
# unit seconds Units of poll period
8989
# prefix EMPTY STRING Prefix to prepend to metric name
90+
# protocol tcp Protocol ("tcp" or "udp") to use
9091

9192
## Examples
9293
# Enable JmxSink for all instances by class name

‎core/pom.xml

Lines changed: 35 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@
3434
<name>Spark Project Core</name>
3535
<url>http://spark.apache.org/</url>
3636
<dependencies>
37+
<dependency>
38+
<groupId>com.google.guava</groupId>
39+
<artifactId>guava</artifactId>
40+
</dependency>
3741
<dependency>
3842
<groupId>com.twitter</groupId>
3943
<artifactId>chill_${scala.binary.version}</artifactId>
@@ -90,32 +94,45 @@
9094
<groupId>org.apache.curator</groupId>
9195
<artifactId>curator-recipes</artifactId>
9296
</dependency>
97+
98+
<!-- Jetty dependencies promoted to compile here so they are shaded
99+
and inlined into spark-core jar -->
93100
<dependency>
94101
<groupId>org.eclipse.jetty</groupId>
95102
<artifactId>jetty-plus</artifactId>
103+
<scope>compile</scope>
96104
</dependency>
97105
<dependency>
98106
<groupId>org.eclipse.jetty</groupId>
99107
<artifactId>jetty-security</artifactId>
108+
<scope>compile</scope>
100109
</dependency>
101110
<dependency>
102111
<groupId>org.eclipse.jetty</groupId>
103112
<artifactId>jetty-util</artifactId>
113+
<scope>compile</scope>
104114
</dependency>
105115
<dependency>
106116
<groupId>org.eclipse.jetty</groupId>
107117
<artifactId>jetty-server</artifactId>
118+
<scope>compile</scope>
108119
</dependency>
109-
<!--
110-
Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
111-
class exposed in the Java API). The plugin will then remove this dependency from the published
112-
pom, so that Guava does not pollute the client's compilation classpath.
113-
-->
114120
<dependency>
115-
<groupId>com.google.guava</groupId>
116-
<artifactId>guava</artifactId>
121+
<groupId>org.eclipse.jetty</groupId>
122+
<artifactId>jetty-http</artifactId>
123+
<scope>compile</scope>
124+
</dependency>
125+
<dependency>
126+
<groupId>org.eclipse.jetty</groupId>
127+
<artifactId>jetty-continuation</artifactId>
117128
<scope>compile</scope>
118129
</dependency>
130+
<dependency>
131+
<groupId>org.eclipse.jetty</groupId>
132+
<artifactId>jetty-servlet</artifactId>
133+
<scope>compile</scope>
134+
</dependency>
135+
119136
<dependency>
120137
<groupId>org.apache.commons</groupId>
121138
<artifactId>commons-lang3</artifactId>
@@ -204,19 +221,19 @@
204221
<artifactId>stream</artifactId>
205222
</dependency>
206223
<dependency>
207-
<groupId>com.codahale.metrics</groupId>
224+
<groupId>io.dropwizard.metrics</groupId>
208225
<artifactId>metrics-core</artifactId>
209226
</dependency>
210227
<dependency>
211-
<groupId>com.codahale.metrics</groupId>
228+
<groupId>io.dropwizard.metrics</groupId>
212229
<artifactId>metrics-jvm</artifactId>
213230
</dependency>
214231
<dependency>
215-
<groupId>com.codahale.metrics</groupId>
232+
<groupId>io.dropwizard.metrics</groupId>
216233
<artifactId>metrics-json</artifactId>
217234
</dependency>
218235
<dependency>
219-
<groupId>com.codahale.metrics</groupId>
236+
<groupId>io.dropwizard.metrics</groupId>
220237
<artifactId>metrics-graphite</artifactId>
221238
</dependency>
222239
<dependency>
@@ -350,59 +367,28 @@
350367
<verbose>true</verbose>
351368
</configuration>
352369
</plugin>
353-
<plugin>
354-
<groupId>org.apache.maven.plugins</groupId>
355-
<artifactId>maven-shade-plugin</artifactId>
356-
<executions>
357-
<execution>
358-
<phase>package</phase>
359-
<goals>
360-
<goal>shade</goal>
361-
</goals>
362-
<configuration>
363-
<shadedArtifactAttached>false</shadedArtifactAttached>
364-
<artifactSet>
365-
<includes>
366-
<include>com.google.guava:guava</include>
367-
</includes>
368-
</artifactSet>
369-
<filters>
370-
<!-- See comment in the guava dependency declaration above. -->
371-
<filter>
372-
<artifact>com.google.guava:guava</artifact>
373-
<includes>
374-
<include>com/google/common/base/Absent*</include>
375-
<include>com/google/common/base/Optional*</include>
376-
<include>com/google/common/base/Present*</include>
377-
</includes>
378-
</filter>
379-
</filters>
380-
</configuration>
381-
</execution>
382-
</executions>
383-
</plugin>
384-
<!--
385-
Copy guava to the build directory. This is needed to make the SPARK_PREPEND_CLASSES
386-
option work in compute-classpath.sh, since it would put the non-shaded Spark classes in
387-
the runtime classpath.
388-
-->
389370
<plugin>
390371
<groupId>org.apache.maven.plugins</groupId>
391372
<artifactId>maven-dependency-plugin</artifactId>
392373
<executions>
374+
<!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
375+
shaded deps. So here we store jars in their original form which are added
376+
when the classpath is computed. -->
393377
<execution>
394378
<id>copy-dependencies</id>
395379
<phase>package</phase>
396380
<goals>
397381
<goal>copy-dependencies</goal>
398382
</goals>
399-
<configuration>
383+
<configuration>
400384
<outputDirectory>${project.build.directory}</outputDirectory>
401385
<overWriteReleases>false</overWriteReleases>
402386
<overWriteSnapshots>false</overWriteSnapshots>
403387
<overWriteIfNewer>true</overWriteIfNewer>
404388
<useSubDirectoryPerType>true</useSubDirectoryPerType>
405-
<includeArtifactIds>guava</includeArtifactIds>
389+
<includeArtifactIds>
390+
guava,jetty-io,jetty-servlet,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server
391+
</includeArtifactIds>
406392
<silent>true</silent>
407393
</configuration>
408394
</execution>

0 commit comments

Comments
 (0)