Skip to content

Commit b0478c0

Browse files
pwendellpwendell
committed
Changes to simplify the build of SPARK-2706
This simply uses two profiles and defines the source location based on the profiles. It also removes a bunch of other excludes. Finally, this doesn't use dynamic profile activiation which could interfere with third-party libraries that try to link against the spark-hive artifact.
1 parent 2b50502 commit b0478c0

File tree

5 files changed

+82
-160
lines changed

5 files changed

+82
-160
lines changed

‎assembly/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,12 @@
197197
<artifactId>spark-hive_${scala.binary.version}</artifactId>
198198
<version>${project.version}</version>
199199
</dependency>
200+
</dependencies>
201+
</profile>
202+
<profile>
203+
<!-- TODO: Move this to "hive" profile once 0.13 JDBC is supported -->
204+
<id>hive-0.12.0</id>
205+
<dependencies>
200206
<dependency>
201207
<groupId>org.apache.spark</groupId>
202208
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>

‎docs/_config.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
highlighter: pygments
22
markdown: kramdown
3-
gems:
4-
- jekyll-redirect-from
53

64
# For some reason kramdown seems to behave differently on different
75
# OS/packages wrt encoding. So we hard code this config.

‎docs/building-spark.md

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,20 @@ mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -DskipTests clean package
9797
mvn -Pyarn-alpha -Phadoop-2.3 -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package
9898
{% endhighlight %}
9999

100+
<!--- TODO: Update this when Hive 0.13 JDBC is added -->
101+
100102
# Building With Hive and JDBC Support
101103
To enable Hive integration for Spark SQL along with its JDBC server and CLI,
102-
add the `-Phive` profile to your existing build options.
104+
add the `-Phive` profile to your existing build options. By default Spark
105+
will build with Hive 0.12.0 bindings. You can also build for Hive 0.13.1 using
106+
the `-Phive-0.13.1` profile. NOTE: currently the JDBC server is only
107+
supported for Hive 12.
103108
{% highlight bash %}
104-
# Apache Hadoop 2.4.X with Hive support
109+
# Apache Hadoop 2.4.X with Hive 12 support
105110
mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -DskipTests clean package
111+
112+
# Apache Hadoop 2.4.X with Hive 13 support
113+
mvn -Pyarn -Phive-0.13.1 -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -DskipTests clean package
106114
{% endhighlight %}
107115

108116
# Spark Tests in Maven
@@ -192,4 +200,4 @@ To run test suites of a specific sub project as follows:
192200
compiler. When run locally as a background process, it speeds up builds of Scala-based projects
193201
like Spark. Developers who regularly recompile Spark with Maven will be the most interested in
194202
Zinc. The project site gives instructions for building and running `zinc`; OS X users can
195-
install it using `brew install zinc`.
203+
install it using `brew install zinc`.

‎pom.xml

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,11 @@
127127
<hbase.version>0.94.6</hbase.version>
128128
<flume.version>1.4.0</flume.version>
129129
<zookeeper.version>3.4.5</zookeeper.version>
130+
<!-- Version used in Maven Hive dependency -->
131+
<hive.version>0.12.0-protobuf-2.5</hive.version>
132+
<!-- Version used for internal directory structure -->
133+
<hive.version.short>0.12.0</hive.version.short>
134+
<derby.version>10.4.2.0</derby.version>
130135
<parquet.version>1.4.3</parquet.version>
131136
<jblas.version>1.2.3</jblas.version>
132137
<jetty.version>8.1.14.v20131031</jetty.version>
@@ -1260,34 +1265,28 @@
12601265
</dependencies>
12611266
</profile>
12621267
<profile>
1263-
<id>hive-default</id>
1264-
<activation>
1265-
<property>
1266-
<name>!hive.version</name>
1267-
</property>
1268-
</activation>
1269-
<properties>
1270-
<hive.version>0.12.0</hive.version>
1271-
<derby.version>10.4.2.0</derby.version>
1272-
</properties>
1273-
</profile>
1274-
<profile>
1275-
<id>hive</id>
1268+
<id>hive-0.12.0</id>
12761269
<activation>
12771270
<activeByDefault>false</activeByDefault>
12781271
</activation>
1272+
<!-- TODO: Move this to "hive" profile once 0.13 JDBC is supported -->
12791273
<modules>
12801274
<module>sql/hive-thriftserver</module>
12811275
</modules>
1276+
<properties>
1277+
<hive.version>0.12.0-protobuf-2.5</hive.version>
1278+
<hive.version.short>0.12.0-protobuf-2.5</hive.version.short>
1279+
<derby.version>10.4.2.0</derby.version>
1280+
</properties>
12821281
</profile>
12831282
<profile>
1284-
<id>hive-versions</id>
1283+
<id>hive-0.13.1</id>
12851284
<activation>
1286-
<property>
1287-
<name>hive.version</name>
1288-
</property>
1285+
<activeByDefault>false</activeByDefault>
12891286
</activation>
12901287
<properties>
1288+
<hive.version>0.13.1</hive.version>
1289+
<hive.version.short>0.13.1</hive.version.short>
12911290
<derby.version>10.10.1.1</derby.version>
12921291
</properties>
12931292
</profile>

‎sql/hive/pom.xml

Lines changed: 49 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@
3636
</properties>
3737

3838
<dependencies>
39+
<dependency>
40+
<groupId>com.twitter</groupId>
41+
<artifactId>parquet-hive-bundle</artifactId>
42+
<version>1.5.0</version>
43+
</dependency>
3944
<dependency>
4045
<groupId>org.apache.spark</groupId>
4146
<artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -46,15 +51,46 @@
4651
<artifactId>spark-sql_${scala.binary.version}</artifactId>
4752
<version>${project.version}</version>
4853
</dependency>
54+
<dependency>
55+
<groupId>org.spark-project.hive</groupId>
56+
<artifactId>hive-metastore</artifactId>
57+
<version>${hive.version}</version>
58+
</dependency>
4959
<dependency>
5060
<groupId>commons-httpclient</groupId>
5161
<artifactId>commons-httpclient</artifactId>
5262
<version>3.1</version>
5363
</dependency>
64+
<dependency>
65+
<groupId>org.spark-project.hive</groupId>
66+
<artifactId>hive-exec</artifactId>
67+
<version>${hive.version}</version>
68+
<exclusions>
69+
<exclusion>
70+
<groupId>commons-logging</groupId>
71+
<artifactId>commons-logging</artifactId>
72+
</exclusion>
73+
</exclusions>
74+
</dependency>
5475
<dependency>
5576
<groupId>org.codehaus.jackson</groupId>
5677
<artifactId>jackson-mapper-asl</artifactId>
5778
</dependency>
79+
<dependency>
80+
<groupId>org.spark-project.hive</groupId>
81+
<artifactId>hive-serde</artifactId>
82+
<version>${hive.version}</version>
83+
<exclusions>
84+
<exclusion>
85+
<groupId>commons-logging</groupId>
86+
<artifactId>commons-logging</artifactId>
87+
</exclusion>
88+
<exclusion>
89+
<groupId>commons-logging</groupId>
90+
<artifactId>commons-logging-api</artifactId>
91+
</exclusion>
92+
</exclusions>
93+
</dependency>
5894
<!-- hive-serde already depends on avro, but this brings in customized config of avro deps from parent -->
5995
<dependency>
6096
<groupId>org.apache.avro</groupId>
@@ -80,76 +116,7 @@
80116
<scope>test</scope>
81117
</dependency>
82118
</dependencies>
83-
84119
<profiles>
85-
<profile>
86-
<id>hive-default</id>
87-
<activation>
88-
<property>
89-
<name>!hive.version</name>
90-
</property>
91-
</activation>
92-
<dependencies>
93-
<dependency>
94-
<groupId>com.twitter</groupId>
95-
<artifactId>parquet-hive-bundle</artifactId>
96-
<version>1.5.0</version>
97-
</dependency>
98-
<dependency>
99-
<groupId>org.spark-project.hive</groupId>
100-
<artifactId>hive-metastore</artifactId>
101-
<version>${hive.version}</version>
102-
</dependency>
103-
<dependency>
104-
<groupId>org.spark-project.hive</groupId>
105-
<artifactId>hive-exec</artifactId>
106-
<version>${hive.version}</version>
107-
<exclusions>
108-
<exclusion>
109-
<groupId>commons-logging</groupId>
110-
<artifactId>commons-logging</artifactId>
111-
</exclusion>
112-
</exclusions>
113-
</dependency>
114-
<dependency>
115-
<groupId>org.spark-project.hive</groupId>
116-
<artifactId>hive-serde</artifactId>
117-
<version>${hive.version}</version>
118-
<exclusions>
119-
<exclusion>
120-
<groupId>commons-logging</groupId>
121-
<artifactId>commons-logging</artifactId>
122-
</exclusion>
123-
<exclusion>
124-
<groupId>commons-logging</groupId>
125-
<artifactId>commons-logging-api</artifactId>
126-
</exclusion>
127-
</exclusions>
128-
</dependency>
129-
</dependencies>
130-
<build>
131-
<plugins>
132-
<plugin>
133-
<groupId>org.codehaus.mojo</groupId>
134-
<artifactId>build-helper-maven-plugin</artifactId>
135-
<executions>
136-
<execution>
137-
<id>add-default-sources</id>
138-
<phase>generate-sources</phase>
139-
<goals>
140-
<goal>add-source</goal>
141-
</goals>
142-
<configuration>
143-
<sources>
144-
<source>v${hive.version}/src/main/scala</source>
145-
</sources>
146-
</configuration>
147-
</execution>
148-
</executions>
149-
</plugin>
150-
</plugins>
151-
</build>
152-
</profile>
153120
<profile>
154121
<id>hive</id>
155122
<build>
@@ -176,91 +143,35 @@
176143
</plugins>
177144
</build>
178145
</profile>
179-
<profile>
180-
<id>hive-versions</id>
181-
<activation>
182-
<property>
183-
<name>hive.version</name>
184-
</property>
185-
</activation>
186-
<dependencies>
187-
<dependency>
188-
<groupId>org.apache.hive</groupId>
189-
<artifactId>hive-metastore</artifactId>
190-
<version>${hive.version}</version>
191-
</dependency>
192-
<dependency>
193-
<groupId>org.apache.hive</groupId>
194-
<artifactId>hive-exec</artifactId>
195-
<version>${hive.version}</version>
196-
<exclusions>
197-
<exclusion>
198-
<groupId>commons-logging</groupId>
199-
<artifactId>commons-logging</artifactId>
200-
</exclusion>
201-
</exclusions>
202-
</dependency>
203-
<dependency>
204-
<groupId>org.apache.hive</groupId>
205-
<artifactId>hive-serde</artifactId>
206-
<version>${hive.version}</version>
207-
<exclusions>
208-
<exclusion>
209-
<groupId>commons-logging</groupId>
210-
<artifactId>commons-logging</artifactId>
211-
</exclusion>
212-
<exclusion>
213-
<groupId>commons-logging</groupId>
214-
<artifactId>commons-logging-api</artifactId>
215-
</exclusion>
216-
</exclusions>
217-
</dependency>
218-
</dependencies>
219-
<build>
220-
<plugins>
146+
</profiles>
147+
148+
<build>
149+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
150+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
151+
<plugins>
152+
<plugin>
153+
<groupId>org.scalatest</groupId>
154+
<artifactId>scalatest-maven-plugin</artifactId>
155+
</plugin>
156+
221157
<plugin>
222158
<groupId>org.codehaus.mojo</groupId>
223159
<artifactId>build-helper-maven-plugin</artifactId>
224160
<executions>
225161
<execution>
226-
<id>add-v13-sources</id>
162+
<id>add-default-sources</id>
227163
<phase>generate-sources</phase>
228164
<goals>
229165
<goal>add-source</goal>
230166
</goals>
231167
<configuration>
232168
<sources>
233-
<source>v${hive.version}/src/main/scala</source>
234-
</sources>
235-
</configuration>
236-
</execution>
237-
<execution>
238-
<id>add-scala-test-sources</id>
239-
<phase>generate-test-sources</phase>
240-
<goals>
241-
<goal>add-test-source</goal>
242-
</goals>
243-
<configuration>
244-
<sources>
245-
<source>src/test/scala</source>
169+
<source>v${hive.version.short}/src/main/scala</source>
246170
</sources>
247171
</configuration>
248172
</execution>
249173
</executions>
250174
</plugin>
251-
</plugins>
252-
</build>
253-
</profile>
254-
</profiles>
255-
256-
<build>
257-
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
258-
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
259-
<plugins>
260-
<plugin>
261-
<groupId>org.scalatest</groupId>
262-
<artifactId>scalatest-maven-plugin</artifactId>
263-
</plugin>
264175

265176
<!-- Deploy datanucleus jars to the spark/lib_managed/jars directory -->
266177
<plugin>

0 commit comments

Comments
 (0)