Skip to content

Commit 21109fb

Browse files
ScrapCodespwendell
ScrapCodes
authored and
pwendell
committed
SPARK-1144 Added license and RAT to check licenses.
Author: Prashant Sharma <[email protected]> Closes #125 from ScrapCodes/rat-integration and squashes the following commits: 64f7c7d [Prashant Sharma] added license headers. fcf28b1 [Prashant Sharma] Review feedback. c0648db [Prashant Sharma] SPARK-1144 Added license and RAT to check licenses.
1 parent 80c2968 commit 21109fb

File tree

7 files changed

+178
-1
lines changed

7 files changed

+178
-1
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@ dist/
4545
spark-*-bin.tar.gz
4646
unit-tests.log
4747
/lib/
48+
rat-results.txt

‎.rat-excludes

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
target
2+
.gitignore
3+
.project
4+
.classpath
5+
.rat-excludes
6+
.*md
7+
derby.log
8+
TAGS
9+
RELEASE
10+
control
11+
docs
12+
fairscheduler.xml.template
13+
log4j.properties
14+
log4j.properties.template
15+
metrics.properties.template
16+
slaves
17+
spark-env.sh
18+
spark-env.sh.template
19+
log4j-defaults.properties
20+
sorttable.js
21+
.*txt
22+
.*data
23+
.*log
24+
cloudpickle.py
25+
join.py
26+
SparkExprTyper.scala
27+
SparkILoop.scala
28+
SparkILoopInit.scala
29+
SparkIMain.scala
30+
SparkImports.scala
31+
SparkJLineCompletion.scala
32+
SparkJLineReader.scala
33+
SparkMemberHandlers.scala
34+
sbt
35+
sbt-launch-lib.bash
36+
plugins.sbt
37+
work
38+
.*\.q
39+
golden

‎dev/check-license

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
21+
acquire_rat_jar () {
22+
23+
URL1="http://search.maven.org/remotecontent?filepath=org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
24+
URL2="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
25+
26+
JAR=$rat_jar
27+
28+
if [[ ! -f "$rat_jar" ]]; then
29+
# Download rat launch jar if it hasn't been downloaded yet
30+
if [ ! -f ${JAR} ]; then
31+
# Download
32+
printf "Attempting to fetch rat\n"
33+
JAR_DL=${JAR}.part
34+
if hash curl 2>/dev/null; then
35+
(curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
36+
elif hash wget 2>/dev/null; then
37+
(wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
38+
else
39+
printf "You do not have curl or wget installed, please install rat manually.\n"
40+
exit -1
41+
fi
42+
fi
43+
if [ ! -f ${JAR} ]; then
44+
# We failed to download
45+
printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n"
46+
exit -1
47+
fi
48+
printf "Launching rat from ${JAR}\n"
49+
fi
50+
}
51+
52+
# Go to the Spark project root directory
53+
FWDIR="$(cd `dirname $0`/..; pwd)"
54+
cd $FWDIR
55+
56+
if test -x "$JAVA_HOME/bin/java"; then
57+
declare java_cmd="$JAVA_HOME/bin/java"
58+
else
59+
declare java_cmd=java
60+
fi
61+
62+
export RAT_VERSION=0.10
63+
export rat_jar=$FWDIR/lib/apache-rat-${RAT_VERSION}.jar
64+
mkdir -p $FWDIR/lib
65+
66+
[[ -f "$rat_jar" ]] || acquire_rat_jar || {
67+
echo "Download failed. Obtain the rat jar manually and place it at $rat_jar"
68+
exit 1
69+
}
70+
71+
$java_cmd -jar $rat_jar -E $FWDIR/.rat-excludes -d $FWDIR > rat-results.txt
72+
73+
ERRORS=$(cat rat-results.txt | grep -e "??")
74+
75+
if test ! -z "$ERRORS"; then
76+
echo "Could not find Apache license headers in the following files:"
77+
echo "$ERRORS"
78+
exit 1
79+
else
80+
echo -e "RAT checks passed."
81+
fi

‎dev/run-tests

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,13 @@ else
3434
fi
3535

3636
JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
37-
[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run, because JDK version is < 1.8."
37+
[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
3838

39+
echo "========================================================================="
40+
echo "Running Apache RAT checks"
41+
echo "========================================================================="
42+
43+
dev/check-license
3944

4045
echo "========================================================================="
4146
echo "Running Scala style checks"

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
118
package org.apache.spark.sql
219
package catalyst
320
package optimizer

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerTest.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
118
package org.apache.spark.sql
219
package catalyst
320
package optimizer

‎sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
118
package org.apache.spark.sql
219
package columnar
320

0 commit comments

Comments
 (0)