issue #195: support scala 2.12 (#223)

* issue #195: added scala version to artifact id

* issue #195: update scalatest to 3.0.8

* issue #195: update source code to be compile with scala 2.12

* issue #195: script to change scala version in maven

* issue #195: bump version to 2.4.2-SNAPSHOT

* issue #195: docs updated

Author: fe2s

.travis.yml

@@ -2,14 +2,17 @@ sudo: required
 language: scala
 jdk:
     - openjdk8
-scala:
-    - 2.11.2
 install:
     - wget http://download.redis.io/releases/redis-5.0.1.tar.gz
     - tar -xzvf redis-5.0.1.tar.gz
     - make -C redis-5.0.1 -j4
     - export PATH=$PWD/redis-5.0.1/src:$PATH
-script: make test
+script:
+    - make test # test with scala 2.11
+    - sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile)
+    - ps aux | grep redis
+    - ./dev/change-scala-version.sh 2.12 # switch to scala 2.12
+    - make test # test with scala 2.12
 cache:
     directories:
         - $HOME/.m2

dev/change-scala-version.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -e
+
+VALID_VERSIONS=( 2.11 2.12 )
+
+SCALA_211_MINOR_VERSION="12"
+SCALA_212_MINOR_VERSION="9"
+
+usage() {
+  echo "Usage: $(basename $0) [-h|--help] <version>
+where :
+  -h| --help Display this help text
+  valid version values : ${VALID_VERSIONS[*]}
+" 1>&2
+  exit 1
+}
+
+if [[ ($# -ne 1) || ( $1 == "--help") ||  $1 == "-h" ]]; then
+  usage
+fi
+
+TO_MAJOR_VERSION=$1
+
+check_scala_version() {
+  for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
+  echo "Invalid Scala version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2
+  exit 1
+}
+
+check_scala_version "$TO_MAJOR_VERSION"
+
+if [ $TO_MAJOR_VERSION = "2.12" ]; then
+  FROM_MAJOR_VERSION="2.11"
+  FROM_MINOR_VERSION=$SCALA_211_MINOR_VERSION
+  TO_MINOR_VERSION=$SCALA_212_MINOR_VERSION
+else
+  FROM_MAJOR_VERSION="2.12"
+  FROM_MINOR_VERSION=$SCALA_212_MINOR_VERSION
+  TO_MINOR_VERSION=$SCALA_211_MINOR_VERSION
+fi
+
+sed_i() {
+  sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
+}
+
+export -f sed_i
+
+# change <artifactId>
+BASEDIR=$(dirname $0)/..
+find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
+  -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_MAJOR_VERSION'/\1_'$TO_MAJOR_VERSION'/g' {}" \;
+
+# change <scala.major.version>
+find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
+  -exec bash -c "sed_i 's/\(<scala.major.version>\)'$FROM_MAJOR_VERSION'/\1'$TO_MAJOR_VERSION'/g' {}" \;
+
+# change <scala.complete.version>
+find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
+  -exec bash -c "sed_i 's/\(<scala.complete.version>.*\.\)'$FROM_MINOR_VERSION'/\1'$TO_MINOR_VERSION'/g' {}" \;
+

doc/dev.md

@@ -25,3 +25,12 @@ To build Spark-Redis skipping tests, run:
 ```
 mvn clean package -DskipTests
 ```
+
+To change scala version use `./dev/change-scala-version.sh` script. It will change scala version in `pom.xml`. For example:
+```
+./dev/change-scala-version.sh 2.12
+```
+
+```
+./dev/change-scala-version.sh 2.11
+```

pom.xml

@@ -2,8 +2,8 @@
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.redislabs</groupId>
-	<artifactId>spark-redis</artifactId>
-	<version>2.4.1-SNAPSHOT</version>
+	<artifactId>spark-redis_2.11</artifactId>
+	<version>2.4.2-SNAPSHOT</version>
 	<name>Spark-Redis</name>
 	<description>A Spark library for Redis</description>
 	<url>http://github.com/RedisLabs/spark-redis</url>
@@ -300,7 +300,7 @@
 		<dependency>
 			<groupId>org.scalatest</groupId>
 			<artifactId>scalatest_${scala.major.version}</artifactId>
-			<version>2.2.1</version>
+			<version>3.0.8</version>
 			<scope>test</scope>
 		</dependency>
     </dependencies>

src/main/scala/org/apache/spark/sql/redis/RedisSourceRelation.scala

@@ -132,7 +132,7 @@ class RedisSourceRelation(override val sqlContext: SQLContext,
     }
 
     // write data
-    data.foreachPartition { partition =>
+    data.foreachPartition { partition: Iterator[Row] =>
       // grouped iterator to only allocate memory for a portion of rows
       partition.grouped(iteratorGroupingSize).foreach { batch =>
         // the following can be optimized to not create a map