Merge pull request apache#195 from cafreeman/sparkr-sql

New 1.3 repo and updates to `column.R`

Author: shivaram

pkg/R/column.R

@@ -26,7 +26,7 @@ column <- function(jc) {
 # TODO: change Dsl to functions once update spark-sql
 # A helper function to create a column from name
 col <- function(x) {
-  column(callJStatic("org.apache.spark.sql.Dsl", "col", x))
+  column(callJStatic("org.apache.spark.sql.functions", "col", x))
 }
 
 # TODO(davies): like, rlike, startwith, substr, getField, getItem
@@ -38,7 +38,7 @@ operators <- list(
 )
 
 functions <- c("min", "max", "sum", "avg", "mean", "count", "abs", "sqrt",
-               "first", "last", "asc", "desc", "lower", "upper", "sumDistinct",
+               "first", "last", "lower", "upper", "sumDistinct",
                "isNull", "isNotNull")
 
 createOperator <- function(op) {
@@ -65,7 +65,7 @@ createFunction <- function(name) {
   setMethod(name,
             signature(x = "Column"),
             function(x) {
-              jc <- callJStatic("org.apache.spark.sql.Dsl", name, x@jc)
+              jc <- callJStatic("org.apache.spark.sql.functions", name, x@jc)
               column(jc)
             })
 }
@@ -77,8 +77,6 @@ createMethods <- function() {
 
   setGeneric("avg", function(x) { standardGeneric("avg") })
   setGeneric("last", function(x) { standardGeneric("last") })
-  setGeneric("asc", function(x) { standardGeneric("asc") })
-  setGeneric("desc", function(x) { standardGeneric("desc") })
   setGeneric("lower", function(x) { standardGeneric("lower") })
   setGeneric("upper", function(x) { standardGeneric("upper") })
   setGeneric("isNull", function(x) { standardGeneric("isNull") })
@@ -92,6 +90,24 @@ createMethods <- function() {
 
 createMethods()
 
+setGeneric("asc", function(x) { standardGeneric("asc") })
+
+setMethod("asc",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJMethod(x@jc, "asc")
+            column(jc)
+          })
+
+setGeneric("desc", function(x) { standardGeneric("desc") })
+
+setMethod("desc",
+          signature(x = "Column"),
+          function(x) {
+            jc <- callJMethod(x@jc, "desc")
+            column(jc)
+          })
+
 setMethod("alias",
           signature(object = "Column"),
           function(object, data) {
@@ -117,7 +133,7 @@ setGeneric("approxCountDistinct", function(x, ...) { standardGeneric("approxCoun
 setMethod("approxCountDistinct",
           signature(x = "Column"),
           function(x, rsd = 0.95) {
-            jc <- callJStatic("org.apache.spark.sql.Dsl", "approxCountDistinct", x@jc, rsd)
+            jc <- callJStatic("org.apache.spark.sql.functions", "approxCountDistinct", x@jc, rsd)
             column(jc)
           })
 
@@ -129,7 +145,7 @@ setMethod("countDistinct",
             jcol <- lapply(list(...), function (x) {
               x@jc
             })
-            jc <- callJStatic("org.apache.spark.sql.Dsl", "countDistinct", x@jc, listToSeq(jcol))
+            jc <- callJStatic("org.apache.spark.sql.functions", "countDistinct", x@jc, listToSeq(jcol))
             column(jc)
           })
 

pkg/inst/tests/test_sparkSQL.R

@@ -166,7 +166,7 @@ test_that("schema(), dtypes(), columns(), names() return the correct values/form
   df <- jsonFile(sqlCtx, jsonPath)
   testSchema <- schema(df)
   expect_true(length(testSchema$fields()) == 2)
-  expect_true(testSchema$fields()[[1]]$dataType.toString() == "IntegerType")
+  expect_true(testSchema$fields()[[1]]$dataType.toString() == "LongType")
   expect_true(testSchema$fields()[[2]]$dataType.simpleString() == "string")
   expect_true(testSchema$fields()[[1]]$name() == "age")
 

pkg/src/build.sbt

@@ -29,7 +29,7 @@ libraryDependencies ++= Seq(
   val excludeHadoop = ExclusionRule(organization = "org.apache.hadoop")
   val sbtYarnFlag = scala.util.Properties.envOrElse("USE_YARN", "")
   val defaultHadoopVersion = "1.0.4"
-  val defaultSparkVersion = "1.3.0"
+  val defaultSparkVersion = "1.3.0-SNAPSHOT-sparkr"
   val hadoopVersion = scala.util.Properties.envOrElse("SPARK_HADOOP_VERSION", defaultHadoopVersion)
   val sparkVersion = scala.util.Properties.envOrElse("SPARK_VERSION", defaultSparkVersion)
   libraryDependencies ++= Seq(
@@ -53,6 +53,7 @@ libraryDependencies ++= Seq(
 }
 
 resolvers ++= Seq(
+  "SparkR 1.3 Snapshots" at "https://s3-us-west-2.amazonaws.com/sparkr-dep-1.3/release",
   "Apache Staging" at "https://repository.apache.org/content/repositories/staging/",
   "Typesafe" at "http://repo.typesafe.com/typesafe/releases",
   "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/",