Skip to content

Commit c203ce2

Browse files
scwfscwf
committed
adding test case
1 parent b3d35c7 commit c203ce2

File tree

6 files changed

+204
-171
lines changed

6 files changed

+204
-171
lines changed

‎sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717

1818
package org.apache.spark.sql.json
1919

20-
import org.apache.spark.sql._
20+
import org.apache.spark.sql.SQLContext
21+
import org.apache.spark.sql.catalyst.types.StructType
2122
import org.apache.spark.sql.sources._
2223

2324
private[sql] class DefaultSource extends RelationProvider {

‎sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
package org.apache.spark.sql.parquet
1818

1919
import java.util.{List => JList}
20-
import scala.Some
21-
import scala.collection.JavaConversions._
2220

2321
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
2422
import org.apache.hadoop.conf.{Configurable, Configuration}
@@ -30,20 +28,14 @@ import parquet.hadoop.util.ContextUtil
3028
import org.apache.spark.annotation.DeveloperApi
3129
import org.apache.spark.{Partition => SparkPartition, Logging}
3230
import org.apache.spark.rdd.{NewHadoopPartition, RDD}
33-
34-
import org.apache.spark.sql.catalyst.expressions.{Row, SpecificMutableRow, Expression, Attribute}
35-
import org.apache.spark.sql.catalyst.types.{IntegerType, StructType}
31+
import org.apache.spark.sql.catalyst.expressions.{Row, And, SpecificMutableRow, Expression, Attribute}
32+
import org.apache.spark.sql.catalyst.types.{StructField, IntegerType, StructType}
3633
import org.apache.spark.sql.sources._
37-
import org.apache.spark.sql.catalyst.types.StructField
38-
import org.apache.spark.sql.sources.GreaterThan
39-
import org.apache.spark.sql.sources.GreaterThanOrEqual
40-
import org.apache.spark.sql.catalyst.expressions.And
41-
import org.apache.spark.sql.sources.LessThanOrEqual
42-
import org.apache.spark.sql.sources.EqualTo
43-
import org.apache.spark.sql.sources.In
44-
import org.apache.spark.sql.sources.LessThan
4534
import org.apache.spark.sql.{SQLConf, SQLContext}
4635

36+
import scala.collection.JavaConversions._
37+
38+
4739
/**
4840
* Allows creation of parquet based tables using the syntax
4941
* `CREATE TABLE ... USING org.apache.spark.sql.parquet`. Currently the only option required

‎sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,15 @@
1717

1818
package org.apache.spark.sql.sources
1919

20-
import org.apache.spark.Logging
21-
import org.apache.spark.sql._
22-
import org.apache.spark.sql.execution.RunnableCommand
23-
import org.apache.spark.util.Utils
24-
2520
import scala.language.implicitConversions
26-
import scala.Some
2721
import scala.util.parsing.combinator.syntactical.StandardTokenParsers
2822
import scala.util.parsing.combinator.PackratParsers
2923

24+
import org.apache.spark.Logging
25+
import org.apache.spark.sql.SQLContext
26+
import org.apache.spark.sql.catalyst.types._
27+
import org.apache.spark.sql.execution.RunnableCommand
28+
import org.apache.spark.util.Utils
3029
import org.apache.spark.sql.catalyst.plans.logical._
3130
import org.apache.spark.sql.catalyst.SqlLexical
3231

@@ -80,19 +79,19 @@ private[sql] class DDLParser extends StandardTokenParsers with PackratParsers wi
8079
* USING org.apache.spark.sql.avro
8180
* OPTIONS (path "../hive/src/test/resources/data/files/episodes.avro")`
8281
* or
83-
* `CREATE TEMPORARY TABLE avroTable(intField int, stringField string)
82+
* `CREATE TEMPORARY TABLE avroTable(intField int, stringField string...)
8483
* USING org.apache.spark.sql.avro
8584
* OPTIONS (path "../hive/src/test/resources/data/files/episodes.avro")`
8685
*/
8786
protected lazy val createTable: Parser[LogicalPlan] =
8887
( CREATE ~ TEMPORARY ~ TABLE ~> ident ~ (USING ~> className) ~ (OPTIONS ~> options) ^^ {
89-
case tableName ~provider ~ opts =>
88+
case tableName ~ provider ~ opts =>
9089
CreateTableUsing(tableName, Seq.empty, provider, opts)
9190
}
9291
|
93-
CREATE ~ TEMPORARY ~ TABLE ~> ident ~
94-
("(" ~> tableCols <~ ",") ~ (USING ~> className) ~ (OPTIONS ~> options) ^^ {
95-
case tableName ~tableColumns ~ provider ~ opts =>
92+
CREATE ~ TEMPORARY ~ TABLE ~> ident
93+
~ tableCols ~ (USING ~> className) ~ (OPTIONS ~> options) ^^ {
94+
case tableName ~ tableColumns ~ provider ~ opts =>
9695
CreateTableUsing(tableName, tableColumns, provider, opts)
9796
}
9897
)
@@ -101,7 +100,9 @@ private[sql] class DDLParser extends StandardTokenParsers with PackratParsers wi
101100
case e1 ~ e2 => (e1, e2)
102101
}
103102

104-
protected lazy val tableCols: Parser[Seq[(String, String)]] = repsep(tableCol, ",")
103+
protected lazy val tableCols: Parser[Seq[(String, String)]] =
104+
"(" ~> repsep(tableCol, ",") <~ ")"
105+
105106

106107
protected lazy val options: Parser[Map[String, String]] =
107108
"(" ~> repsep(pair, ",") <~ ")" ^^ { case s: Seq[(String, String)] => s.toMap }

‎sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala

Lines changed: 69 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -88,85 +88,97 @@ class FilteredScanSuite extends DataSourceTest {
8888
| to '10'
8989
|)
9090
""".stripMargin)
91+
92+
sql(
93+
"""
94+
|CREATE TEMPORARY TABLE oneToTenFiltered_with_schema(a int, b int)
95+
|USING org.apache.spark.sql.sources.FilteredScanSource
96+
|OPTIONS (
97+
| from '1',
98+
| to '10'
99+
|)
100+
""".stripMargin)
91101
}
102+
Seq("oneToTenFiltered", "oneToTenFiltered_with_schema").foreach { table =>
92103

93-
sqlTest(
94-
"SELECT * FROM oneToTenFiltered",
95-
(1 to 10).map(i => Row(i, i * 2)).toSeq)
104+
sqlTest(
105+
s"SELECT * FROM $table",
106+
(1 to 10).map(i => Row(i, i * 2)).toSeq)
96107

97-
sqlTest(
98-
"SELECT a, b FROM oneToTenFiltered",
99-
(1 to 10).map(i => Row(i, i * 2)).toSeq)
108+
sqlTest(
109+
s"SELECT a, b FROM $table",
110+
(1 to 10).map(i => Row(i, i * 2)).toSeq)
100111

101-
sqlTest(
102-
"SELECT b, a FROM oneToTenFiltered",
103-
(1 to 10).map(i => Row(i * 2, i)).toSeq)
112+
sqlTest(
113+
s"SELECT b, a FROM $table",
114+
(1 to 10).map(i => Row(i * 2, i)).toSeq)
104115

105-
sqlTest(
106-
"SELECT a FROM oneToTenFiltered",
107-
(1 to 10).map(i => Row(i)).toSeq)
116+
sqlTest(
117+
s"SELECT a FROM $table",
118+
(1 to 10).map(i => Row(i)).toSeq)
108119

109-
sqlTest(
110-
"SELECT b FROM oneToTenFiltered",
111-
(1 to 10).map(i => Row(i * 2)).toSeq)
120+
sqlTest(
121+
s"SELECT b FROM $table",
122+
(1 to 10).map(i => Row(i * 2)).toSeq)
112123

113-
sqlTest(
114-
"SELECT a * 2 FROM oneToTenFiltered",
115-
(1 to 10).map(i => Row(i * 2)).toSeq)
124+
sqlTest(
125+
s"SELECT a * 2 FROM $table",
126+
(1 to 10).map(i => Row(i * 2)).toSeq)
116127

117-
sqlTest(
118-
"SELECT A AS b FROM oneToTenFiltered",
119-
(1 to 10).map(i => Row(i)).toSeq)
128+
sqlTest(
129+
s"SELECT A AS b FROM $table",
130+
(1 to 10).map(i => Row(i)).toSeq)
120131

121-
sqlTest(
122-
"SELECT x.b, y.a FROM oneToTenFiltered x JOIN oneToTenFiltered y ON x.a = y.b",
123-
(1 to 5).map(i => Row(i * 4, i)).toSeq)
132+
sqlTest(
133+
s"SELECT x.b, y.a FROM $table x JOIN $table y ON x.a = y.b",
134+
(1 to 5).map(i => Row(i * 4, i)).toSeq)
124135

125-
sqlTest(
126-
"SELECT x.a, y.b FROM oneToTenFiltered x JOIN oneToTenFiltered y ON x.a = y.b",
127-
(2 to 10 by 2).map(i => Row(i, i)).toSeq)
136+
sqlTest(
137+
s"SELECT x.a, y.b FROM $table x JOIN $table y ON x.a = y.b",
138+
(2 to 10 by 2).map(i => Row(i, i)).toSeq)
128139

129-
sqlTest(
130-
"SELECT * FROM oneToTenFiltered WHERE a = 1",
131-
Seq(1).map(i => Row(i, i * 2)).toSeq)
140+
sqlTest(
141+
s"SELECT * FROM $table WHERE a = 1",
142+
Seq(1).map(i => Row(i, i * 2)).toSeq)
132143

133-
sqlTest(
134-
"SELECT * FROM oneToTenFiltered WHERE a IN (1,3,5)",
135-
Seq(1,3,5).map(i => Row(i, i * 2)).toSeq)
144+
sqlTest(
145+
s"SELECT * FROM $table WHERE a IN (1,3,5)",
146+
Seq(1,3,5).map(i => Row(i, i * 2)).toSeq)
136147

137-
sqlTest(
138-
"SELECT * FROM oneToTenFiltered WHERE A = 1",
139-
Seq(1).map(i => Row(i, i * 2)).toSeq)
148+
sqlTest(
149+
s"SELECT * FROM $table WHERE A = 1",
150+
Seq(1).map(i => Row(i, i * 2)).toSeq)
140151

141-
sqlTest(
142-
"SELECT * FROM oneToTenFiltered WHERE b = 2",
143-
Seq(1).map(i => Row(i, i * 2)).toSeq)
152+
sqlTest(
153+
s"SELECT * FROM $table WHERE b = 2",
154+
Seq(1).map(i => Row(i, i * 2)).toSeq)
144155

145-
testPushDown("SELECT * FROM oneToTenFiltered WHERE A = 1", 1)
146-
testPushDown("SELECT a FROM oneToTenFiltered WHERE A = 1", 1)
147-
testPushDown("SELECT b FROM oneToTenFiltered WHERE A = 1", 1)
148-
testPushDown("SELECT a, b FROM oneToTenFiltered WHERE A = 1", 1)
149-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a = 1", 1)
150-
testPushDown("SELECT * FROM oneToTenFiltered WHERE 1 = a", 1)
156+
testPushDown(s"SELECT * FROM $table WHERE A = 1", 1)
157+
testPushDown(s"SELECT a FROM $table WHERE A = 1", 1)
158+
testPushDown(s"SELECT b FROM $table WHERE A = 1", 1)
159+
testPushDown(s"SELECT a, b FROM $table WHERE A = 1", 1)
160+
testPushDown(s"SELECT * FROM $table WHERE a = 1", 1)
161+
testPushDown(s"SELECT * FROM $table WHERE 1 = a", 1)
151162

152-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a > 1", 9)
153-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a >= 2", 9)
163+
testPushDown(s"SELECT * FROM $table WHERE a > 1", 9)
164+
testPushDown(s"SELECT * FROM $table WHERE a >= 2", 9)
154165

155-
testPushDown("SELECT * FROM oneToTenFiltered WHERE 1 < a", 9)
156-
testPushDown("SELECT * FROM oneToTenFiltered WHERE 2 <= a", 9)
166+
testPushDown(s"SELECT * FROM $table WHERE 1 < a", 9)
167+
testPushDown(s"SELECT * FROM $table WHERE 2 <= a", 9)
157168

158-
testPushDown("SELECT * FROM oneToTenFiltered WHERE 1 > a", 0)
159-
testPushDown("SELECT * FROM oneToTenFiltered WHERE 2 >= a", 2)
169+
testPushDown(s"SELECT * FROM $table WHERE 1 > a", 0)
170+
testPushDown(s"SELECT * FROM $table WHERE 2 >= a", 2)
160171

161-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a < 1", 0)
162-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a <= 2", 2)
172+
testPushDown(s"SELECT * FROM $table WHERE a < 1", 0)
173+
testPushDown(s"SELECT * FROM $table WHERE a <= 2", 2)
163174

164-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a > 1 AND a < 10", 8)
175+
testPushDown(s"SELECT * FROM $table WHERE a > 1 AND a < 10", 8)
165176

166-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a IN (1,3,5)", 3)
177+
testPushDown(s"SELECT * FROM $table WHERE a IN (1,3,5)", 3)
167178

168-
testPushDown("SELECT * FROM oneToTenFiltered WHERE a = 20", 0)
169-
testPushDown("SELECT * FROM oneToTenFiltered WHERE b = 1", 10)
179+
testPushDown(s"SELECT * FROM $table WHERE a = 20", 0)
180+
testPushDown(s"SELECT * FROM $table WHERE b = 1", 10)
181+
}
170182

171183
def testPushDown(sqlString: String, expectedCount: Int): Unit = {
172184
test(s"PushDown Returns $expectedCount: $sqlString") {

‎sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala

Lines changed: 58 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -64,54 +64,66 @@ class PrunedScanSuite extends DataSourceTest {
6464
| to '10'
6565
|)
6666
""".stripMargin)
67-
}
68-
69-
sqlTest(
70-
"SELECT * FROM oneToTenPruned",
71-
(1 to 10).map(i => Row(i, i * 2)).toSeq)
72-
73-
sqlTest(
74-
"SELECT a, b FROM oneToTenPruned",
75-
(1 to 10).map(i => Row(i, i * 2)).toSeq)
76-
77-
sqlTest(
78-
"SELECT b, a FROM oneToTenPruned",
79-
(1 to 10).map(i => Row(i * 2, i)).toSeq)
80-
81-
sqlTest(
82-
"SELECT a FROM oneToTenPruned",
83-
(1 to 10).map(i => Row(i)).toSeq)
84-
85-
sqlTest(
86-
"SELECT a, a FROM oneToTenPruned",
87-
(1 to 10).map(i => Row(i, i)).toSeq)
88-
89-
sqlTest(
90-
"SELECT b FROM oneToTenPruned",
91-
(1 to 10).map(i => Row(i * 2)).toSeq)
92-
93-
sqlTest(
94-
"SELECT a * 2 FROM oneToTenPruned",
95-
(1 to 10).map(i => Row(i * 2)).toSeq)
96-
97-
sqlTest(
98-
"SELECT A AS b FROM oneToTenPruned",
99-
(1 to 10).map(i => Row(i)).toSeq)
100-
101-
sqlTest(
102-
"SELECT x.b, y.a FROM oneToTenPruned x JOIN oneToTenPruned y ON x.a = y.b",
103-
(1 to 5).map(i => Row(i * 4, i)).toSeq)
10467

105-
sqlTest(
106-
"SELECT x.a, y.b FROM oneToTenPruned x JOIN oneToTenPruned y ON x.a = y.b",
107-
(2 to 10 by 2).map(i => Row(i, i)).toSeq)
68+
sql(
69+
"""
70+
|CREATE TEMPORARY TABLE oneToTenPruned_with_schema(a int, b int)
71+
|USING org.apache.spark.sql.sources.PrunedScanSource
72+
|OPTIONS (
73+
| from '1',
74+
| to '10'
75+
|)
76+
""".stripMargin)
77+
}
10878

109-
testPruning("SELECT * FROM oneToTenPruned", "a", "b")
110-
testPruning("SELECT a, b FROM oneToTenPruned", "a", "b")
111-
testPruning("SELECT b, a FROM oneToTenPruned", "b", "a")
112-
testPruning("SELECT b, b FROM oneToTenPruned", "b")
113-
testPruning("SELECT a FROM oneToTenPruned", "a")
114-
testPruning("SELECT b FROM oneToTenPruned", "b")
79+
Seq("oneToTenPruned", "oneToTenPruned_with_schema").foreach { table =>
80+
sqlTest(
81+
s"SELECT * FROM $table",
82+
(1 to 10).map(i => Row(i, i * 2)).toSeq)
83+
84+
sqlTest(
85+
s"SELECT a, b FROM $table",
86+
(1 to 10).map(i => Row(i, i * 2)).toSeq)
87+
88+
sqlTest(
89+
s"SELECT b, a FROM $table",
90+
(1 to 10).map(i => Row(i * 2, i)).toSeq)
91+
92+
sqlTest(
93+
s"SELECT a FROM $table",
94+
(1 to 10).map(i => Row(i)).toSeq)
95+
96+
sqlTest(
97+
s"SELECT a, a FROM $table",
98+
(1 to 10).map(i => Row(i, i)).toSeq)
99+
100+
sqlTest(
101+
s"SELECT b FROM $table",
102+
(1 to 10).map(i => Row(i * 2)).toSeq)
103+
104+
sqlTest(
105+
s"SELECT a * 2 FROM $table",
106+
(1 to 10).map(i => Row(i * 2)).toSeq)
107+
108+
sqlTest(
109+
s"SELECT A AS b FROM $table",
110+
(1 to 10).map(i => Row(i)).toSeq)
111+
112+
sqlTest(
113+
s"SELECT x.b, y.a FROM $table x JOIN $table y ON x.a = y.b",
114+
(1 to 5).map(i => Row(i * 4, i)).toSeq)
115+
116+
sqlTest(
117+
s"SELECT x.a, y.b FROM $table x JOIN $table y ON x.a = y.b",
118+
(2 to 10 by 2).map(i => Row(i, i)).toSeq)
119+
120+
testPruning(s"SELECT * FROM $table", "a", "b")
121+
testPruning(s"SELECT a, b FROM $table", "a", "b")
122+
testPruning(s"SELECT b, a FROM $table", "b", "a")
123+
testPruning(s"SELECT b, b FROM $table", "b")
124+
testPruning(s"SELECT a FROM $table", "a")
125+
testPruning(s"SELECT b FROM $table", "b")
126+
}
115127

116128
def testPruning(sqlString: String, expectedColumns: String*): Unit = {
117129
test(s"Columns output ${expectedColumns.mkString(",")}: $sqlString") {

0 commit comments

Comments
 (0)