Skip to content

Commit 222e4ff

Browse files
author
Davies Liu
committed
Merge branch 'master' into r_docs
Conflicts: R/pkg/R/DataFrame.R R/pkg/R/SQLContext.R R/pkg/inst/profile/shell.R R/pkg/inst/tests/test_sparkSQL.R docs/sql-programming-guide.md
2 parents 89684ce + deb4113 commit 222e4ff

File tree

1,399 files changed

+118780
-18535
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,399 files changed

+118780
-18535
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ scalastyle.txt
6565
scalastyle-output.xml
6666
R-unit-tests.log
6767
R/unit-tests.out
68+
python/lib/pyspark.zip
6869

6970
# For Hive
7071
metastore_db/

‎.rat-excludes

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ TAGS
1515
RELEASE
1616
control
1717
docs
18+
docker.properties.template
1819
fairscheduler.xml.template
1920
spark-defaults.conf.template
2021
log4j.properties
@@ -29,7 +30,12 @@ spark-env.sh.template
2930
log4j-defaults.properties
3031
bootstrap-tooltip.js
3132
jquery-1.11.1.min.js
33+
d3.min.js
34+
dagre-d3.min.js
35+
graphlib-dot.min.js
3236
sorttable.js
37+
vis.min.js
38+
vis.min.css
3339
.*avsc
3440
.*txt
3541
.*json
@@ -67,5 +73,12 @@ logs
6773
.*scalastyle-output.xml
6874
.*dependency-reduced-pom.xml
6975
known_translations
76+
json_expectation
77+
local-1422981759269/*
78+
local-1422981780767/*
79+
local-1425081759269/*
80+
local-1426533911241/*
81+
local-1426633911242/*
82+
local-1430917381534/*
7083
DESCRIPTION
7184
NAMESPACE

‎CONTRIBUTING.md

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
## Contributing to Spark
22

3-
Contributions via GitHub pull requests are gladly accepted from their original
4-
author. Along with any pull requests, please state that the contribution is
5-
your original work and that you license the work to the project under the
6-
project's open source license. Whether or not you state this explicitly, by
7-
submitting any copyrighted material via pull request, email, or other means
8-
you agree to license the material under the project's open source license and
9-
warrant that you have the legal authority to do so.
3+
*Before opening a pull request*, review the
4+
[Contributing to Spark wiki](https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark).
5+
It lists steps that are required before creating a PR. In particular, consider:
6+
7+
- Is the change important and ready enough to ask the community to spend time reviewing?
8+
- Have you searched for existing, related JIRAs and pull requests?
9+
- Is this a new feature that can stand alone as a package on http://spark-packages.org ?
10+
- Is the change being proposed clearly explained and motivated?
1011

11-
Please see the [Contributing to Spark wiki page](https://cwiki.apache.org/SPARK/Contributing+to+Spark)
12-
for more information.
12+
When you contribute code, you affirm that the contribution is your original work and that you
13+
license the work to the project under the project's open source license. Whether or not you
14+
state this explicitly, by submitting any copyrighted material via pull request, email, or
15+
other means you agree to license the material under the project's open source license and
16+
warrant that you have the legal authority to do so.

‎LICENSE

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,36 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
643643
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
644644
THE SOFTWARE.
645645

646+
========================================================================
647+
For d3 (core/src/main/resources/org/apache/spark/ui/static/d3.min.js):
648+
========================================================================
649+
650+
Copyright (c) 2010-2015, Michael Bostock
651+
All rights reserved.
652+
653+
Redistribution and use in source and binary forms, with or without
654+
modification, are permitted provided that the following conditions are met:
655+
656+
* Redistributions of source code must retain the above copyright notice, this
657+
list of conditions and the following disclaimer.
658+
659+
* Redistributions in binary form must reproduce the above copyright notice,
660+
this list of conditions and the following disclaimer in the documentation
661+
and/or other materials provided with the distribution.
662+
663+
* The name Michael Bostock may not be used to endorse or promote products
664+
derived from this software without specific prior written permission.
665+
666+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
667+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
668+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
669+
DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
670+
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
671+
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
672+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
673+
OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
674+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
675+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
646676

647677
========================================================================
648678
For Scala Interpreter classes (all .scala files in repl/src/main/scala
@@ -806,6 +836,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
806836
See the License for the specific language governing permissions and
807837
limitations under the License.
808838

839+
========================================================================
840+
For vis.js (core/src/main/resources/org/apache/spark/ui/static/vis.min.js):
841+
========================================================================
842+
Copyright (C) 2010-2015 Almende B.V.
843+
844+
Vis.js is dual licensed under both
845+
846+
* The Apache 2.0 License
847+
http://www.apache.org/licenses/LICENSE-2.0
848+
849+
and
850+
851+
* The MIT License
852+
http://opensource.org/licenses/MIT
853+
854+
Vis.js may be distributed under either license.
809855

810856
========================================================================
811857
BSD-style licenses
@@ -814,6 +860,7 @@ BSD-style licenses
814860
The following components are provided under a BSD-style license. See project link for details.
815861

816862
(BSD 3 Clause) core (com.github.fommil.netlib:core:1.1.2 - https://github.com/fommil/netlib-java/core)
863+
(BSD 3 Clause) JPMML-Model (org.jpmml:pmml-model:1.1.15 - https://github.com/jpmml/jpmml-model)
817864
(BSD 3-clause style license) jblas (org.jblas:jblas:1.2.3 - http://jblas.org/)
818865
(BSD License) AntLR Parser Generator (antlr:antlr:2.7.7 - http://www.antlr.org/)
819866
(BSD License) Javolution (javolution:javolution:5.5.1 - http://javolution.org)

‎R/pkg/DESCRIPTION

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,21 @@ Suggests:
1515
Description: R frontend for Spark
1616
License: Apache License (== 2.0)
1717
Collate:
18+
'schema.R'
1819
'generics.R'
1920
'jobj.R'
20-
'SQLTypes.R'
2121
'RDD.R'
2222
'pairRDD.R'
2323
'column.R'
2424
'group.R'
2525
'DataFrame.R'
2626
'SQLContext.R'
27+
'backend.R'
2728
'broadcast.R'
29+
'client.R'
2830
'context.R'
2931
'deserialize.R'
3032
'serialize.R'
3133
'sparkR.R'
32-
'backend.R'
33-
'client.R'
3434
'utils.R'
3535
'zzz.R'

‎R/pkg/NAMESPACE

Lines changed: 57 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,44 @@
1-
#exportPattern("^[[:alpha:]]+")
2-
exportClasses("RDD")
3-
exportClasses("Broadcast")
4-
exportMethods(
5-
"aggregateByKey",
6-
"aggregateRDD",
7-
"cache",
8-
"checkpoint",
9-
"coalesce",
10-
"cogroup",
11-
"collect",
12-
"collectAsMap",
13-
"collectPartition",
14-
"combineByKey",
15-
"count",
16-
"countByKey",
17-
"countByValue",
18-
"distinct",
19-
"Filter",
20-
"filterRDD",
21-
"first",
22-
"flatMap",
23-
"flatMapValues",
24-
"fold",
25-
"foldByKey",
26-
"foreach",
27-
"foreachPartition",
28-
"fullOuterJoin",
29-
"glom",
30-
"groupByKey",
31-
"join",
32-
"keyBy",
33-
"keys",
34-
"length",
35-
"lapply",
36-
"lapplyPartition",
37-
"lapplyPartitionsWithIndex",
38-
"leftOuterJoin",
39-
"lookup",
40-
"map",
41-
"mapPartitions",
42-
"mapPartitionsWithIndex",
43-
"mapValues",
44-
"maximum",
45-
"minimum",
46-
"numPartitions",
47-
"partitionBy",
48-
"persist",
49-
"pipeRDD",
50-
"reduce",
51-
"reduceByKey",
52-
"reduceByKeyLocally",
53-
"repartition",
54-
"rightOuterJoin",
55-
"sampleRDD",
56-
"saveAsTextFile",
57-
"saveAsObjectFile",
58-
"sortBy",
59-
"sortByKey",
60-
"sumRDD",
61-
"take",
62-
"takeOrdered",
63-
"takeSample",
64-
"top",
65-
"unionRDD",
66-
"unpersist",
67-
"value",
68-
"values",
69-
"zipRDD",
70-
"zipWithIndex",
71-
"zipWithUniqueId"
72-
)
1+
# Imports from base R
2+
importFrom(methods, setGeneric, setMethod, setOldClass)
3+
useDynLib(SparkR, stringHashCode)
734

745
# S3 methods exported
75-
export(
76-
"textFile",
77-
"objectFile",
78-
"parallelize",
79-
"hashCode",
80-
"includePackage",
81-
"broadcast",
82-
"setBroadcastValue",
83-
"setCheckpointDir"
84-
)
856
export("sparkR.init")
867
export("sparkR.stop")
878
export("print.jobj")
88-
useDynLib(SparkR, stringHashCode)
89-
importFrom(methods, setGeneric, setMethod, setOldClass)
90-
91-
# SparkRSQL
929

9310
exportClasses("DataFrame")
9411

95-
exportMethods("columns",
12+
exportMethods("arrange",
13+
"cache",
14+
"collect",
15+
"columns",
16+
"count",
17+
"describe",
9618
"distinct",
9719
"dtypes",
20+
"except",
9821
"explain",
9922
"filter",
23+
"first",
24+
"group_by",
10025
"groupBy",
10126
"head",
10227
"insertInto",
10328
"intersect",
10429
"isLocal",
30+
"join",
10531
"limit",
10632
"orderBy",
33+
"mutate",
10734
"names",
35+
"persist",
10836
"printSchema",
10937
"registerTempTable",
38+
"rename",
11039
"repartition",
111-
"sampleDF",
40+
"sample",
41+
"sample_frac",
11242
"saveAsParquetFile",
11343
"saveAsTable",
11444
"saveDF",
@@ -117,43 +47,68 @@ exportMethods("columns",
11747
"selectExpr",
11848
"show",
11949
"showDF",
120-
"sortDF",
121-
"subtract",
122-
"toJSON",
123-
"toRDD",
50+
"summarize",
51+
"take",
12452
"unionAll",
53+
"unpersist",
12554
"where",
12655
"withColumn",
127-
"withColumnRenamed")
56+
"withColumnRenamed",
57+
"write.df")
12858

12959
exportClasses("Column")
13060

13161
exportMethods("abs",
62+
"acos",
13263
"alias",
13364
"approxCountDistinct",
13465
"asc",
66+
"asin",
67+
"atan",
68+
"atan2",
13569
"avg",
13670
"cast",
71+
"cbrt",
72+
"ceiling",
13773
"contains",
74+
"cos",
75+
"cosh",
13876
"countDistinct",
13977
"desc",
14078
"endsWith",
79+
"exp",
80+
"expm1",
81+
"floor",
14182
"getField",
14283
"getItem",
84+
"hypot",
14385
"isNotNull",
14486
"isNull",
14587
"last",
14688
"like",
89+
"log",
90+
"log10",
91+
"log1p",
14792
"lower",
14893
"max",
14994
"mean",
15095
"min",
96+
"n",
97+
"n_distinct",
98+
"rint",
15199
"rlike",
100+
"sign",
101+
"sin",
102+
"sinh",
152103
"sqrt",
153104
"startsWith",
154105
"substr",
155106
"sum",
156107
"sumDistinct",
108+
"tan",
109+
"tanh",
110+
"toDegrees",
111+
"toRadians",
157112
"upper")
158113

159114
exportClasses("GroupedData")
@@ -168,15 +123,20 @@ export("cacheTable",
168123
"createExternalTable",
169124
"dropTempTable",
170125
"jsonFile",
171-
"jsonRDD",
172126
"loadDF",
173127
"parquetFile",
128+
"read.df",
174129
"sql",
175130
"table",
176131
"tableNames",
177132
"tables",
178-
"toDF",
179133
"uncacheTable")
180134

181-
export("print.structType",
182-
"print.structField")
135+
export("structField",
136+
"structField.jobj",
137+
"structField.character",
138+
"print.structField",
139+
"structType",
140+
"structType.jobj",
141+
"structType.structField",
142+
"print.structType")

0 commit comments

Comments
 (0)