Skip to content

Commit 2bff86f

Browse files
HyukjinKwonHyukjinKwon
committed
[SPARK-32245][INFRA] Run Spark tests in Github Actions
This PR aims to run the Spark tests in Github Actions. To briefly explain the main idea: - Reuse `dev/run-tests.py` with SBT build - Reuse the modules in `dev/sparktestsupport/modules.py` to test each module - Pass the modules to test into `dev/run-tests.py` directly via `TEST_ONLY_MODULES` environment variable. For example, `pyspark-sql,core,sql,hive`. - `dev/run-tests.py` _does not_ take the dependent modules into account but solely the specified modules to test. Another thing to note might be `SlowHiveTest` annotation. Running the tests in Hive modules takes too much so the slow tests are extracted and it runs as a separate job. It was extracted from the actual elapsed time in Jenkins: ![Screen Shot 2020-07-09 at 7 48 13 PM](https://user-images.githubusercontent.com/6477701/87050238-f6098e80-c238-11ea-9c4a-ab505af61381.png) So, Hive tests are separated into to jobs. One is slow test cases, and the other one is the other test cases. _Note that_ the current GitHub Actions build virtually copies what the default PR builder on Jenkins does (without other profiles such as JDK 11, Hadoop 2, etc.). The only exception is Kinesis https://github.com/apache/spark/pull/29057/files#diff-04eb107ee163a50b61281ca08f4e4c7bR23 Last week and onwards, the Jenkins machines became very unstable for many reasons: - Apparently, the machines became extremely slow. Almost all tests can't pass. - One machine (worker 4) started to have the corrupt `.m2` which fails the build. - Documentation build fails time to time for an unknown reason in Jenkins machine specifically. This is disabled for now at apache#29017. - Almost all PRs are basically blocked by this instability currently. The advantages of using Github Actions: - To avoid depending on few persons who can access to the cluster. - To reduce the elapsed time in the build - we could split the tests (e.g., SQL, ML, CORE), and run them in parallel so the total build time will significantly reduce. - To control the environment more flexibly. - Other contributors can test and propose to fix Github Actions configurations so we can distribute this build management cost. Note that: - The current build in Jenkins takes _more than 7 hours_. With Github actions it takes _less than 2 hours_ - We can now control the environments especially for Python easily. - The test and build look more stable than the Jenkins'. No, dev-only change. Tested at #4 Closes apache#29057 from HyukjinKwon/migrate-to-github-actions. Authored-by: HyukjinKwon <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 6cdc32f commit 2bff86f

File tree

20 files changed

+397
-170
lines changed

20 files changed

+397
-170
lines changed

‎.github/workflows/master.yml

Lines changed: 177 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -1,156 +1,227 @@
11
name: master
22

33
on:
4-
push:
5-
branches:
6-
- branch-3.0
74
pull_request:
85
branches:
96
- branch-3.0
107

118
jobs:
9+
# TODO(SPARK-32248): Recover JDK 11 builds
10+
# Build: build Spark and run the tests for specified modules.
1211
build:
13-
12+
name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
1413
runs-on: ubuntu-latest
1514
strategy:
15+
fail-fast: false
1616
matrix:
17-
java: [ '1.8', '11' ]
18-
hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
19-
hive: [ 'hive-1.2', 'hive-2.3' ]
20-
exclude:
21-
- java: '11'
22-
hive: 'hive-1.2'
23-
- hadoop: 'hadoop-3.2'
24-
hive: 'hive-1.2'
25-
name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
26-
17+
java:
18+
- 1.8
19+
hadoop:
20+
- hadoop3.2
21+
hive:
22+
- hive2.3
23+
# TODO(SPARK-32246): We don't test 'streaming-kinesis-asl' for now.
24+
# Kinesis tests depends on external Amazon kinesis service.
25+
# Note that the modules below are from sparktestsupport/modules.py.
26+
modules:
27+
- |-
28+
core, unsafe, kvstore, avro,
29+
network_common, network_shuffle, repl, launcher
30+
examples, sketch, graphx
31+
- |-
32+
catalyst, hive-thriftserver
33+
- |-
34+
streaming, sql-kafka-0-10, streaming-kafka-0-10,
35+
mllib-local, mllib,
36+
yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
37+
- |-
38+
pyspark-sql, pyspark-mllib
39+
- |-
40+
pyspark-core, pyspark-streaming, pyspark-ml
41+
- |-
42+
sparkr
43+
# Here, we split Hive and SQL tests into some of slow ones and the rest of them.
44+
included-tags: [""]
45+
excluded-tags: [""]
46+
comment: [""]
47+
include:
48+
# Hive tests
49+
- modules: hive
50+
java: 1.8
51+
hadoop: hadoop3.2
52+
hive: hive2.3
53+
included-tags: org.apache.spark.tags.SlowHiveTest
54+
comment: "- slow tests"
55+
- modules: hive
56+
java: 1.8
57+
hadoop: hadoop3.2
58+
hive: hive2.3
59+
excluded-tags: org.apache.spark.tags.SlowHiveTest
60+
comment: "- other tests"
61+
# SQL tests
62+
- modules: sql
63+
java: 1.8
64+
hadoop: hadoop3.2
65+
hive: hive2.3
66+
included-tags: org.apache.spark.tags.ExtendedSQLTest
67+
comment: "- slow tests"
68+
- modules: sql
69+
java: 1.8
70+
hadoop: hadoop3.2
71+
hive: hive2.3
72+
excluded-tags: org.apache.spark.tags.ExtendedSQLTest
73+
comment: "- other tests"
74+
env:
75+
TEST_ONLY_MODULES: ${{ matrix.modules }}
76+
TEST_ONLY_EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
77+
TEST_ONLY_INCLUDED_TAGS: ${{ matrix.included-tags }}
78+
HADOOP_PROFILE: ${{ matrix.hadoop }}
79+
HIVE_PROFILE: ${{ matrix.hive }}
80+
# GitHub Actions' default miniconda to use in pip packaging test.
81+
CONDA_PREFIX: /usr/share/miniconda
2782
steps:
28-
- uses: actions/checkout@master
29-
# We split caches because GitHub Action Cache has a 400MB-size limit.
30-
- uses: actions/cache@v1
83+
- name: Checkout Spark repository
84+
uses: actions/checkout@v2
85+
# Cache local repositories. Note that GitHub Actions cache has a 2G limit.
86+
- name: Cache Scala, SBT, Maven and Zinc
87+
uses: actions/cache@v1
3188
with:
3289
path: build
3390
key: build-${{ hashFiles('**/pom.xml') }}
3491
restore-keys: |
3592
build-
36-
- uses: actions/cache@v1
37-
with:
38-
path: ~/.m2/repository/com
39-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
40-
restore-keys: |
41-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
42-
- uses: actions/cache@v1
93+
- name: Cache Maven local repository
94+
uses: actions/cache@v2
4395
with:
44-
path: ~/.m2/repository/org
45-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
46-
restore-keys: |
47-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
48-
- uses: actions/cache@v1
49-
with:
50-
path: ~/.m2/repository/net
51-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
96+
path: ~/.m2/repository
97+
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-${{ hashFiles('**/pom.xml') }}
5298
restore-keys: |
53-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
54-
- uses: actions/cache@v1
99+
${{ matrix.java }}-${{ matrix.hadoop }}-maven-
100+
- name: Cache Ivy local repository
101+
uses: actions/cache@v2
55102
with:
56-
path: ~/.m2/repository/io
57-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
103+
path: ~/.ivy2/cache
104+
key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/plugins.sbt') }}
58105
restore-keys: |
59-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
60-
- name: Set up JDK ${{ matrix.java }}
106+
${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
107+
- name: Install JDK ${{ matrix.java }}
61108
uses: actions/setup-java@v1
62109
with:
63110
java-version: ${{ matrix.java }}
64-
- name: Build with Maven
65-
run: |
66-
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
67-
export MAVEN_CLI_OPTS="--no-transfer-progress"
68-
mkdir -p ~/.m2
69-
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
70-
rm -rf ~/.m2/repository/org/apache/spark
71-
72-
73-
lint:
74-
runs-on: ubuntu-latest
75-
name: Linters (Java/Scala/Python), licenses, dependencies
76-
steps:
77-
- uses: actions/checkout@master
78-
- uses: actions/setup-java@v1
111+
# PySpark
112+
- name: Install PyPy3
113+
# SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
114+
# Note that order of Python installations here matters because default python3 is
115+
# overridden by pypy3.
116+
uses: actions/setup-python@v2
117+
if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
79118
with:
80-
java-version: '11'
81-
- uses: actions/setup-python@v1
119+
python-version: pypy3
120+
architecture: x64
121+
- name: Install Python 2.7
122+
uses: actions/setup-python@v2
123+
if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
82124
with:
83-
python-version: '3.x'
84-
architecture: 'x64'
85-
- name: Scala
86-
run: ./dev/lint-scala
87-
- name: Java
88-
run: ./dev/lint-java
89-
- name: Python
90-
run: |
91-
pip install flake8 sphinx numpy
92-
./dev/lint-python
93-
- name: License
94-
run: ./dev/check-license
95-
- name: Dependencies
96-
run: ./dev/test-dependencies.sh
97-
98-
lintr:
99-
runs-on: ubuntu-latest
100-
name: Linter (R)
101-
steps:
102-
- uses: actions/checkout@master
103-
- uses: actions/setup-java@v1
125+
python-version: 2.7
126+
architecture: x64
127+
- name: Install Python 3.6
128+
uses: actions/setup-python@v2
129+
if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
104130
with:
105-
java-version: '11'
106-
- uses: r-lib/actions/setup-r@v1
131+
python-version: 3.6
132+
architecture: x64
133+
- name: Install Python packages
134+
if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
135+
# PyArrow is not supported in PyPy yet, see ARROW-2651.
136+
# TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason.
137+
run: |
138+
python3 -m pip install numpy pyarrow pandas scipy
139+
python3 -m pip list
140+
python2 -m pip install numpy pyarrow pandas scipy
141+
python2 -m pip list
142+
pypy3 -m pip install numpy pandas
143+
pypy3 -m pip list
144+
# SparkR
145+
- name: Install R 3.6
146+
uses: r-lib/actions/setup-r@v1
147+
if: contains(matrix.modules, 'sparkr')
107148
with:
108-
r-version: '3.6.2'
109-
- name: Install lib
149+
r-version: 3.6
150+
- name: Install R packages
151+
if: contains(matrix.modules, 'sparkr')
110152
run: |
111153
sudo apt-get install -y libcurl4-openssl-dev
112-
- name: install R packages
154+
sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')"
155+
# Show installed packages in R.
156+
sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]'
157+
# Run the tests.
158+
- name: "Run tests: ${{ matrix.modules }}"
113159
run: |
114-
sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
115-
sudo Rscript -e "devtools::install_github('jimhester/[email protected]')"
116-
- name: package and install SparkR
117-
run: ./R/install-dev.sh
118-
- name: lint-r
119-
run: ./dev/lint-r
160+
# Hive tests become flaky when running in parallel as it's too intensive.
161+
if [[ "$TEST_ONLY_MODULES" == "hive" ]]; then export SERIAL_SBT_TESTS=1; fi
162+
mkdir -p ~/.m2
163+
./dev/run-tests --parallelism 2
164+
rm -rf ~/.m2/repository/org/apache/spark
120165
121-
docs:
166+
# Static analysis, and documentation build
167+
lint:
168+
name: Linters, licenses, dependencies and documentation generation
122169
runs-on: ubuntu-latest
123-
name: Generate documents
124170
steps:
125-
- uses: actions/checkout@master
126-
- uses: actions/cache@v1
171+
- name: Checkout Spark repository
172+
uses: actions/checkout@v2
173+
- name: Cache Maven local repository
174+
uses: actions/cache@v2
127175
with:
128176
path: ~/.m2/repository
129177
key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
130178
restore-keys: |
131-
docs-maven-repo-
132-
- uses: actions/setup-java@v1
179+
docs-maven-
180+
- name: Install JDK 1.8
181+
uses: actions/setup-java@v1
133182
with:
134-
java-version: '1.8'
135-
- uses: actions/setup-python@v1
183+
java-version: 1.8
184+
- name: Install Python 3.6
185+
uses: actions/setup-python@v2
136186
with:
137-
python-version: '3.x'
138-
architecture: 'x64'
139-
- uses: actions/setup-ruby@v1
187+
python-version: 3.6
188+
architecture: x64
189+
- name: Install Python linter dependencies
190+
run: |
191+
pip3 install flake8 sphinx numpy
192+
- name: Install R 3.6
193+
uses: r-lib/actions/setup-r@v1
140194
with:
141-
ruby-version: '2.7'
142-
- uses: r-lib/actions/setup-r@v1
195+
r-version: 3.6
196+
- name: Install R linter dependencies and SparkR
197+
run: |
198+
sudo apt-get install -y libcurl4-openssl-dev
199+
sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
200+
sudo Rscript -e "devtools::install_github('jimhester/[email protected]')"
201+
./R/install-dev.sh
202+
- name: Install Ruby 2.7 for documentation generation
203+
uses: actions/setup-ruby@v1
143204
with:
144-
r-version: '3.6.2'
145-
- name: Install lib and pandoc
205+
ruby-version: 2.7
206+
- name: Install dependencies for documentation generation
146207
run: |
147208
sudo apt-get install -y libcurl4-openssl-dev pandoc
148-
- name: Install packages
149-
run: |
150209
pip install sphinx mkdocs numpy
151210
gem install jekyll jekyll-redirect-from rouge
152-
sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
153-
- name: Run jekyll build
211+
sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
212+
- name: Scala linter
213+
run: ./dev/lint-scala
214+
- name: Java linter
215+
run: ./dev/lint-java
216+
- name: Python linter
217+
run: ./dev/lint-python
218+
- name: R linter
219+
run: ./dev/lint-r
220+
- name: License test
221+
run: ./dev/check-license
222+
- name: Dependencies test
223+
run: ./dev/test-dependencies.sh
224+
- name: Run documentation build
154225
run: |
155226
cd docs
156227
jekyll build

‎common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.tags;
19+
20+
import org.scalatest.TagAnnotation;
21+
22+
import java.lang.annotation.ElementType;
23+
import java.lang.annotation.Retention;
24+
import java.lang.annotation.RetentionPolicy;
25+
import java.lang.annotation.Target;
26+
27+
@TagAnnotation
28+
@Retention(RetentionPolicy.RUNTIME)
29+
@Target({ElementType.METHOD, ElementType.TYPE})
30+
public @interface SlowHiveTest { }

‎core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -685,7 +685,8 @@ class MasterSuite extends SparkFunSuite
685685
}
686686
}
687687

688-
test("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") {
688+
// TODO(SPARK-32250): Enable the test back. It is flaky in GitHub Actions.
689+
ignore("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") {
689690
val master = makeAliveMaster()
690691
var worker: MockExecutorLaunchFailWorker = null
691692
try {

0 commit comments

Comments
 (0)