Skip to content

Commit 969993e

Browse files
maxulyssemaxulysse
authored
Merge pull request #1288 from adamrtalbot/nf-test-ci
Add an nf-test CI
2 parents c9bff2e + 9f7d685 commit 969993e

File tree

7 files changed

+181
-13
lines changed

7 files changed

+181
-13
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 131 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,34 +11,40 @@ on:
1111
- master
1212
- dev
1313

14+
env:
15+
NXF_ANSI_LOG: false
16+
NFTEST_VER: "0.8.1"
17+
1418
# Cancel if a newer run is started
1519
concurrency:
1620
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
1721
cancel-in-progress: true
1822

1923
jobs:
20-
changes:
21-
name: Check for changes
24+
pytest-changes:
25+
name: Check for changes (pytest)
2226
runs-on: ubuntu-latest
2327
outputs:
2428
# Expose matched filters as job 'tags' output variable
2529
tags: ${{ steps.filter.outputs.changes }}
2630
steps:
2731
- uses: actions/checkout@v3
28-
- uses: dorny/paths-filter@v2
32+
33+
- uses: frouioui/paths-filter@main
2934
id: filter
3035
with:
3136
filters: "tests/config/tags.yml"
37+
token: ""
3238

33-
test:
39+
pytest:
3440
name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
3541
runs-on: ubuntu-latest
36-
needs: changes
37-
if: needs.changes.outputs.tags != '[]'
42+
needs: pytest-changes
43+
if: needs.pytest-changes.outputs.tags != '[]'
3844
strategy:
3945
fail-fast: false
4046
matrix:
41-
tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
47+
tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"]
4248
profile: ["docker"]
4349
# profile: ["docker", "singularity", "conda"]
4450
TEST_DATA_BASE:
@@ -50,6 +56,7 @@ jobs:
5056
NXF_ANSI_LOG: false
5157
TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
5258
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
59+
5360
steps:
5461
- name: Check out pipeline code
5562
uses: actions/checkout@v3
@@ -154,10 +161,126 @@ jobs:
154161
!/home/runner/pytest_workflow_*/*/work/conda
155162
!/home/runner/pytest_workflow_*/*/work/singularity
156163
164+
nftest-changes:
165+
name: Check for changes (nf-test)
166+
runs-on: ubuntu-latest
167+
outputs:
168+
tags: ${{ steps.filter.outputs.changes }}
169+
170+
steps:
171+
- uses: actions/checkout@v3
172+
173+
- name: Combine all tags.yml files
174+
id: get_tags
175+
run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
176+
177+
- name: debug
178+
run: cat .github/tags.yml
179+
180+
- uses: frouioui/paths-filter@main
181+
id: filter
182+
with:
183+
filters: ".github/tags.yml"
184+
token: ""
185+
186+
nftest:
187+
name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
188+
runs-on: ubuntu-latest
189+
needs: nftest-changes
190+
if: needs.nftest-changes.outputs.tags != '[]'
191+
strategy:
192+
fail-fast: false
193+
matrix:
194+
tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"]
195+
profile: ["docker"]
196+
# profile: ["docker", "singularity", "conda"]
197+
TEST_DATA_BASE:
198+
- "test-datasets/data"
199+
NXF_VER:
200+
- "23.04.0"
201+
- "latest-everything"
202+
env:
203+
NXF_ANSI_LOG: false
204+
TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
205+
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
206+
207+
steps:
208+
- uses: actions/checkout@v3
209+
210+
- uses: actions/setup-java@v3
211+
with:
212+
distribution: "temurin"
213+
java-version: "17"
214+
- name: Setup Nextflow
215+
uses: nf-core/setup-nextflow@v1
216+
217+
- name: Cache nf-test installation
218+
id: cache-software
219+
uses: actions/cache@v3
220+
with:
221+
path: |
222+
/usr/local/bin/nf-test
223+
/home/runner/.nf-test/nf-test.jar
224+
key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
225+
226+
- name: Install nf-test
227+
if: steps.cache-software.outputs.cache-hit != 'true'
228+
run: |
229+
wget -qO- https://code.askimed.com/install/nf-test | bash
230+
sudo mv nf-test /usr/local/bin/
231+
232+
- name: Setup apptainer
233+
if: matrix.profile == 'singularity'
234+
uses: eWaterCycle/setup-apptainer@main
235+
236+
- name: Set up Singularity
237+
if: matrix.profile == 'singularity'
238+
run: |
239+
mkdir -p $NXF_SINGULARITY_CACHEDIR
240+
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
241+
242+
- name: Set up miniconda
243+
uses: conda-incubator/setup-miniconda@v2
244+
with:
245+
miniconda-version: "latest"
246+
auto-update-conda: true
247+
channels: conda-forge,bioconda,defaults
248+
python-version: ${{ matrix.python-version }}
249+
250+
- name: Conda setup
251+
run: |
252+
conda clean -a
253+
conda install -n base conda-libmamba-solver
254+
conda config --set solver libmamba
255+
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
256+
echo $(realpath python) >> $GITHUB_PATH
257+
258+
# Set up secrets
259+
- name: Set up nextflow secrets
260+
if: env.SENTIEON_LICENSE_BASE64 != null
261+
run: |
262+
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
263+
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
264+
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
265+
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
266+
SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
267+
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
268+
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
269+
270+
# Test the module
271+
- name: Run nf-test
272+
run: |
273+
nf-test test \
274+
--profile=${{ matrix.profile }} \
275+
--tag ${{ matrix.tags }} \
276+
--tap=test.tap \
277+
--verbose
278+
157279
confirm-pass:
158280
runs-on: ubuntu-latest
159281
needs:
160-
- test
282+
- pytest
283+
- nftest
161284
if: always()
162285
steps:
163286
- name: All tests ok

‎.gitignore‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
*.code-workspace
2+
*.pyc
23
.nextflow*
3-
work/
4-
data/
5-
results/
64
.DS_Store
7-
testing/
5+
.nf-test.log
6+
.nf-test/
7+
data/
88
testing*
9-
*.pyc
9+
testing/
1010
test-datasets/
11+
results/
12+
work/

‎.nf-core.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ lint:
55
- .github/workflows/awsfulltest.yml
66
- conf/modules.config
77
files_unchanged:
8+
- .gitignore
89
- assets/nf-core-sarek_logo_light.png
910
- docs/images/nf-core-sarek_logo_dark.png
1011
- docs/images/nf-core-sarek_logo_light.png

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ Rapaselet is a delta formed by the Rapaätno river between the Bielloriehppe mas
9494
- [#1173](https://github.com/nf-core/sarek/pull/1173) - CI tests for VQSR track with stub runs
9595
- [#1122](https://github.com/nf-core/sarek/pull/1122), [#1196](https://github.com/nf-core/sarek/pull/1196) - Add `annotation cache` functionality
9696
- [#1184](https://github.com/nf-core/sarek/pull/1184) - Stub-based CI-test of Sentieon joint-germline variant-calling with VQSR
97+
- [#1288](https://github.com/nf-core/sarek/pull/1288) - Add nf-test continuous integration.
9798

9899
### Changed
99100

‎nf-test.config‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
config {
2+
3+
testsDir "tests"
4+
workDir ".nf-test"
5+
configFile "conf/test.config"
6+
profile "test"
7+
8+
}

‎tests/main.nf.test‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
nextflow_pipeline {
2+
3+
name "Test pipeline"
4+
script "../main.nf"
5+
tag "pipeline"
6+
tag "pipeline_sarek"
7+
8+
test("Run with profile test") {
9+
10+
when {
11+
params {
12+
outdir = "results"
13+
max_cpus = 2
14+
max_memory = '6.GB'
15+
max_time = '6.h'
16+
input = "$projectDir/tests/csv/3.0/fastq_pair.csv"
17+
validationSchemaIgnoreParams = 'test_data_base,test_data'
18+
}
19+
}
20+
21+
then {
22+
assertAll(
23+
{ assert workflow.success }
24+
)
25+
}
26+
}
27+
28+
}

‎tests/tags.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pipeline_sarek:
2+
- "**.nf"
3+
- "**.config"
4+
- "**.nf.test"
5+
- "**.json"

0 commit comments

Comments
 (0)