Skip to content

Commit 40050f7

Browse files
maxulyssemaxulysse
authored
Merge pull request #806 from maxulysse/dev_more_complete_CI
More complete CI
2 parents 3393afe + 93f4db2 commit 40050f7

File tree

100 files changed

+5402
-4590
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+5402
-4590
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,12 @@ name: nf-core CI
22
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
33
on:
44
push:
5-
branches:
6-
- dev
5+
branches: [dev]
76
pull_request:
87
release:
98
types: [published]
109

11-
env:
12-
NXF_ANSI_LOG: false
13-
10+
# Cancel if a newer run is started
1411
concurrency:
1512
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
1613
cancel-in-progress: true
@@ -34,9 +31,11 @@ jobs:
3431
exclude:
3532
- NXF_VER: "latest-everything"
3633
profile: "conda"
34+
env:
35+
NXF_ANSI_LOG: false
3736
steps:
3837
- name: Check out pipeline code
39-
uses: actions/checkout@v2
38+
uses: actions/checkout@v3
4039

4140
- name: Set up Python
4241
uses: actions/setup-python@v2
@@ -107,88 +106,3 @@ jobs:
107106
/home/runner/pytest_workflow_*/*/work
108107
!/home/runner/pytest_workflow_*/*/work/conda
109108
!/home/runner/pytest_workflow_*/*/work/singularity
110-
111-
test_all:
112-
name: Run pipeline with test data (complete)
113-
# Only run on push if this is the nf-core dev branch (merged PRs)
114-
if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/sarek') }}"
115-
runs-on: ubuntu-latest
116-
strategy:
117-
# HACK Remove after DSL2 rewrite is done
118-
fail-fast: false
119-
matrix:
120-
NXF_VER:
121-
- "21.10.3"
122-
test:
123-
- "aligner"
124-
- "alignment_to_fastq"
125-
- "annotation"
126-
- "cnvkit"
127-
- "controlfreec"
128-
- "deepvariant"
129-
- "freebayes"
130-
- "gatk4_spark"
131-
- "haplotypecaller"
132-
- "intervals"
133-
- "manta"
134-
- "markduplicates"
135-
- "mpileup"
136-
- "msisensorpro"
137-
- "mutect2"
138-
- "prepare_recalibration"
139-
- "recalibrate"
140-
- "save_mapped"
141-
- "save_output_as_bam"
142-
- "skip_markduplicates"
143-
- "skip_qc"
144-
- "split_fastq"
145-
- "strelka"
146-
- "strelkabp"
147-
- "targeted"
148-
- "tiddit"
149-
- "trimming"
150-
- "tumor_normal_pair"
151-
- "umi"
152-
- "variantcalling_channel"
153-
profile: ["docker"]
154-
steps:
155-
- name: Check out pipeline code
156-
uses: actions/checkout@v2
157-
158-
- name: Install Nextflow
159-
uses: nf-core/setup-nextflow@v1
160-
with:
161-
version: "${{ matrix.NXF_VER }}"
162-
163-
- name: Set up Python
164-
uses: actions/setup-python@v2
165-
with:
166-
python-version: "3.x"
167-
168-
- name: Install dependencies
169-
run: python -m pip install --upgrade pip pytest-workflow
170-
171-
- name: Run pipeline with tests settings
172-
uses: Wandalen/wretry.action@v1.0.11
173-
with:
174-
command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes
175-
attempt_limit: 3
176-
177-
- name: Output log on failure
178-
if: failure()
179-
run: |
180-
sudo apt install bat > /dev/null
181-
batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
182-
183-
- name: Upload logs on failure
184-
if: failure()
185-
uses: actions/upload-artifact@v2
186-
with:
187-
name: logs-${{ matrix.profile }}
188-
path: |
189-
/home/runner/pytest_workflow_*/*/.nextflow.log
190-
/home/runner/pytest_workflow_*/*/log.out
191-
/home/runner/pytest_workflow_*/*/log.err
192-
/home/runner/pytest_workflow_*/*/work
193-
!/home/runner/pytest_workflow_*/*/work/conda
194-
!/home/runner/pytest_workflow_*/*/work/singularity

‎.github/workflows/pytest-workflow.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: pytest-workflow
2+
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
3+
on:
4+
pull_request:
5+
branches: [dev]
6+
7+
# Cancel if a newer run is started
8+
concurrency:
9+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
10+
cancel-in-progress: true
11+
12+
jobs:
13+
changes:
14+
name: Check for changes
15+
runs-on: ubuntu-latest
16+
outputs:
17+
# Expose matched filters as job 'tags' output variable
18+
tags: ${{ steps.filter.outputs.changes }}
19+
steps:
20+
- uses: actions/checkout@v3
21+
- uses: dorny/paths-filter@v2
22+
id: filter
23+
with:
24+
filters: "tests/config/pytest_tags.yml"
25+
26+
test:
27+
name: ${{ matrix.tags }} ${{ matrix.profile }}
28+
runs-on: ubuntu-latest
29+
needs: changes
30+
if: needs.changes.outputs.tags != '[]'
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
35+
profile: ["docker", "singularity", "conda"]
36+
NXF_VER:
37+
- "21.10.3"
38+
- "latest-everything"
39+
exclude:
40+
- profile: "conda"
41+
NXF_VER: "latest-everything"
42+
- profile: "conda"
43+
tags: umi
44+
- profile: "conda"
45+
tags: deepvariant
46+
- profile: "conda"
47+
tags: haplotypecaller
48+
- profile: "conda"
49+
tags: merge
50+
- profile: "conda"
51+
tags: snpeff
52+
- profile: "conda"
53+
tags: vep
54+
- profile: "singularity"
55+
tags: merge
56+
env:
57+
NXF_ANSI_LOG: false
58+
steps:
59+
- name: Check out pipeline code
60+
uses: actions/checkout@v3
61+
62+
- name: Set up Python
63+
uses: actions/setup-python@v2
64+
with:
65+
python-version: "3.x"
66+
67+
- uses: actions/cache@v2
68+
with:
69+
path: ~/.cache/pip
70+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
71+
restore-keys: |
72+
${{ runner.os }}-pip-
73+
74+
- name: Install Python dependencies
75+
run: python -m pip install --upgrade pip pytest-workflow
76+
77+
- uses: actions/cache@v2
78+
with:
79+
path: /usr/local/bin/nextflow
80+
key: ${{ runner.os }}
81+
restore-keys: |
82+
${{ runner.os }}-nextflow-
83+
84+
- name: Install Nextflow
85+
uses: nf-core/setup-nextflow@v1
86+
with:
87+
version: "${{ matrix.NXF_VER }}"
88+
89+
- name: Set up Singularity
90+
if: matrix.profile == 'singularity'
91+
uses: eWaterCycle/setup-singularity@v5
92+
with:
93+
singularity-version: 3.7.1
94+
95+
- name: Set up miniconda
96+
if: matrix.profile == 'conda'
97+
uses: conda-incubator/setup-miniconda@v2
98+
with:
99+
auto-update-conda: true
100+
channels: conda-forge,bioconda,defaults
101+
python-version: ${{ matrix.python-version }}
102+
103+
- name: Conda clean
104+
if: matrix.profile == 'conda'
105+
run: conda clean -a
106+
107+
- name: Run pytest-workflow
108+
uses: Wandalen/wretry.action@v1.0.11
109+
with:
110+
command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
111+
attempt_limit: 3
112+
113+
- name: Output log on failure
114+
if: failure()
115+
run: |
116+
sudo apt install bat > /dev/null
117+
batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
118+
119+
- name: Upload logs on failure
120+
if: failure()
121+
uses: actions/upload-artifact@v2
122+
with:
123+
name: logs-${{ matrix.profile }}
124+
path: |
125+
/home/runner/pytest_workflow_*/*/.nextflow.log
126+
/home/runner/pytest_workflow_*/*/log.out
127+
/home/runner/pytest_workflow_*/*/log.err
128+
/home/runner/pytest_workflow_*/*/work
129+
!/home/runner/pytest_workflow_*/*/work/conda
130+
!/home/runner/pytest_workflow_*/*/work/singularity

‎.nf-core.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
repository_type: pipeline
22
lint:
3+
files_exist:
4+
- conf/modules.config
5+
- conf/test.config
6+
- conf/test_full.config
37
files_unchanged:
48
- assets/multiqc_config.yml
59
- assets/nf-core-sarek_logo_light.png

0 commit comments

Comments
 (0)