Update expected files for all tests in yml files

CHANGELOG.md

@@ -96,6 +96,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#644](https://github.com/nf-core/sarek/pull/644) - Use `-Y` for `bwa-mem(2)` and remove `-M`
 - [#645](https://github.com/nf-core/sarek/pull/645) - Merge `tests/nextflow.config` in `conf/test.config`
 - [#646](https://github.com/nf-core/sarek/pull/646) - Update `nextflow_schema.json` to reflect new parameters and functions, removes `--annotation_cache`, removes `--ascat_chromosomes`
+- [#649](https://github.com/nf-core/sarek/pull/649) - Update, simplify and add more files to all `test_*.yml` files
 - [#651](https://github.com/nf-core/sarek/pull/651) - Added TIDDIT_SOMATIC subworkflow
 - [#653](https://github.com/nf-core/sarek/pull/653) - Coherent results subfolder structure between preprocessing, variantcalling and reporting
 - [#659](https://github.com/nf-core/sarek/pull/659) - Update usage.md docu section on `How to run ASCAT with WES`

conf/modules.config

@@ -181,7 +181,7 @@ process {
     }
 
     withName: 'TABIX_BGZIPTABIX_INTERVAL_SPLIT' {
-        ext.prefix       = {"${meta.id}.bed"}
+        ext.prefix       = {"${meta.id}"}
         publishDir       = [
             enabled: params.save_reference,
             mode: params.publish_dir_mode,
@@ -446,7 +446,7 @@ process {
     }
 
 
-    if (params.step == 'mapping' || params.step == 'markduplicates'|| params.step == 'prepare_recalibration'|| params.step == 'recalibrate') {
+    if ((params.step == 'mapping' || params.step == 'markduplicates'|| params.step == 'prepare_recalibration'|| params.step == 'recalibrate') && (!(params.skip_tools && params.skip_tools.split(',').contains('baserecalibrator')))) {
         withName: 'NFCORE_SAREK:SAREK:(RECALIBRATE|RECALIBRATE_SPARK):MERGE_INDEX_CRAM:MERGE_CRAM' {
             ext.prefix       = { "${meta.id}.recal" }
             ext.when         = { meta.num_intervals > 1 }
@@ -529,7 +529,7 @@ process {
         ]
     }
 
-    if (params.step == 'mapping' || params.step == 'markduplicates'|| params.step == 'prepare_recalibration'|| params.step == 'recalibrate') {
+    if ((params.step == 'mapping' || params.step == 'markduplicates'|| params.step == 'prepare_recalibration'|| params.step == 'recalibrate') && (!(params.skip_tools && params.skip_tools.split(',').contains('baserecalibrator')))) {
         withName: 'NFCORE_SAREK:SAREK:CRAM_QC:MOSDEPTH' {
             ext.prefix       = { "${meta.id}.recal" }
         }

conf/test.config

@@ -16,7 +16,6 @@ try {
 }
 
 params {
-
     config_profile_name        = 'Test profile'
     config_profile_description = 'Minimal test dataset to check pipeline function'
 

subworkflows/local/prepare_recalibration_csv.nf

@@ -5,19 +5,35 @@
 workflow PREPARE_RECALIBRATION_CSV {
     take:
         cram_table_bqsr // channel: [mandatory] meta, cram, crai, table
+        skip_tools
 
     main:
         // Creating csv files to restart from this step
-        cram_table_bqsr.collectFile(keepHeader: true, skip: 1, sort: true, storeDir: "${params.outdir}/csv") { meta, cram, crai, table ->
-            patient = meta.patient
-            sample  = meta.sample
-            sex     = meta.sex
-            status  = meta.status
-            suffix_aligned = params.save_output_as_bam ? "bam" : "cram"
-            suffix_index   = params.save_output_as_bam ? "bam.bai" : "cram.crai"
-            cram = "${params.outdir}/preprocessing/${sample}/markduplicates/${cram.baseName}.${suffix_aligned}"
-            crai = "${params.outdir}/preprocessing/${sample}/markduplicates/${crai.baseName.minus(".cram")}.${suffix_index}"
-            table = "${params.outdir}/preprocessing/${sample}/recal_table/${sample}.recal.table"
-            ["markduplicates.csv", "patient,sex,status,sample,cram,crai,table\n${patient},${sex},${status},${sample},${cram},${crai},${table}\n"]
+        if (!(skip_tools && (skip_tools.split(',').contains('markduplicates')))) {
+            cram_table_bqsr.collectFile(keepHeader: true, skip: 1, sort: true, storeDir: "${params.outdir}/csv") { meta, cram, crai, table ->
+                patient = meta.patient
+                sample  = meta.sample
+                sex     = meta.sex
+                status  = meta.status
+                suffix_aligned = params.save_output_as_bam ? "bam" : "cram"
+                suffix_index   = params.save_output_as_bam ? "bam.bai" : "cram.crai"
+                cram = "${params.outdir}/preprocessing/${sample}/markduplicates/${cram.baseName}.${suffix_aligned}"
+                crai = "${params.outdir}/preprocessing/${sample}/markduplicates/${crai.baseName.minus(".cram")}.${suffix_index}"
+                table = "${params.outdir}/preprocessing/${sample}/recal_table/${sample}.recal.table"
+                ["markduplicates.csv", "patient,sex,status,sample,cram,crai,table\n${patient},${sex},${status},${sample},${cram},${crai},${table}\n"]
+            }
+        } else {
+            cram_table_bqsr.collectFile(keepHeader: true, skip: 1, sort: true, storeDir: "${params.outdir}/csv") { meta, cram, crai, table ->
+                patient = meta.patient
+                sample  = meta.sample
+                sex     = meta.sex
+                status  = meta.status
+                suffix_aligned = params.save_output_as_bam ? "bam" : "cram"
+                suffix_index   = params.save_output_as_bam ? "bam.bai" : "cram.crai"
+                cram = "${params.outdir}/preprocessing/${sample}/mapped/${cram.baseName}.${suffix_aligned}"
+                crai = "${params.outdir}/preprocessing/${sample}/mapped/${crai.baseName.minus(".cram")}.${suffix_index}"
+                table = "${params.outdir}/preprocessing/${sample}/recal_table/${sample}.recal.table"
+                ["sorted.csv", "patient,sex,status,sample,cram,crai,table\n${patient},${sex},${status},${sample},${cram},${crai},${table}\n"]
+            }
         }
 }

tests/csv/3.0/mapped_single_bam.csv

@@ -1,3 +1,2 @@
 patient,status,sample,bam,bai
 test,0,test,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai
-test1,0,test1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai

tests/test_aligner.yml

@@ -5,10 +5,10 @@
     - bwa-mem2
     - preprocessing
   files:
-    - path: results/multiqc
     - path: results/csv/markduplicates.csv
     - path: results/csv/markduplicates_no_table.csv
     - path: results/csv/recalibrated.csv
+    - path: results/multiqc
     - path: results/preprocessing/markduplicates/test/test.md.cram
     - path: results/preprocessing/markduplicates/test/test.md.cram.crai
     - path: results/preprocessing/recal_table/test/test.recal.table
@@ -19,15 +19,25 @@
     - path: results/reference/bwamem2/genome.fasta.ann
     - path: results/reference/bwamem2/genome.fasta.bwt.2bit.64
     - path: results/reference/bwamem2/genome.fasta.pac
+    - path: results/reference/dbsnp/dbsnp_146.hg38.vcf.gz.tbi
+    - path: results/reference/dict/genome.dict
+    - path: results/reference/fai/genome.fasta.fai
+    - path: results/reference/intervals/chr22_1-40001.bed
+    - path: results/reference/intervals/chr22_1-40001.bed.gz
+    - path: results/reference/intervals/genome.bed
+    - path: results/reference/known_indels/mills_and_1000G.indels.vcf.gz.tbi
     - path: results/reports/fastqc/test-test_L1
+    - path: results/reports/markduplicates/test/test.md.metrics
     - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt
-    - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.md.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi
     - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
-    - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
     - path: results/reports/samtools/test/test.md.cram.stats
     - path: results/reports/samtools/test/test.recal.cram.stats
 - name: Run dragmap
@@ -37,15 +47,17 @@
     - dragmap
     - preprocessing
   files:
-    - path: results/multiqc
     - path: results/csv/markduplicates.csv
     - path: results/csv/markduplicates_no_table.csv
     - path: results/csv/recalibrated.csv
+    - path: results/multiqc
     - path: results/preprocessing/markduplicates/test/test.md.cram
     - path: results/preprocessing/markduplicates/test/test.md.cram.crai
     - path: results/preprocessing/recal_table/test/test.recal.table
     - path: results/preprocessing/recalibrated/test/test.recal.cram
     - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
+    - path: results/reference/dbsnp/dbsnp_146.hg38.vcf.gz.tbi
+    - path: results/reference/dict/genome.dict
     - path: results/reference/dragmap/hash_table.cfg
     - path: results/reference/dragmap/hash_table.cfg.bin
     - path: results/reference/dragmap/hash_table.cmp
@@ -54,14 +66,22 @@
     - path: results/reference/dragmap/reference.bin
     - path: results/reference/dragmap/repeat_mask.bin
     - path: results/reference/dragmap/str_table.bin
+    - path: results/reference/fai/genome.fasta.fai
+    - path: results/reference/intervals/chr22_1-40001.bed
+    - path: results/reference/intervals/chr22_1-40001.bed.gz
+    - path: results/reference/intervals/genome.bed
+    - path: results/reference/known_indels/mills_and_1000G.indels.vcf.gz.tbi
     - path: results/reports/fastqc/test-test_L1
+    - path: results/reports/markduplicates/test/test.md.metrics
     - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt
-    - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.md.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi
     - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
-    - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
     - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
     - path: results/reports/samtools/test/test.md.cram.stats
     - path: results/reports/samtools/test/test.recal.cram.stats

tests/test_annotation.yml

@@ -6,10 +6,10 @@
   files:
     - path: results/annotation/test/test_snpEff.ann.vcf.gz
     - path: results/annotation/test/test_snpEff.ann.vcf.gz.tbi
-    - path: results/reports/snpeff/test/test_snpEff.csv
+    - path: results/multiqc
     - path: results/reports/snpeff/test/snpEff_summary.html
+    - path: results/reports/snpeff/test/test_snpEff.csv
     - path: results/reports/snpeff/test/test_snpEff.genes.txt
-    - path: results/multiqc
 - name: Run VEP
   command: nextflow run main.nf -profile test,annotation,docker --tools vep --skip_tools multiqc
   tags:
@@ -29,6 +29,42 @@
   files:
     - path: results/annotation/test/test_snpEff_VEP.ann.vcf.gz
     - path: results/annotation/test/test_snpEff_VEP.ann.vcf.gz.tbi
+    - path: results/reports/EnsemblVEP/test/test_snpEff_VEP.summary.html
+    - path: results/annotation/test/test_snpEff.ann.vcf.gz
+      should_exist: false
+    - path: results/annotation/test/test_snpEff.ann.vcf.gz.tbi
+      should_exist: false
+    - path: results/annotation/test/test_VEP.ann.vcf.gz
+      should_exist: false
+    - path: results/annotation/test/test_VEP.ann.vcf.gz.tbi
+      should_exist: false
+    - path: results/reports/snpeff/test/snpEff_summary.html
+      should_exist: false
+    - path: results/reports/snpeff/test/test_snpEff.csv
+      should_exist: false
+    - path: results/reports/snpeff/test/test_snpEff.genes.txt
+      should_exist: false
+    - path: results/reports/EnsemblVEP/test/test_VEP.summary.html
+      should_exist: false
+- name: Run snpEff, VEP and both consecutively
+  command: nextflow run main.nf -profile test,annotation,docker --tools merge,snpeff,vep --skip_tools multiqc
+  tags:
+    - annotation
+    - merge
+    - snpeff
+    - vep
+  files:
+    - path: results/annotation/test/test_VEP.ann.vcf.gz
+    - path: results/annotation/test/test_VEP.ann.vcf.gz.tbi
+    - path: results/annotation/test/test_snpEff.ann.vcf.gz
+    - path: results/annotation/test/test_snpEff.ann.vcf.gz.tbi
+    - path: results/annotation/test/test_snpEff_VEP.ann.vcf.gz
+    - path: results/annotation/test/test_snpEff_VEP.ann.vcf.gz.tbi
+    - path: results/reports/EnsemblVEP/test/test_VEP.summary.html
+    - path: results/reports/EnsemblVEP/test/test_snpEff_VEP.summary.html
+    - path: results/reports/snpeff/test/snpEff_summary.html
+    - path: results/reports/snpeff/test/test_snpEff.csv
+    - path: results/reports/snpeff/test/test_snpEff.genes.txt
 - name: Run VEP with fasta
   command: nextflow run main.nf -profile test,annotation,docker --tools vep --vep_include_fasta --skip_tools multiqc
   tags:

tests/test_bam_remap.yml

@@ -2,16 +2,42 @@
   command: nextflow run main.nf -profile test,alignment_to_fastq,docker
   tags:
     - alignment_to_fastq
-
   files:
     - path: results/cat/test-1_1.merged.fastq.gz
     - path: results/cat/test-1_2.merged.fastq.gz
+    - path: results/collate/test-1.mapped_1.fq.gz
+    - path: results/collate/test-1.mapped_2.fq.gz
+    - path: results/collate/test-1.mapped_other.fq.gz
+    - path: results/collate/test-1.mapped_singleton.fq.gz
+    - path: results/collate/test-1.unmapped_1.fq.gz
+    - path: results/collate/test-1.unmapped_2.fq.gz
+    - path: results/collate/test-1.unmapped_other.fq.gz
+    - path: results/collate/test-1.unmapped_singleton.fq.gz
     - path: results/csv/markduplicates.csv
     - path: results/csv/markduplicates_no_table.csv
     - path: results/csv/recalibrated.csv
-    - path: results/multiqc/multiqc_report.html
-    - path: results/pipeline_info
-    - path: results/preprocessing/
-    - path: results/reports
-    - path: results/samtools
-    - path: results/collate
+    - path: results/multiqc
+    - path: results/preprocessing/markduplicates/test/test.md.cram
+    - path: results/preprocessing/markduplicates/test/test.md.cram.crai
+    - path: results/preprocessing/recal_table/test/test.recal.table
+    - path: results/preprocessing/recalibrated/test/test.recal.cram
+    - path: results/preprocessing/recalibrated/test/test.recal.cram.crai
+    - path: results/reports/fastqc/test-1
+    - path: results/reports/markduplicates/test/test.md.metrics
+    - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt
+    - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
+    - path: results/reports/mosdepth/test/test.md.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi
+    - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
+    - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
+    - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
+    - path: results/reports/mosdepth/test/test.recal.regions.bed.gz
+    - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
+    - path: results/reports/samtools/test/test.md.cram.stats
+    - path: results/reports/samtools/test/test.recal.cram.stats
+    - path: results/samtools/test-1.bam
+    - path: results/samtools/test-1.map_map.bam
+    - path: results/samtools/test-1.map_unmap.bam
+    - path: results/samtools/test-1.unmap_map.bam
+    - path: results/samtools/test-1.unmap_unmap.bam