Merge pull request #5 from nickhsmith/nf-core-dev

Nf core dev

Author: nickhsmith

.github/workflows/ci.yml

@@ -19,6 +19,8 @@ jobs:
     if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/sarek') }}
     runs-on: ubuntu-latest
     strategy:
+      # HACK Remove after DSL2 rewrite is done
+      fail-fast: false
       matrix:
         # Nextflow versions
         include:
@@ -34,11 +36,11 @@ jobs:
           - 'default'
           - 'split_fastq'
           - 'gatk4_spark'
-          - 'save_bam_mapped'
+          #- 'save_bam_mapped'
           - 'skip_markduplicates'
-          # - 'targeted'
+          - 'targeted'
           - 'tumor_normal_pair'
-          # - 'variant_calling'
+          - 'variant_calling'
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -62,3 +64,22 @@ jobs:
 
       - name: Run pipeline with tests settings
         run: pytest --tag ${{ matrix.test }} --kwdof
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.profile }}
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/work
+            !/home/runner/pytest_workflow_*/*/work/conda
+            !/home/runner/pytest_workflow_*/*/work/singularity

.gitpod.yml

@@ -0,0 +1,14 @@
+image: nfcore/gitpod:latest
+
+vscode:
+  extensions: # based on nf-core.nf-core-extensionpack
+    - codezombiech.gitignore                 # Language support for .gitignore files
+    # - cssho.vscode-svgviewer                 # SVG viewer
+    - davidanson.vscode-markdownlint         # Markdown/CommonMark linting and style checking for Visual Studio Code
+    - eamodio.gitlens                        # Quickly glimpse into whom, why, and when a line or code block was changed
+    - EditorConfig.EditorConfig              # override user/workspace settings with settings found in .editorconfig files
+    - Gruntfuggly.todo-tree                  # Display TODO and FIXME in a tree view in the activity bar
+    - mechatroner.rainbow-csv                # Highlight columns in csv files in different colors
+    # - nextflow.nextflow                      # Nextflow syntax highlighting
+    - oderwat.indent-rainbow                 # Highlight indentation level
+    - streetsidesoftware.code-spell-checker  # Spelling checker for source code

bin/concatenateVCFs.sh

@@ -58,16 +58,16 @@ then
 
     CONTIGS=($(cut -f1 ${genomeIndex}))
 
-    # Concatenate VCFs in the correct order
+    #Concatenate VCFs in the correct order
     (
         cat header
 
         for chr in "${CONTIGS[@]}"; do
             # Skip if globbing would not match any file to avoid errors such as
             # "ls: cannot access chr3_*.vcf.gz: No such file or directory" when chr3
             # was not processed.
-            pattern="*_${chr}_*.vcf.gz"
-            if ! compgen -G "${pattern}" > /dev/null; then continue; fi
+            pattern="*_${chr}_*.vcf"
+            if ! compgen -G "${pattern}" > /dev/null ; then continue; fi
 
             # ls -v sorts by numeric value ("version"), which means that chr1_100_
             # is sorted *after* chr1_99_.
@@ -83,14 +83,16 @@ then
                 tail -n +$((L+1)) <(zcat ${vcf})
             done
         done
-    ) | bgzip -@${cpus} > rawcalls.vcf.gz
-    tabix rawcalls.vcf.gz
+    ) | bgzip -@${cpus} > rawcalls.unsorted.vcf.gz
 else
-    VCF=$(ls no_intervals*.vcf.gz)
-    mv -v $VCF rawcalls.vcf.gz
-    tabix rawcalls.vcf.gz
+    VCF=$(ls no_intervals*.vcf)
+    cp $VCF rawcalls.unsorted.vcf
+    bgzip -@${cpus} rawcalls.unsorted.vcf
 fi
 
+bcftools sort rawcalls.unsorted.vcf.gz | bgzip > rawcalls.vcf.gz
+tabix -p vcf rawcalls.vcf.gz
+
 set +u
 
 # Now we have the concatenated VCF file, check for WES/panel targets, and generate a subset if there is a BED provided
@@ -100,5 +102,5 @@ if [ ! -z ${targetBED+x} ]; then
     tabix ${outputFile}.gz
 else
     # Rename the raw calls as WGS results
-    for f in rawcalls*; do mv -v $f ${outputFile}${f#rawcalls.vcf}; done
+    for f in rawcalls.vcf*; do mv -v $f ${outputFile}${f#rawcalls.vcf}; done
 fi

conf/genomes.config

@@ -29,15 +29,6 @@ params {
             fasta                 = "${params.genomes_base}/human_g1k_v37_decoy.small.fasta"
             known_indels          = "${params.genomes_base}/dbsnp_138.b37.small.vcf.gz"
         }
-        'smallGRCh38' {
-            dbsnp                 = "${params.genomes_base}/dbsnp_146_hg38_chr20_tso-only.vcf.gz"
-            fasta                 = "${params.genomes_base}/Homo_sapiens_assembly38_chr20.fasta"
-            known_indels          = "${params.genomes_base}/Mills_and_1000G_gold_standard_indels_hg38_chr20.vcf.gz"
-            snpeff_db             = 'GRCh38.99'
-            vep_genome            = 'GRCh38'
-            vep_species           = 'homo_sapiens'
-            vep_cache_version     = '99'
-        }
         'custom' {
             fasta                 = null
         }

conf/test.config

@@ -18,7 +18,7 @@ params {
     // Limit resources so that this can run on GitHub Actions
     max_cpus   = 2
     max_memory = '6.GB'
-    max_time   = '6.h'
+    max_time   = '8.h'
 
     // Input data
     input = "${baseDir}/tests/csv/3.0/fastq_single.csv"
@@ -28,11 +28,12 @@ params {
     genome            = 'small_hg38'
     genomes_base      = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules'
 
-    dbsnp             = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz"
-    fasta             = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.fasta"
-    germline_resource = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz"
-    intervals         = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.interval_list"
-    known_indels      = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz"
+    dbsnp                   = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz"
+    fasta                   = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.fasta"
+    germline_resource       = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz"
+    intervals               = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.interval_list"
+    known_indels            = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz"
+    nucleotides_per_second  = 20
 
     snpeff_db         = 'WBcel235.99'
     vep_species       = 'caenorhabditis_elegans'
@@ -53,7 +54,7 @@ profiles {
         params.input               = "${baseDir}/tests/csv/3.0/fastq_pair.csv"
     }
     prepare_recalibration {
-        params.input               = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-mapped-normal-https.csv'
+        params.input               = "${baseDir}/tests/csv/3.0/mapped_single.csv"
         params.step                = 'prepare_recalibration'
     }
     save_bam_mapped {
@@ -65,19 +66,28 @@ profiles {
     split_fastq {
         params.split_fastq         = 150000
         params.save_split_fastqs   = true
-
     }
-    targeted {
-        params.target_bed          = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/target.bed'
-        params.tools               = 'manta,strelka'
+    no_intervals {
+        params.no_intervals          = true
     }
-    tool {
-        params.input               = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-recal-normal-https.csv'
-        params.step                = 'variant_calling'
+    targeted {
+        params.intervals           = "${params.genomes_base}/data/genomics/homo_sapiens/genome/multi_intervals.bed"
+        params.wes                 = true
     }
-    tool_pair {
-        params.input               = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-recal-pair-https.csv'
+    tools {
+        params.input               = "${baseDir}/tests/csv/3.0/recalibrated.csv"
+        params.dbsnp               = "${params.genomes_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbsnp_138.hg38.vcf.gz"
+        params.fasta               = "${params.genomes_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta"
+        params.germline_resource   = "${params.genomes_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/gnomAD.r2.1.1.vcf.gz"
+        params.intervals           = "${params.genomes_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed"
+        params.pon                 = "${params.genomes_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/mills_and_1000G.indels.hg38.vcf.gz"
         params.step                = 'variant_calling'
+        params.tools               = 'deepvariant,freebayes,haplotypecaller,manta,msisensorpro,mutect2,strelka,snpeff,vep' //tiddit
+        params.joint_germline      = true
+        params.wes                 = true
+        params.genome              = 'WBcel235'
+        params.vep_genome          = 'WBcel235'
+        //params.vep_cache           =
     }
     trimming {
         params.clip_r1             = 1
@@ -89,21 +99,9 @@ profiles {
     use_gatk_spark {
         params.use_gatk_spark      = 'bqsr,markduplicates'
     }
-    umi_quiaseq {
-        params.genome              = 'smallGRCh38'
-        params.genomes_base        = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/reference/chr20_hg38'
-        params.input               = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-umi-qiaseq-https.csv'
-        params.read_structure1     = '12M11S+T'
-        params.read_structure2     = '12M11S+T'
-        params.umi                 = true
-    }
-    umi_tso {
-        genome                     = 'smallGRCh38'
-        genomes_base               = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/reference/chr20_hg38'
-        input                      = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-umi-tso-https.csv'
-        read_structure1            = '7M1S+T'
-        read_structure2            = '7M1S+T'
-        umi                        = true
+    umi {
+        params.input               = "${baseDir}/tests/csv/3.0/fastq_umi.csv"
+        params.umi_read_structure  = '7M1S+T'
     }
 }