Merge pull request #585 from FriederikeHanssen/fix_spark_for_the_last_time

update spark modules

Author: maxulysse

CHANGELOG.md

@@ -53,11 +53,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#539](https://github.com/nf-core/sarek/pull/539) - Update `CITATIONS.md`
 - [#544](https://github.com/nf-core/sarek/pull/544) - `Mutect2` is no longer compatible with `--no_intervals`
 - [#551](https://github.com/nf-core/sarek/pull/551) - Sync `TEMPLATE` with `tools` `2.4`
+- [#562](https://github.com/nf-core/sarek/pull/562) - Restart from `--step annotate` is now also requiring a CSV file.
 - [#563](https://github.com/nf-core/sarek/pull/563) - Updated subway map
 - [#571](https://github.com/nf-core/sarek/pull/571) - Including and using GATK4's mergeVcfs
 - [#572](https://github.com/nf-core/sarek/pull/572) - Adjusted subway map svg for firefox compatibility
 - [#578](https://github.com/nf-core/sarek/pull/578) - Updated module deeptools/bamcoverage
-- [#562](https://github.com/nf-core/sarek/pull/562) - Restart from `--step annotate` is now also requiring a CSV file.
+- [#585](https://github.com/nf-core/sarek/pull/585) - Remove explicit BAM to CRAM conversion after MarkduplicatesSpark; tool does it internally
 
 ### Fixed
 
@@ -83,6 +84,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#524](https://github.com/nf-core/sarek/pull/524) - Fix intervals usage by counting the actual list of scatter/gather files produced and not overall number of intervals
 - [#549](https://github.com/nf-core/sarek/pull/549) - Fix unique lanes required for Freebayes: issue [#311](https://github.com/nf-core/sarek/issues/311), replaces `meta.clone()` with actual copy of map to avoid issues with <https://nfcore.slack.com/archives/C027CM7P08M/p1644241819942339>
 - [#567](https://github.com/nf-core/sarek/pull/567) - Fix interval name resolving during scatter/gather by moving logic to modules.config causing name to be correctly resolved on process execution; also fixed duplicate naming when variant callers produce multiple vcf files by adding field `type` to `meta` map
+- [#585](https://github.com/nf-core/sarek/pull/585) - Fix Spark usage for GATK4 modules.
 
 ### Deprecated
 

conf/modules.config

@@ -339,8 +339,7 @@ process {
 
     withName: 'GATK4_MARKDUPLICATES_SPARK' {
         ext.args         = '--remove-sequencing-duplicates false -VS LENIENT'
-        //TODO after step markduplicates we only deal with cram now, so this is not necessary, fix in PR that updates spark modules
-        ext.prefix       = { !(params.skip_tools && (params.skip_tools.contains('bamqc') || params.skip_tools.contains('deeptools'))) ? "${meta.id}.md.bam" : "${meta.id}.md.cram" }
+        ext.prefix       = { "${meta.id}.md.cram" }
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/preprocessing/${meta.id}/markduplicates" },

modules.json

@@ -88,7 +88,7 @@
                 "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"
             },
             "gatk4/applybqsrspark": {
-                "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"
+                "git_sha": "6a4732ef3b76b54d75533e6be9ba57e3008d4853"
             },
             "gatk4/applyvqsr": {
                 "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"
@@ -97,7 +97,7 @@
                 "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"
             },
             "gatk4/baserecalibratorspark": {
-                "git_sha": "fd7ee6799abf1bd5228b4b59ce541f3aeaa2280b"
+                "git_sha": "6a4732ef3b76b54d75533e6be9ba57e3008d4853"
             },
             "gatk4/calculatecontamination": {
                 "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"
@@ -139,7 +139,7 @@
                 "git_sha": "df2620cfc7e4c21b14ed03c1c928f09fbabf83c4"
             },
             "gatk4/markduplicatesspark": {
-                "git_sha": "e04970b7d249365cafa5a52912f9a28840481c05"
+                "git_sha": "8e8f4c9c51a7b229dd45e9b287d48115c238baf3"
             },
             "gatk4/mergemutectstats": {
                 "git_sha": "169b2b96c1167f89ab07127b7057c1d90a6996c7"

modules/nf-core/modules/gatk4/applybqsrspark/main.nf

@@ -144,7 +144,7 @@ profiles {
     }
     docker {
         docker.enabled         = true
-        docker.userEmulation   = true
+        docker.userEmulation   = { use_gatk_spark ? false : true }
         singularity.enabled    = false
         podman.enabled         = false
         shifter.enabled        = false