Bug fix consensus

.github/workflows/nf-test.yml

@@ -50,6 +50,7 @@ jobs:
           NFT_VER: ${{ env.NFT_VER }}
         with:
           max_shards: 7
+          tags: ${{ github.event_name == 'pull_request' && 'small' || '' }}
 
       - name: debug
         run: |
@@ -98,6 +99,7 @@ jobs:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
+          tags: ${{ github.event_name == 'pull_request' && 'small' || '' }}
 
       - name: Report test status
         if: ${{ always() }}

.gitignore

@@ -8,3 +8,5 @@ testing*
 *.pyc
 null/
 .lineage/
+.nf-test/
+.nf-test.log

.nf-core.yml

@@ -7,6 +7,7 @@ lint:
     - .github/ISSUE_TEMPLATE/config.yml
     - .github/workflows/awstest.yml
     - .github/workflows/awsfulltest.yml
+    - .github/CONTRIBUTING.md
   files_unchanged:
     - CODE_OF_CONDUCT.md
     - assets/nf-core-lrsomatic_logo_light.png
@@ -17,6 +18,9 @@ lint:
     - .github/PULL_REQUEST_TEMPLATE.md
     - assets/email_template.txt
     - docs/README.md
+    - .github/workflows/branch.yml
+    - .github/workflows/linting_comment.yml
+    - .github/workflows/linting.yml
   multiqc_config:
     - report_comment
   nextflow_config:

conf/modules.config

@@ -287,14 +287,20 @@ process {
     //
 
     withName: '.*:MINIMAP2_ALIGN' {
-        ext.prefix = { "${meta.id}_${meta.type}_mapped" }
+        ext.prefix = { "${meta.id}_${meta.type}_rep${meta.replicate}_mapped" }
         ext.args = {
+            def platform_arg = meta.platform == 'pb' ?
+                ( params.minimap2_pb_model ? "-ax $params.minimap2_pb_model" : "-ax map-hifi" ) :
+                ( params.minimap2_ont_model ? "-ax $params.minimap2_ont_model" : "-ax lr:hq" )
+            def secondary_arg = params.save_secondary_alignment ? "--secondary=yes" : "--secondary=no"
+            def pl = meta.platform == 'pb' ? 'PACBIO' : 'ONT'
+            def rg = "@RG\\tID:${meta.id}_${meta.type}_rep${meta.replicate}\\tSM:${meta.id}\\tLB:${meta.id}_${meta.type}\\tPL:${pl}"
             [
-                meta.platform == 'pb' ? ( params.minimap2_pb_model ? "-ax $params.minimap2_pb_model" : "-ax map-hifi" ) :
-                ( params.minimap2_ont_model ? "-ax $params.minimap2_ont_model" : "-ax lr:hq " ),
-                params.save_secondary_alignment ? "--secondary=yes " : "--secondary=no ",
+                platform_arg,
+                secondary_arg,
                 "-y",
-                "-Y"
+                "-Y",
+                "-R '${rg}'"
             ].join(' ').trim()
         }
         ext.args4  = "-T '*'"
@@ -303,6 +309,19 @@ process {
         ]
     }
 
+    withName: '.*:SAMTOOLS_MERGE' {
+        ext.prefix = { "${meta.id}_${meta.type}_mapped" }
+        publishDir = [
+            enabled: false
+        ]
+    }
+
+    withName: '.*:SAMTOOLS_INDEX_MERGE' {
+        publishDir = [
+            enabled: false
+        ]
+    }
+
     //
     // Methylation related processes
     //
@@ -359,13 +378,20 @@ process {
         ]
     }
     withName: '.*:LONGPHASE_PHASE_SOMATIC' {
-        ext.prefix = { "somatic_smallvariants" }
+        ext.prefix = { "somatic_smallvariants_combined" }
         ext.args   = {
                 [
                     meta.platform == 'pb' ? '--pb' : '--ont',
                     "--indels",
                 ].join(' ').trim()
             }
+        // Intermediate output (somatic+germline combined); filtered version published below
+        publishDir = [
+            enabled: false
+        ]
+    }
+    withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' {
+        ext.prefix = { "somatic_smallvariants" }
         publishDir = [
             path: { "${params.outdir}/${meta.id}/variants/phased" },
             mode: params.publish_dir_mode,
@@ -435,6 +461,16 @@ process {
             enabled: false
         ]
     }
+    withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' {
+        ext.prefix = { "${meta.id}_germline_sorted" }
+        ext.args   = { '-Oz -W=tbi' }
+        publishDir = [ enabled: false ]
+    }
+    withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' {
+        ext.prefix = { "${meta.id}_somatic_sorted" }
+        ext.args   = { '-Oz -W=tbi' }
+        publishDir = [ enabled: false ]
+    }
     withName: '.*:GERMLINE_CONSENSUS:SORT_POST_NORM' {
         ext.prefix = { "${meta.id}.${meta.caller}_norm_sorted" }
         ext.args = { '-Oz -W=tbi' }

conf/test.config

@@ -52,7 +52,7 @@ params {
     fasta    = "https://github.com/IntGenomicsLab/test-datasets/main/references/GRCh38_chr19.fasta.gz"
 
     // Additional params
-    genome             = "CHM13"
+    genome             = "GRCh38"
     vep_genome         = "WBcel235"
     vep_species        = "caenorhabditis_elegans"
     skip_wakhan        = true

conf/test_full.config

@@ -1,11 +1,12 @@
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Nextflow config file for running minimal tests
+    Nextflow config file for running full tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Defines input files and everything required to run a fast and simple pipeline test.
+    Defines input files and everything required to run a full pipeline test with both
+    callers in union combine mode.
 
     Use as follows:
-        nextflow run IntGenomicsLab/lrsomatic -profile test,<docker/singularity> --outdir <OUTDIR>
+        nextflow run IntGenomicsLab/lrsomatic -profile test_full,<docker/singularity> --outdir <OUTDIR>
 
 ----------------------------------------------------------------------------------------
 */
@@ -21,17 +22,23 @@ process {
 }
 
 params {
-    config_profile_name        = 'Test profile'
-    config_profile_description = 'Minimal test dataset to check pipeline function'
+    config_profile_name        = 'Full test profile'
+    config_profile_description = 'Full test dataset with both callers in union combine mode'
 
     // Input data
     input    = "https://github.com/IntGenomicsLab/test-datasets/main/samplesheets/samplesheet_lr-somatic.csv"
     fasta    = "https://github.com/IntGenomicsLab/test-datasets/main/references/GRCh38_chr19.fasta.gz"
 
     // Additional params
-    genome             = "GRCh38"
-    vep_genome         = "WBcel235"
-    vep_species        = "caenorhabditis_elegans"
-    skip_wakhan        = true
-    skip_ascat         = true
+    genome                = "GRCh38"
+    vep_genome            = "WBcel235"
+    vep_species           = "caenorhabditis_elegans"
+    skip_wakhan           = true
+    skip_ascat            = true
+
+    // Both callers, union combine mode
+    germline_var_keep     = 'clair, deepvariant'
+    somatic_var_keep      = 'clair, deepsomatic'
+    germline_var_combine  = 'all'
+    somatic_var_combine   = 'all'
 }

modules.json

@@ -152,6 +152,11 @@
                         "git_sha": "1d2fbdcbca677bbe8da0f9d0d2bb7c02f2cab1c9",
                         "installed_by": ["modules"]
                     },
+                    "samtools/merge": {
+                        "branch": "master",
+                        "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120",
+                        "installed_by": ["modules"]
+                    },
                     "samtools/stats": {
                         "branch": "master",
                         "git_sha": "fe93fde0845f907fc91ad7cc7d797930408824df",

modules/local/bcftools/view/environment.yml

@@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://github.com/nf-core/modules/master/modules/environment-schema.json
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  # renovate: datasource=conda depName=bioconda/htslib
+  - bioconda::bcftools=1.22
+  - bioconda::htslib=1.22.1

modules/local/bcftools/view/main.nf

@@ -0,0 +1,40 @@
+process BCFTOOLS_VIEW {
+    tag "${meta.id}"
+    label 'process_medium'
+
+    conda "${moduleDir}/environment.yml"
+    container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
+        ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/47/474a5ea8dc03366b04df884d89aeacc4f8e6d1ad92266888e7a8e7958d07cde8/data'
+        : 'community.wave.seqera.io/library/bcftools_htslib:0a3fa2654b52006f'}"
+
+    input:
+    tuple val(meta), path(vcf), path(tbi), path(targets), path(targets_tbi)
+
+    output:
+    tuple val(meta), path("*.vcf.gz"),  emit: vcf
+    tuple val(meta), path("*.tbi"),     emit: tbi
+    tuple val("${task.process}"), val('bcftools'), eval("bcftools --version | sed '1!d; s/^.*bcftools //'"), topic: versions, emit: versions_bcftools
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args   = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    bcftools view \\
+        -T ${targets} \\
+        -Oz \\
+        -W=tbi \\
+        ${args} \\
+        -o ${prefix}.vcf.gz \\
+        ${vcf}
+    """
+
+    stub:
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    echo '' | gzip > ${prefix}.vcf.gz
+    touch ${prefix}.vcf.gz.tbi
+    """
+}

modules/local/bcftools/view/meta.yml

@@ -0,0 +1,56 @@
+name: bcftools_view
+description: Filter VCF to positions defined by a targets file using bcftools view -T
+keywords:
+  - filtering
+  - VCF
+  - variant calling
+tools:
+  - view:
+      description: VCF/BCF conversion, view, subset and filter VCF/BCF files.
+      homepage: http://samtools.github.io/bcftools/bcftools.html
+      documentation: http://www.htslib.org/doc/bcftools.html
+      tool_dev_url: https://github.com/samtools/bcftools
+      doi: "10.1093/bioinformatics/btp352"
+      licence: ["MIT"]
+      identifier: biotools:bcftools
+input:
+  - - meta:
+        type: map
+        description: Groovy Map containing sample information e.g. [ id:'test' ]
+    - vcf:
+        type: file
+        description: Input VCF/BCF file to filter
+        pattern: "*.{vcf.gz,vcf,bcf}"
+    - tbi:
+        type: file
+        description: Tabix index of the input VCF
+        pattern: "*.tbi"
+    - targets:
+        type: file
+        description: VCF file used as position filter (-T)
+        pattern: "*.{vcf.gz,vcf,bcf}"
+    - targets_tbi:
+        type: file
+        description: Tabix index of the targets VCF
+        pattern: "*.tbi"
+output:
+  vcf:
+    - - meta:
+          type: map
+          description: Groovy Map containing sample information
+      - "*.vcf.gz":
+          type: file
+          description: Filtered VCF file
+          pattern: "*.vcf.gz"
+  tbi:
+    - - meta:
+          type: map
+          description: Groovy Map containing sample information
+      - "*.tbi":
+          type: file
+          description: Tabix index of filtered VCF
+          pattern: "*.tbi"
+authors:
+  - "@rforsyth"
+maintainers:
+  - "@rforsyth"

modules/local/deepsomatic/postprocessvariants/main.nf

@@ -28,7 +28,7 @@ process DEEPSOMATIC_POSTPROCESSVARIANTS {
         error "DEEPSOMATIC module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
     def args = task.ext.args ?: ''
-    prefix = task.ext.prefix ?: "${meta.id}"
+    prefix = task.ext.prefix ?: "${meta.id}_somatic"
 
     def regions = intervals ? "--regions ${intervals}" : ""
     def variant_calls_tfrecord_name = variant_calls_tfrecord_files[0].name.replaceFirst(/-\d{5}-of-\d{5}/, "")
@@ -121,7 +121,7 @@ process DEEPSOMATIC_POSTPROCESSVARIANTS {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         error "DEEPVARIANT module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    prefix = task.ext.prefix ?: "${meta.id}"
+    prefix = task.ext.prefix ?: "${meta.id}_somatic"
     """
     echo "" | gzip > ${prefix}.vcf.gz
     touch ${prefix}.vcf.gz.tbi