Merge pull request #62 from uclahs-cds/yashpatel-compress-output

Compressed output
uclahs-cds · Apr 1, 2024 · 015011a · 015011a
2 parents 628b9e1 + 723987a
commit 015011a
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ---
 
+## [v1.1.0] - 2024-03-28
+### Added
+- Compressed output for the expanded set of target intervals
+
+---
+
 ## [v1.0.0] - 2024-03-27
 ### Added
 - Add workflow to generate SVG images from embedded PlantUML source

diff --git a/README.md b/README.md
@@ -120,6 +120,7 @@ A directed acyclic graph of your pipeline.
 | ------------ | ------------------------ |
 | `output_dir` | Location where generated output should be saved. |
 | `*target-with-enriched-off-target-intervals.bed` | New target file including original target intervals and intervals encompassing coverage-enriched off-target dbSNP sites. |
+| `*target-with-enriched-off-target-intervals.bed.gz` | New compressed target file including original target intervals and intervals encompassing coverage-enriched off-target dbSNP sites. |
 |`*off-target-dbSNP-depth-per-base.bed`|Per-base read depth at dbSNP loci outside of targeted regions.|
 | `*collapsed_coverage.bed` | Per-base read depth at specified target intervals, collapsed by interval. (OPTIONAL) Set `target_depth` in config file. |
 |`*target-depth-per-base.bed`|Per-base read depth at target intervals (not collapsed). (OPTIONAL) set `save_raw_target_bed` in config file.|

diff --git a/main.nf b/main.nf
@@ -17,14 +17,18 @@ include { run_depth_filter } from './module/filter_off_target_depth.nf'
 include { merge_bedfiles_BEDtools } from './module/merge_bedfiles_bedtools.nf'
 
 include { generate_checksum_PipeVal as generate_sha512sum } from './external/pipeline-Nextflow-module/modules/PipeVal/generate-checksum/main.nf' addParams(
-   options: [
-      output_dir: "${params.workflow_output_dir}/output/",
-      log_output_dir: "${params.log_output_dir}/process-log/",
-      docker_image_version: params.pipeval_version,
-      main_process: "./",
-      checksum_alg: "sha512"
-      ]
-   )
+    options: [
+        output_dir: "${params.workflow_output_dir}/output/",
+        log_output_dir: "${params.log_output_dir}/process-log/",
+        docker_image_version: params.pipeval_version,
+        main_process: "./",
+        checksum_alg: "sha512"
+    ])
+include { compress_index_VCF as compress_index_BED } from './external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams(
+    options: [
+        output_dir: "${params.workflow_output_dir}/",
+        log_output_dir: "${params.log_output_dir}/process-log/"
+    ])
 
 // Log info here
 log.info """\
@@ -217,9 +221,13 @@ workflow {
                     params.target_bed,
                     run_slop_BEDtools_expand_dbSNP.out.bed
                     )
-
-                checksum_ch = checksum_ch.mix(merge_bedfiles_BEDtools.out.bed.flatten())
 
+                compress_index_BED(
+                    merge_bedfiles_BEDtools.out.bed
+                        .map{ it -> [params.sample_id, it] }
+                )
+
+                checksum_ch = checksum_ch.mix(compress_index_BED.out.index_out.map{ it -> [it[1], it[2]]})
                 }
 
                 }

diff --git a/module/merge_bedfiles_bedtools.nf b/module/merge_bedfiles_bedtools.nf
@@ -9,9 +9,10 @@ include { generate_standard_filename } from '../external/pipeline-Nextflow-modul
 process merge_bedfiles_BEDtools {
     container params.docker_image_bedtools
 
-    publishDir path: "${params.workflow_output_dir}/output/",
+    publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.replace(':','/')}",
         pattern: "*.bed",
-        mode: "copy"
+        mode: "copy",
+        enabled: params.save_intermediate_files
 
     publishDir path: "${params.log_output_dir}/process-log/",
         pattern: ".command.*",

diff --git a/nextflow.config b/nextflow.config
@@ -3,5 +3,5 @@ manifest {
     name = 'calculate-targeted-coverage'
     author = 'Nicole Zeltser'
     description = 'Nextflow pipeline for calculating read-depth related statistics for targeted sequencing experiments'
-    version = '1.0.0'
+    version = '1.1.0'
 }
diff --git a/nftest.yml b/nftest.yml
@@ -18,6 +18,9 @@ cases:
       - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_off-target-dbSNP-depth-per-base.bed
         expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.off-target-dbSNP_depth-per-base.bed
         method: md5
-      - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed
-        expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed
+      - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz
+        expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz
+        method: md5
+      - actual: calculate-targeted-coverage-*/TWGSAMIN000001-T002-S02-F/SAMtools-*/output/BEDtools-*_TWGSAMIN000001_TWGSAMIN000001-T002-S02-F_target-with-enriched-off-target-intervals.bed.gz.tbi
+        expect: /hot/software/pipeline/pipeline-calculate-targeted-coverage/Nextflow/development/output/TWGSAMIN000001-T002-S02-F.target_with_enriched_off-target_intervals.bed.gz.tbi
         method: md5
diff --git a/test/configtest-F16.json b/test/configtest-F16.json
@@ -33,7 +33,7 @@
       "author": "Nicole Zeltser",
       "description": "Nextflow pipeline for calculating read-depth related statistics for targeted sequencing experiments",
       "name": "calculate-targeted-coverage",
-      "version": "1.0.0"
+      "version": "1.1.0"
     },
     "params": {
       "bait_bed": "",
@@ -58,7 +58,7 @@
       "input": {
         "bam": "/hot/resource/SMC-HET/tumours/A-mini/bams/n1/output/S2.T-n1.bam"
       },
-      "log_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z",
+      "log_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z",
       "max_cpus": "16",
       "max_memory": "31 GB",
       "merge_operation": "collapse",
@@ -71,7 +71,7 @@
       "off_target_depth": true,
       "off_target_slop": "500",
       "output_dir": "/tmp/nf-config-test-outputs",
-      "output_dir_base": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F",
+      "output_dir_base": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F",
       "output_enriched_target_file": true,
       "patient": "null",
       "picard_CollectHsMetrics_extra_args": "",
@@ -92,7 +92,7 @@
       "target_interval_list": "",
       "ucla_cds": true,
       "work_dir": "/scratch/4674821",
-      "workflow_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/SAMtools-1.16.1"
+      "workflow_output_dir": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/SAMtools-1.16.1"
     },
     "params_schema": {
       "bait_bed": {
@@ -320,15 +320,15 @@
     },
     "report": {
       "enabled": true,
-      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/report.html"
+      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/report.html"
     },
     "timeline": {
       "enabled": true,
-      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/timeline.html"
+      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/timeline.html"
     },
     "trace": {
       "enabled": true,
-      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.0.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.0.0-19970704T165655Z/nextflow-log/trace.txt"
+      "file": "/tmp/nf-config-test-outputs/calculate-targeted-coverage-1.1.0/TWGSAMIN000001-T002-S02-F/log-calculate-targeted-coverage-1.1.0-19970704T165655Z/nextflow-log/trace.txt"
     },
     "tz": "sun.util.calendar.ZoneInfo[id=\"UTC\",offset=0,dstSavings=0,useDaylight=false,transitions=0,lastRule=null]",
     "workDir": "/scratch/4674821",