Merge pull request #141 from GlobalPneumoSeq/dev

Release Version 1.0.0-rc12

README.md

@@ -1,6 +1,6 @@
 # GPS Pipeline <!-- omit in toc -->
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-24.04.4-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-24.10.3-23aa62.svg)](https://www.nextflow.io/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/singularity/)
 [![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/quicklaunch?pipeline=https://github.com/GlobalPneumoSeq/gps-pipeline)
@@ -266,7 +266,7 @@ The pipeline is compatible with [Launchpad](https://docs.seqera.io/platform/late
 ## Serotype
 | Option | Values | Description |
 | --- | ---| --- |
-| `--seroba_db_remote` | Any valid URL to a SeroBA release in `.tar.gz` or `.tgz` format<br />(Default: [SeroBA v1.0.7](https://github.com/sanger-bentley-group/seroba/archive/refs/tags/v1.0.7.tar.gz))| URL to a SeroBA release. |
+| `--seroba_db_remote` | Any valid URL to a SeroBA release in `.tar.gz` or `.tgz` format<br />(Default: [SeroBA v2.0.4](https://github.com/GlobalPneumoSeq/seroba/archive/refs/tags/v2.0.4.tar.gz))| URL to a SeroBA release. |
 | `--seroba_kmer` | Any integer value<br />(Default: `71`) | Kmer size for creating the KMC database of SeroBA. |
 
 ## Lineage
@@ -492,7 +492,7 @@ This project uses open-source components. You can find the homepage or source co
 [SeroBA](https://sanger-pathogens.github.io/seroba/)
 - **SeroBA: rapid high-throughput serotyping of Streptococcus pneumoniae from whole genome sequence data**. Epping L, van Tonder, AJ, Gladstone RA, GPS Consortium, Bentley SD, Page AJ, Keane JA, Microbial Genomics 2018, doi: [10.1099/mgen.0.000186](http://mgen.microbiologyresearch.org/content/journal/mgen/10.1099/mgen.0.000186)
 - License (GPL-3.0): https://github.com/sanger-pathogens/seroba/blob/master/LICENSE
-- This project uses a Docker image of a [fork](https://github.com/sanger-bentley-group/seroba)
+- This project uses a Docker image of a [fork](https://github.com/GlobalPneumoSeq/seroba)
   - The fork provides SeroBA with the latest updates as the original repository is no longer maintained
   - The Docker image provides the containerised environment with SeroBA for `GET_SEROBA_DB` and `SEROTYPE` processes of the `serotype.nf` module
 
@@ -508,13 +508,13 @@ This project uses open-source components. You can find the homepage or source co
 - License (GPL-3.0): https://github.com/tseemann/shovill/blob/master/LICENSE
 - This tool is used in `ASSEMBLY_SHOVILL` process of the `assembly.nf` module
 
-[SPN-PBP-AMR](https://cgps.gitbook.io/pathogenwatch/technical-descriptions/antimicrobial-resistance-prediction/spn-pbp-amr)
+[SPN-PBP-AMR](https://cgps.gitbook.io/pathogenwatch/technical-descriptions/antimicrobial-resistance-prediction/spn-pbp-amr) (CDC PBP AMR Predictor)
 - [Pathogenwatch](https://pathogen.watch/) ([@pathogenwatch-oss](https://github.com/pathogenwatch-oss))
 - License (MIT): https://github.com/pathogenwatch-oss/spn-resistance-pbp/blob/main/LICENSE
 - This is a modified version of [AMR predictor](https://github.com/BenJamesMetcalf/Spn_Scripts_Reference) by Ben Metcalf ([@BenJamesMetcalf](https://github.com/BenJamesMetcalf)) at the Centre for Disease Control (CDC)
-- This project uses a Docker image of a [fork](https://github.com/sanger-bentley-group/spn-pbp-amr)
+- This project uses a Docker image of a [fork](https://github.com/GlobalPneumoSeq/spn-pbp-amr)
   - The fork changes the Docker image from a Docker executable image to a Docker environment for Nextflow integration
-  - The Docker image provides the containerised environment with SPN-PBP-MAR for `PBP_RESISTANCE` process of the `amr.nf` module 
+  - The Docker image provides the containerised environment with SPN-PBP-AMR for `PBP_RESISTANCE` process of the `amr.nf` module 
 
 [Unicycler](https://github.com/rrwick/Unicycler)
 - **Wick RR, Judd LM, Gorrie CL, Holt KE**. Unicycler: resolving bacterial genome assemblies from short and long sequencing reads. *PLoS Comput Biol* 2017.

bin/convert_sam_to_sorted_bam.sh

@@ -1,9 +1,13 @@
 # Convet SAM to sorted BAM file
 # Remove source SAM file if $LITE is true
 
-samtools view -@ "$(nproc)" -b "$SAM" > "$BAM"
+# Thread usage is capped as memory usage is per thread, and the speed gain level-off as thread count increases 
+AVAILABLE_THREAD=$(nproc)
+THREAD=$(( AVAILABLE_THREAD > MAX_THREAD ? MAX_THREAD : AVAILABLE_THREAD ))
 
-samtools sort -@ "$(nproc)" -o "$SORTED_BAM" "$BAM"
+samtools view -@ "$THREAD" -b "$SAM" > "$BAM"
+
+samtools sort -@ "$THREAD" -o "$SORTED_BAM" "$BAM"
 rm "$BAM"
 
 if [ "$LITE" = true ]; then

bin/get_serotype.sh

@@ -1,6 +1,6 @@
 # Run SeroBA to serotype samples
 
-seroba runSerotyping "${SEROBA_DB}" "$READ1" "$READ2" "$SAMPLE_ID" && SEROTYPE=$(awk -F'\t' '{ print $2 }' "${SAMPLE_ID}/pred.tsv")
+seroba runSerotyping "${SEROBA_DB}" "$READ1" "$READ2" "$SAMPLE_ID" && SEROTYPE=$(awk -F',' 'NR==2 { print $3 }' "${SAMPLE_ID}/pred.csv")
 
 echo \"Serotype\" > "$SEROTYPE_REPORT"
 echo \""$SEROTYPE"\" >> "$SEROTYPE_REPORT"

main.nf

@@ -1,7 +1,7 @@
 #!/usr/bin/env nextflow
 
 // Version of this release
-pipelineVersion = '1.0.0-rc11'
+pipelineVersion = '1.0.0-rc12'
 
 // Import workflow modules
 include { PIPELINE } from "$projectDir/workflows/pipeline"

modules/amr.nf

@@ -1,4 +1,5 @@
 // Run PBP AMR predictor to assign pbp genes and estimate samples' MIC (minimum inhibitory concentration) for 6 Beta-lactam antibiotics
+// The process will fail if an empty output is generated
 process PBP_RESISTANCE {
     label 'spn_pbp_amr_container'
     label 'farm_low'
@@ -15,6 +16,7 @@ process PBP_RESISTANCE {
     json='result.json'
     """
     spn_pbp_amr "$assembly" > "$json"
+    grep -q '[^[:space:]]' "$json" || exit 1
     """
 }
 

modules/mapping.nf

@@ -67,11 +67,13 @@ process SAM_TO_SORTED_BAM {
 
     script:
     sorted_bam="${sample_id}_mapped_sorted.bam"
+    max_thread="8"
     """
     SAM="$sam"
     BAM="mapped.bam"
     SORTED_BAM="$sorted_bam"
     LITE="$lite"
+    MAX_THREAD="$max_thread"
 
     source convert_sam_to_sorted_bam.sh
     source get_ref_coverage.sh

nextflow

@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 [[ "$NXF_DEBUG" == 'x' ]] && set -x
-NXF_VER=${NXF_VER:-'24.04.4'}
+NXF_VER=${NXF_VER:-'24.10.3'}
 NXF_ORG=${NXF_ORG:-'nextflow-io'}
 NXF_HOME=${NXF_HOME:-$HOME/.nextflow}
 NXF_PROT=${NXF_PROT:-'https'}
@@ -26,6 +26,43 @@ NXF_CLI="$0 $@"
 NXF_CLI_OPTS=${NXF_CLI_OPTS:-}
 NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005}
 
+function cmp_ver() {
+    local IFS=.
+    local i ver1 ver2
+    read -r -a ver1 <<< "${1//./ }"
+    read -r -a ver2 <<< "${2//./ }"
+
+    # Compare major, minor, and patch numbers
+    for ((i=0; i<3; i++)); do
+        ver1[i]=${ver1[i]//[^0-9]}
+        ver2[i]=${ver2[i]//[^0-9]}
+        [[ ${ver1[i]:-0} -lt ${ver2[i]:-0} ]] && echo -1 && return
+        [[ ${ver1[i]:-0} -gt ${ver2[i]:-0} ]] && echo 1 && return
+    done
+
+    # Extract suffixes for comparison
+    local suffix1="${1##*.}"
+    local suffix2="${2##*.}"
+    suffix1="${suffix1//[0-9]}"
+    suffix2="${suffix2//[0-9]}"
+
+    # Compare suffixes
+    [[ -z $suffix1 && -n $suffix2 ]] && echo 1 && return
+    [[ -n $suffix1 && -z $suffix2 ]] && echo -1 && return
+    [[ $suffix1 < $suffix2 ]] && echo -1 && return
+    [[ $suffix1 > $suffix2 ]] && echo 1 && return
+
+    # Versions are equal
+    echo 0
+}
+
+# if the nextflow version is greater or equals to "24.07.0-edge" the new shadow jar launcher
+# should be used, otherwise fallback on the legacy behavior setting the variable NXF_LEGACY_LAUNCHER
+NXF_LEGACY_LAUNCHER=1
+if [[ $(cmp_ver "$NXF_VER" "24.07.0-edge") -ge 0 ]]; then
+  unset NXF_LEGACY_LAUNCHER
+fi
+
 export NXF_CLI
 export NXF_ORG
 export NXF_HOME
@@ -147,7 +184,7 @@ function check_latest() {
     [[ $cmd != run ]] && return 0
     [[ $NXF_OFFLINE == true || $NXF_DISABLE_CHECK_LATEST == true ]] && return 0
     local latest=$(get_ver "$NXF_BASE/$(current_ver)/version?current=$NXF_VER")
-    if [[ -n "$latest" && "$latest" != $NXF_VER ]]; then
+    if [[ -n "$latest" && $(cmp_ver "$latest" "$NXF_VER") -gt 0 ]]; then
       echo_yellow "Nextflow $latest is available - Please consider updating your version to it"
     fi
 }
@@ -249,18 +286,19 @@ NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar}
 NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR}
 NXF_PATH=$(dirname "$NXF_BIN")
 NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR}
-NXF_GRAB=${NXF_GRAB:-''}
-NXF_CLASSPATH=${NXF_CLASSPATH:-''}
 NXF_HOST=${HOSTNAME:-localhost}
 [[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST}
+# both NXF_GRAB and NXF_CLASSPATH are not supported any more as of version 24.04.7-edge
+NXF_GRAB=${NXF_GRAB:-''}
+NXF_CLASSPATH=${NXF_CLASSPATH:-''}
 
 # Determine the path to this file
-if [[ $NXF_PACK = all ]]; then
+if [[ $NXF_PACK = dist ]]; then
     NXF_BIN=$(which "$0" 2>/dev/null)
     [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0"
 fi
 
-# use nextflow custom java home path 
+# use nextflow custom java home path
 if [[ "$NXF_JAVA_HOME" ]]; then
   JAVA_HOME="$NXF_JAVA_HOME"
   unset JAVA_CMD
@@ -311,8 +349,8 @@ else
     version_check="^(1.7|1.8)"
     version_message="Java 7 or 8"
   else
-    version_check="^(1.8|9|10|11|12|13|14|15|16|17|18|19|20|21|22)"
-    version_message="Java 8 or later (up to 21)"
+    version_check="^(1.8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23)"
+    version_message="Java 8 or later (up to 22)"
   fi
   if [[ ! $JAVA_VER =~ $version_check ]]; then
       echo_red "ERROR: Cannot find Java or it's a wrong version -- please make sure that $version_message is installed"
@@ -323,8 +361,8 @@ else
       fi
       exit 1
   fi
-  if [[ ! $JAVA_VER =~ ^(11|12|13|14|15|16|17|18|19|20|21|22) ]]; then
-      echo_yellow "NOTE: Nextflow is not tested with Java $JAVA_VER -- It's recommended the use of version 11 up to 22\n"
+  if [[ ! $JAVA_VER =~ ^(11|12|13|14|15|16|17|18|19|20|21|22|23) ]]; then
+      echo_yellow "NOTE: Nextflow is not tested with Java $JAVA_VER -- It's recommended the use of version 11 up to 23\n"
   fi
   mkdir -p "$(dirname "$JAVA_KEY")"
   [[ -f $JAVA_VER ]] && echo $JAVA_VER > "$JAVA_KEY"
@@ -346,10 +384,11 @@ if [[ $cmd == console ]]; then bg=1;
 else JAVA_OPTS+=(-Djava.awt.headless=true)
 fi
 
-[[ "$JAVA_VER" =~ ^(21|22) ]] && [[ ! "$NXF_ENABLE_VIRTUAL_THREADS" ]] && NXF_ENABLE_VIRTUAL_THREADS=true
-[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME")
-[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG)
-[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true)
+if [[ $NXF_LEGACY_LAUNCHER ]]; then
+  [[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME")
+  [[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG)
+  [[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true)
+fi
 [[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1)
 [[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS)
 [[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH
@@ -401,9 +440,14 @@ LAUNCH_FILE="${NXF_LAUNCHER}/classpath-$(env_md5)"
 if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ] && [[ "$remote_debug" -ne 1 ]]; then
     declare -a launcher="($(cat "$LAUNCH_FILE"))"
 else
-    # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file
-    cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN"))
-    [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1
+    if [[ $NXF_LEGACY_LAUNCHER ]]; then
+      # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file
+      cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN"))
+      [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1
+    else
+      # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file
+      cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")
+    fi
 
     # first string between double quotes is the full path to java, also blank spaces are included
     # remainder string are arguments
@@ -417,7 +461,7 @@ else
     [[ "$NXF_JVM_ARGS" ]] && launcher+=($NXF_JVM_ARGS)
     [[ "$remote_debug" ]] && launcher+=(-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=$NXF_REMOTE_DEBUG_PORT)
 
-    if [[ "$JAVA_VER" =~ ^(9|10|11|12|13|14|15|16|17|18|19|20|21|22) ]]; then
+    if [[ "$JAVA_VER" =~ ^(9|10|11|12|13|14|15|16|17|18|19|20|21|22|23) ]]; then
       launcher+=(--add-opens=java.base/java.lang=ALL-UNNAMED)
       launcher+=(--add-opens=java.base/java.io=ALL-UNNAMED)
       launcher+=(--add-opens=java.base/java.nio=ALL-UNNAMED)
@@ -437,14 +481,16 @@ else
       launcher+=(--add-opens=java.base/java.util.regex=ALL-UNNAMED)
       if [[ "$NXF_ENABLE_VIRTUAL_THREADS" == 'true' ]]; then
         if [[ "$JAVA_VER" =~ ^(19|20) ]]; then launcher+=(--enable-preview)
-        elif [[ ! "$JAVA_VER" =~ ^(21|22) ]]; then die "Virtual threads require Java 19 or later - current version $JAVA_VER"
+        elif [[ ! "$JAVA_VER" =~ ^(21|22|23) ]]; then die "Virtual threads require Java 19 or later - current version $JAVA_VER"
         fi
       fi
       launcher+=("${cmd_tail[@]}")
     else
       launcher+=("${cmd_tail[@]}")
     fi
 
+    # create the launch file only when using the legacy launcher (capsule)
+    if [[ $NXF_LEGACY_LAUNCHER ]]; then
     # Don't show errors if the LAUNCH_FILE can't be created
     if mkdir -p "${NXF_LAUNCHER}" 2>/dev/null; then
         STR=''
@@ -455,6 +501,7 @@ else
     else
         echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?"
     fi
+    fi
 
 fi
 

nextflow.config

@@ -27,7 +27,7 @@ params {
     assembly_publish = "link"
 
     // Default link for SeroBA repository, and KMC kmer size for SeroBA 
-    seroba_db_remote = "https://github.com/sanger-bentley-group/seroba/archive/refs/tags/v1.0.7.tar.gz"
+    seroba_db_remote = "https://github.com/GlobalPneumoSeq/seroba/archive/refs/tags/v2.0.4.tar.gz"
     seroba_kmer = 71
 
     // Default link for Kraken2 Database, and usage of memory mapping
@@ -111,7 +111,7 @@ process {
         container = 'staphb/kraken2:2.1.2-no-db'
     }
     withLabel: seroba_container {
-        container = 'sangerbentleygroup/seroba:1.0.7'
+        container = 'sangerbentleygroup/seroba:2.0.4'
     }
 }