Demultiplex - RNAseq meta workflow

_viash.yaml

@@ -7,7 +7,14 @@ links:
   issue_tracker: https://github.com/viash-hub/playground/issues
   repository: https://github.com/viash-hub/playground
 
-viash_version: 0.9.0-RC6
+viash_version: 0.9.2
+
+info:
+  test_resources:
+    - path: gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta
+      dest: resources_test
 
 config_mods: |
   .requirements.commands := ['ps']
+  .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}
+  .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")'

nextflow.config

@@ -1,3 +1,7 @@
+params {
+  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
+}
+
 process.container = "nextflow/nextflow:21.04.3"
 docker {
   enabled = true

src/config/labels.config

@@ -0,0 +1,98 @@
+process {
+  container = 'nextflow/bash:latest'
+
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
+
+  // Retry for exit codes that have something to do with memory issues
+  errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
+  maxRetries = 3
+  maxMemory = 192.GB
+
+  // Resource labels
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
+
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
+
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        maxMemory = 50.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 12 }
+
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
+}
+
+def get_memory(to_compare) {
+    if (!process.containsKey("maxMemory") || !process.maxMemory) {
+      return to_compare
+    }
+
+    try {
+      if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
+        return process.maxMemory
+      }
+      else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
+        return max_memory as nextflow.util.MemoryUnit
+      }
+      else {
+        return to_compare
+      }  
+    } catch (all) {
+          println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
+          System.exit(1)
+    }
+  }

src/meta_workflows/demultiplex_rnaseq/README.qmd

@@ -0,0 +1,18 @@
+---
+title: A meta workflow for demultiplexing and RNA sequencing analysis
+format: gfm
+---
+
+# Introduction
+
+To perform demultiplexing and analysis of RNAseq data, we will create a meta workflow that includes the demultiplex and rnaseq workflows from Viash Hub. 
+
+1. Start by creating a new workflow configuration with all the necessary inputs and outputs.
+2. Add the demultiplex and rnaseq workflows as dependencies in the configuration.
+3. Create the Nextflow script for the workflow.
+4. Add the demultiplex and rnaseq workflows as steps in the Nextflow workflow.
+5. Add any intermediate steps that are required to gather output fastq files from the demultiplex workflow and modify the state as required by the rnaseq workflow. 
+
+# Test data
+We will use the same test data from <https://github.com/nf-core/test-datasets/raw/refs/heads/demultiplex/testdata/NovaSeq6000>. The genome fasta was obtained from <https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/858/895/GCA_009858895.3_ASM985889v3/GCA_009858895.3_ASM985889v3_genomic.fna.gz> and the annotation file was obtained from <http://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz>.
+The `example.yaml` parameter file can be used to test the meta-workflow.