diff --git a/_viash.yaml b/_viash.yaml index 4b90754..c8c1bef 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -7,7 +7,14 @@ links: issue_tracker: https://github.com/viash-hub/playground/issues repository: https://github.com/viash-hub/playground -viash_version: 0.9.0-RC6 +viash_version: 0.9.2 + +info: + test_resources: + - path: gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta + dest: resources_test config_mods: | .requirements.commands := ['ps'] + .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} + .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")' diff --git a/nextflow.config b/nextflow.config index 9889b4d..84625cf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,3 +1,7 @@ +params { + rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString() +} + process.container = "nextflow/nextflow:21.04.3" docker { enabled = true diff --git a/src/config/labels.config b/src/config/labels.config new file mode 100644 index 0000000..a3a1cd3 --- /dev/null +++ b/src/config/labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 50.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/README.qmd b/src/meta_workflows/demultiplex_rnaseq/README.qmd new file mode 100644 index 0000000..565e7b2 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/README.qmd @@ -0,0 +1,18 @@ +--- +title: A meta workflow for demultiplexing and RNA sequencing analysis +format: gfm +--- + +# Introduction + +To perform demultiplexing and analysis of RNAseq data, we will create a meta workflow that includes the demultiplex and rnaseq workflows from Viash Hub. + +1. Start by creating a new workflow configuration with all the necessary inputs and outputs. +2. Add the demultiplex and rnaseq workflows as dependencies in the configuration. +3. Create the Nextflow script for the workflow. +4. Add the demultiplex and rnaseq workflows as steps in the Nextflow workflow. +5. Add any intermediate steps that are required to gather output fastq files from the demultiplex workflow and modify the state as required by the rnaseq workflow. + +# Test data +We will use the same test data from . The genome fasta was obtained from and the annotation file was obtained from . +The `example.yaml` parameter file can be used to test the meta-workflow. \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml new file mode 100644 index 0000000..1540425 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -0,0 +1,915 @@ +name: demultiplex_rnaseq +description: Demultiplexing of raw sequencing data and RNA sequencing analysis. + +argument_groups: + + - name: Demultiplex Input arguments + arguments: + - name: "--input" + description: Directory containing raw sequencing data + type: file + required: true + - name: "--sample_sheet" + description: | + Sample sheet as input for BCL Convert. If not specified, + will try to autodetect the sample sheet in the input directory + type: file + required: false + + - name: Reference genome options + arguments: + - name: "--fasta" + type: file + description: Path to FASTA genome file. + required: true + - name: "--gtf" + type: file + description: Path to GTF annotation file. This parameter is *mandatory* if --genome is not specified. + required: false + - name: "--gff" + type: file + description: Path to GFF3 annotation file. Required if "--gtf" is not specified. + required: false + - name: "--additional_fasta" + type: file + description: FASTA file to concatenate to genome FASTA file e.g. containing spike-in sequences. + - name: "--transcript_fasta" + type: file + description: Path to FASTA transcriptome file. + - name: "--gene_bed" + type: file + description: Path to BED file containing gene intervals. This will be created from the GTF file if not specified. + - name: "--star_index" + type: file + description: Path to directory or tar.gz archive for pre-built STAR index. + - name: "--rsem_index" + type: file + description: Path to directory or tar.gz archive for pre-built RSEM index. + - name: "--salmon_index" + type: file + description: Path to directory or tar.gz archive for pre-built Salmon index. + - name: "--kallisto_index" + type: file + description: Path to directory or tar.gz archive for pre-built Kallisto index. + - name: "--gencode" + type: boolean_true + description: Specify if the GTF annotation is in GENCODE format. + - name: "--gtf_extra_attributes" + type: string + description: Additional gene identifiers from the input GTF file when running Salmon. More than one value can be specified separated by comma. + default: gene_name + - name: "--gtf_group_features" + type: string + description: Define the attribute type used to group features in the GTF file when running Salmon. + default: gene_id + - name: "--featurecounts_group_type" + type: string + description: The attribute type used to group feature types in the GTF file when generating the biotype plot with featureCounts. + default: gene_biotype + - name: "--featurecounts_feature_type" + type: string + description: By default, the pipeline assigns reads based on the 'exon' attribute within the GTF file. + default: exon + - name: --star_sjdb_gtf_feature_exon + type: string + description: Feature type in GTF file to be used as exons for building transcripts + + - name: Read trimming options + arguments: + - name: "--trimmer" + type: string + description: Specify the trimming tool to use. + choices: ["trimgalore", "fastp"] + default: "trimgalore" + - name: "--min_trimmed_reads" + type: integer + description: Minimum number of trimmed reads below which samples are removed from further processing. Some downstream steps in the pipeline will fail if this threshold is too low. + default: 10000 + + - name: Read filtering options + arguments: + - name: "--bbsplit_fasta_list" + type: file + description: Path to comma-separated file containing a list of reference genomes to filter reads against with BBSplit. To use BBSplit, "--skip_bbsplit" must be explicitly set to "false". The file should contain 2 (comma separated) columns - short name and full path to reference genome(s) + - name: "--bbsplit_index" + type: file + description: Path to directory or tar.gz archive for pre-built BBSplit index. + - name: "--remove_ribo_rna" + type: boolean_true + description: Enable the removal of reads derived from ribosomal RNA using SortMeRNA. + - name: "--ribo_database_manifest" + type: file + description: Text file containing paths to fasta files (one per line) that will be used to create the database for SortMeRNA. + default: src/assets/rrna-db-defaults.txt + + - name: UMI options + arguments: + - name: "--with_umi" + type: boolean_true + description: Enable UMI-based read deduplication. + - name: "--umitools_extract_method" + type: string + description: UMI pattern to use. + default: "string" + choices: [string, regex] + - name: "--umitools_bc_pattern" + type: string + description: The UMI barcode pattern to use e.g. 'NNNNNN' indicates that the first 6 nucleotides of the read are from the UMI. + - name: "--umitools_bc_pattern2" + type: string + description: The UMI barcode pattern to use if the UMI is located in read 2. + - name: "--umi_discard_read" + type: integer + description: After UMI barcode extraction discard either R1 or R2 by setting this parameter to 1 or 2, respectively. + choices: [0, 1, 2] + default: 0 + - name: "--umitools_umi_separator" + type: string + description: The character that separates the UMI in the read name. Most likely a colon if you skipped the extraction with UMI-tools and used other software. + default: "_" + - name: "--umitools_grouping_method" + type: string + description: Method to use to determine read groups by subsuming those with similar UMIs. All methods start by identifying the reads with the same mapping position, but treat similar yet nonidentical UMIs differently. + choices: [ "unique", "percentile", "cluster", "adjacency", "directional" ] + default: "directional" + - name: "--umi_dedup_stats" + type: boolean_true + description: Generate output stats when running "umi_tools dedup". + + - name: Alignment options + arguments: + - name: "--aligner" + type: string + description: Specifies the alignment algorithm to use - available options are 'star_salmon', 'star_rsem' and 'hisat2'. + choices: [star_salmon, star_rsem, hisat2] + default: "star_salmon" + - name: "--pseudo_aligner" + type: string + description: Specifies the pseudo aligner to use - available options are 'salmon'. Runs in addition to '--aligner'. + choices: [salmon, kallisto] + default: "salmon" + - name: "--pseudo_aligner_kmer_size" + type: integer + description: Kmer length passed to indexing step of pseudoaligners. + default: 31 + - name: "--kallisto_quant_fragment_length" + type: integer + description: For single-end mode only, the estimated average fragment length to use for quantification with Kallisto. + - name: "--kallisto_quant_fragment_length_sd" + type: integer + description: For single-end mode only, the estimated standard deviation of the fragment length for quantification with Kallisto. + - name: "--bam_csi_index" + type: boolean_true + description: Create a CSI index for BAM files instead of the traditional BAI index. This will be required for genomes with larger chromosome sizes. + - name: "--salmon_quant_libtype" + type: string + description: Override Salmon library type inferred based on strandedness defined in meta object. + - name: "--min_mapped_reads" + type: integer + description: Minimum percentage of uniquely mapped reads below which samples are removed from further processing. + default: 5 + - name: "--stringtie_ignore_gtf" + type: boolean_true + description: Perform reference-guided de novo assembly of transcripts using StringTie, i.e. don't restrict to those in GTF file. + - name: "--extra_stringtie_args" + type: string + default: '-v' + description: Extra arguments to pass to stringtie command in addition to defaults defined by the pipeline. + - name: "--save_unaligned" + type: boolean_true + description: Where possible, save unaligned reads from either STAR, HISAT2 or Salmon to the results directory. + - name: "--save_align_intermeds" + type: boolean_true + description: Save the intermediate BAM files from the alignment step. + - name: "--skip_alignment" + type: boolean_true + description: Skip all of the alignment-based processes within the pipeline. + - name: "--skip_pseudo_alignment" + type: boolean_true + description: Skip all of the pseudo-alignment-based processes within the pipeline. + + - name: Process skipping options + arguments: + - name: "--skip_fastqc" + type: boolean + description: Skip FatQC step. + default: false + - name: "--skip_trimming" + type: boolean + description: Skip the adapter trimming step. + default: false + - name: "--skip_bbsplit" + type: boolean_true + description: Skip BBSplit for removal of non-reference genome reads. + - name: "--skip_umi_extract" + type: boolean + description: Skip umi_tools extract step. + default: false + - name: "--skip_qc" + type: boolean_true + description: Skip all QC steps except for MultiQC. + - name: "--skip_markduplicates" + type: boolean_true + description: Skip picard MarkDuplicates step. + - name: "--skip_stringtie" + type: boolean_true + description: Skip StringTie. + - name: "--skip_biotype_qc" + type: boolean_true + description: Skip additional featureCounts process for biotype QC. + - name: "--skip_bigwig" + type: boolean_true + description: Skip bigWig file creation. + - name: "--skip_preseq" + type: boolean_true + description: Skip Preseq. + - name: "--skip_deseq2_qc" + type: boolean_true + description: Skip DESeq2 PCA and heatmap plotting. + - name: --skip_dupradar + type: boolean_true + description: Skip dupRadar. + - name: --skip_qualimap + type: boolean_true + description: Skip Qualimap. + - name: --skip_rseqc + type: boolean_true + description: Skip RSeQC. + - name: --skip_multiqc + type: boolean_true + description: Skip MultiQC. + + - name: Other process arguments + arguments: + - name: "--extra_picard_args" + type: string + default: ' --ASSUME_SORTED true --REMOVE_DUPLICATES false --VALIDATION_STRINGENCY LENIENT --TMP_DIR tmp' + description: Extra arguments to pass to picard MarkDuplicates command in addition to defaults defined by the pipeline. + - name: "--extra_preseq_args" + type: string + description: Extra arguments to pass to preseq lc_extrap command in addition to defaults defined by the pipeline + default: '-verbose -seed 1 -seg_len 100000000' + - name: "--deseq2_vst" + type: boolean + default: true + description: Use vst transformation instead of rlog with DESeq2 + - name: "--rseqc_modules" + type: string + multiple: true + multiple_sep: ";" + description: Specify the RSeQC modules to run_wf (comma-separated list) + default: bam_stat;inner_distance;infer_experiment;junction_annotation;junction_saturation;read_distribution;read_duplication + choices: [ "bam_stat", "inner_distance", "infer_experiment", "junction_annotation", "junction_saturation", "read_distribution", "read_duplication", "tin" ] + + - name: MultiQC paramenters + arguments: + - name: "--multiqc_custom_config" + type: file + default: src/assets/multiqc_config.yml + - name: "--multiqc_title" + type: string + - name: "--multiqc_methods_description" + type: file + default: src/assets/methods_description_template.yml + + + - name: Demultiplex Output arguments + arguments: + - name: --output_demultiplexed + description: Directory to write demultiplexed output to + type: file + direction: output + required: false + default: "$id/demultiplex" + - name: --output_falco + description: Directory to write Falco output to + type: file + direction: output + required: false + default: "$id/falco" + - name: --output_multiqc + description: Directory to write MultiQC output to + type: file + direction: output + required: false + default: "$id/multiqc_report.html" + + - name: RNAseq Output + arguments: + # Reference files + - name: "--output_fasta" + type: file + direction: output + default: reference/genome.fasta + - name: "--output_gtf" + type: file + direction: output + default: reference/gene_annotation.gtf + - name: "--output_transcript_fasta" + type: file + direction: output + default: reference/transcriptome.fasta + - name: "--output_gene_bed" + type: file + direction: output + default: reference/gene_annotation.bed + - name: "--output_star_index" + type: file + direction: output + description: Path to STAR index. + default: reference/index/STAR + - name: "--output_salmon_index" + type: file + direction: output + description: Path to Salmon index. + default: reference/index/Salmon + - name: "--output_bbsplit_index" + type: file + direction: output + description: Path to BBSplit index. + default: reference/index/BBSplit + - name: "--output_kallisto_index" + type: file + direction: output + description: Path to Kallisto index. + default: reference/index/Kallisto + + # fastq + - name: "--output_fastq_1" + type: file + direction: output + required: false + must_exist: false + description: Path to output directory + default: fastq/$id.read_1.fastq.gz + - name: "--output_fastq_2" + type: file + direction: output + required: false + must_exist: false + description: Path to output directory + default: fastq/$id.read_2.fastq.gz + + # FastQC + - name: "--fastqc_html_1" + type: file + direction: output + description: FastQC HTML report for read 1. + required: false + must_exist: false + default: fastqc_raw/$id.read_1.fastqc.html + - name: "--fastqc_html_2" + type: file + direction: output + description: FastQC HTML report for read 2. + required: false + must_exist: false + default: fastqc_raw/$id.read_2.fastqc.html + - name: "--fastqc_zip_1" + type: file + direction: output + description: FastQC report archive for read 1. + required: false + must_exist: false + default: fastqc_raw/$id.read_1.fastqc.zip + - name: "--fastqc_zip_2" + type: file + direction: output + description: FastQC report archive for read 2. + required: false + must_exist: false + default: fastqc_raw/$id.read_2.fastqc.zip + - name: "--trim_html_1" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_1.trimmed_fastqc.html + - name: "--trim_html_2" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_2.trimmed_fastqc.html + - name: "--trim_zip_1" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_1.trimmed_fastqc.zip + - name: "--trim_zip_2" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_2.trimmed_fastqc.zip + + # TrimGalore + - name: "--trim_log_1" + type: file + direction: output + required: false + must_exist: false + default: trimgalore/$id.read_1.trimming_report.txt + - name: "--trim_log_2" + type: file + direction: output + required: false + must_exist: false + default: trimgalore/$id.read_2.trimming_report.txt + + # fastp + - name: --fastp_trim_json + type: file + description: The fastp json format report file name + default: fastp/$id_out.json + direction: output + - name: --fastp_trim_html + type: file + description: The fastp html format report file name + default: fastp/$id_out.html + direction: output + + # SortMeRNA + - name: "--sortmerna_log" + type: file + direction: output + default: sortmerna/$id.log + required: false + must_exist: false + description: Sortmerna log file. + + # STAR + - name: "--star_alignment" + type: file + direction: output + default: STAR/$id + - name: "--genome_bam_sorted" + type: file + direction: output + default: STAR/genome_processed/$id.genome.bam + - name: "--genome_bam_index" + type: file + direction: output + default: STAR/genome_processed/$id.genome.bam.bai + - name: "--transcriptome_bam" + type: file + direction: output + default: STAR/transcriptome_processed/$id.transcriptome.bam + - name: "--transcriptome_bam_index" + type: file + direction: output + default: STAR/transcriptome_processed/$id.transcriptome.bam.bai + - name: "--star_log" + type: file + direction: output + default: STAR/log/$id.log + + # samtools + - name: "--genome_bam_stats" + type: file + direction: output + default: samtools_stats/$id.genome.stats + - name: "--genome_bam_flagstat" + type: file + direction: output + default: samtools_stats/$id.genome.flagstat + - name: "--genome_bam_idxstats" + type: file + direction: output + default: samtools_stats/$id.genome.idxstats + - name: "--transcriptome_bam_stats" + type: file + direction: output + default: samtools_stats/$id.transcriptome.stats + - name: "--transcriptome_bam_flagstat" + type: file + direction: output + default: samtools_stats/$id.transcriptome.flagstat + - name: "--transcriptome_bam_idxstats" + type: file + direction: output + default: samtools_stats/$id.transcriptome.idxstats + + # Transcript quantification + - name: "--salmon_quant_results" + type: file + direction: output + default: STAR_Salmon/$id + - name: "--salmon_quant_results_file" + type: file + direction: output + default: STAR_Salmon/$id/quant.sf + - name: "--pseudo_quant_results" + type: file + direction: output + default: Pseudo_align_quant/$id + + # RSEM + - name: "--rsem_counts_gene" + type: file + description: Expression counts on gene level + default: RSEM/$id.genes.results + direction: output + - name: "--rsem_counts_transcripts" + type: file + description: Expression counts on transcript level + default: RSEM/$id.isoforms.results + direction: output + - name: "--bam_star_rsem" + type: file + description: BAM file generated by STAR (from RSEM) + default: RSEM/$id.STAR.genome.bam + direction: output + - name: "--bam_genome_rsem" + type: file + description: Genome BAM file (from RSEM) + default: RSEM/$id.genome.bam + direction: output + - name: "--bam_transcript_rsem" + type: file + description: Transcript BAM file (from RSEM) + default: RSEM/$id.transcript.bam + direction: output + + # Quantification (alignment) + - name: "--tpm_gene" + type: file + direction: output + default: transcript_quantification/gene_tpm.tsv + - name: "--counts_gene" + type: file + direction: output + default: transcript_quantification/gene_counts.tsv + - name: "--counts_gene_length_scaled" + type: file + direction: output + default: transcript_quantification/gene_counts_length_scaled.tsv + - name: "--counts_gene_scaled" + type: file + direction: output + default: transcript_quantification/gene_counts_scaled.tsv + - name: "--tpm_transcript" + type: file + direction: output + default: transcript_quantification/transcript_tpm.tsv + - name: "--counts_transcript" + type: file + direction: output + default: transcript_quantification/transcript_counts.tsv + - name: "--quant_merged_summarizedexperiment" + type: file + direction: output + default: transcript_quantification/summarizedexperiment + + # MarkDuplicates + - name: "--markduplicates_metrics" + type: file + direction: output + default: picard/$id.MarkDuplicates.metrics.txt + + # StringTie + - name: "--stringtie_transcript_gtf" + type: file + direction: output + default: stringtie/$id.transcripts.gtf + - name: "--stringtie_coverage_gtf" + type: file + direction: output + default: stringtie/$id.coverage.gtf + - name: "--stringtie_abundance" + type: file + direction: output + default: stringtie/$id.gene_abundance.txt + - name: "--stringtie_ballgown" + type: file + direction: output + default: stringtie/$id.ballgown + + # featureCounts + - name: "--featurecounts" + type: file + direction: output + default: featurecounts/$id.featureCounts.txt + - name: "--featurecounts_summary" + type: file + direction: output + default: featurecounts/$id.featureCounts.txt.summary + - name: "--featurecounts_multiqc" + type: file + direction: output + must_exist: false + default: featurecounts/$id.featureCounts_mqc.tsv + - name: "--featurecounts_rrna_multiqc" + type: file + direction: output + must_exist: false + default: featurecounts/$id.featureCounts_rrna_mqc.tsv + + # bedGraph + - name: "--bedgraph_forward" + type: file + direction: output + default: bedgraph/$id.forward.bedgraph + - name: "--bedgraph_reverse" + type: file + direction: output + default: bedgraph/$id.reverse.bedgraph + + # bigWig + - name: "--bigwig_forward" + type: file + direction: output + default: bigwig/$id.forward.bigwig + - name: "--bigwig_reverse" + type: file + direction: output + default: bigwig/$id.reverse.bigwig + + # preseq lc_extrap + - name: "--preseq_output" + type: file + direction: output + default: preseq/$id.lc_extrap.txt + + # RSeQC + - name: "--bamstat_output" + type: file + direction: output + required: false + description: Path to output file (txt) of mapping quality statistics + default: RSeQC/bamstat/$id.mapping_quality.txt + - name: "--strandedness_output" + type: file + direction: output + required: false + default: RSeQC/inferexperiment/$id.strandedness.txt + description: Path to output report (txt) of inferred strandedness + - name: "--inner_dist_output_stats" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/$id.inner_distance.stats + description: output file (txt) with summary statistics of inner distances of paired reads + - name: "--inner_dist_output_dist" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/txt/$id.inner_distance.txt + description: output file (txt) with inner distances of all paired reads + - name: "--inner_dist_output_freq" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/txt/$id.inner_distance_freq.txt + description: output file (txt) with frequencies of inner distances of all paired reads + - name: "--inner_dist_output_plot" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/pdf/$id.inner_distance_plot.pdf + description: output file (pdf) with histogram plot of of inner distances of all paired reads + - name: "--inner_dist_output_plot_r" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/rscript/$id.inner_distance_plot.r + description: output file (R) with script of histogram plot of of inner distances of all paired reads + - name: "--junction_annotation_output_log" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/log/$id.junction_annotation.log + description: output log of junction annotation script + - name: "--junction_annotation_output_plot_r" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/rscript/$id.junction_annotation_plot.r + description: R script to generate splice_junction and splice_events plot + - name: "--junction_annotation_output_junction_bed" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/bed/$id.junction_annotation.bed + description: junction annotation file (bed format) + - name: "--junction_annotation_output_junction_interact" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/bed/$id.junction_annotation.Interact.bed + description: interact file (bed format) of junctions. Can be uploaded to UCSC genome browser or converted to bigInteract (using bedToBigBed program) for visualization. + - name: "--junction_annotation_output_junction_sheet" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/xls/$id.junction_annotation.xls + description: junction annotation file (xls format) + - name: "--junction_annotation_output_splice_events_plot" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/pdf/$id.splice_events.pdf + description: plot of splice events (pdf) + - name: "--junction_annotation_output_splice_junctions_plot" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/pdf/$id.splice_junctions_plot.pdf + description: plot of junctions (pdf) + - name: "--junction_saturation_output_plot_r" + type: file + direction: output + required: false + default: RSeQC/junctionsaturation/rscript/$id.junction_saturation_plot.r + description: r script to generate junction_saturation_plot plot + - name: "--junction_saturation_output_plot" + type: file + direction: output + required: false + default: RSeQC/junctionsaturation/pdf/$id.junction_saturation_plot.pdf + description: plot of junction saturation (pdf + - name: "--read_distribution_output" + type: file + direction: output + required: false + default: RSeQC/readdistribution/$id.read_distribution.txt + description: output file (txt) of read distribution analysis. + - name: "--read_duplication_output_duplication_rate_plot_r" + type: file + direction: output + required: false + default: RSeQC/readduplication/rscrpt/$id.duplication_rate_plot.r + description: R script for generating duplication rate plot + - name: "--read_duplication_output_duplication_rate_plot" + type: file + direction: output + required: false + default: RSeQC/readduplication/pdf/$id.duplication_rate_plot.pdf + description: duplication rate plot (pdf) + - name: "--read_duplication_output_duplication_rate_mapping" + type: file + direction: output + required: false + default: RSeQC/readduplication/xls/$id.duplication_rate_mapping.xls + description: Summary of mapping-based read duplication + - name: "--read_duplication_output_duplication_rate_sequence" + type: file + direction: output + required: false + default: RSeQC/readduplication/xls/$id.duplication_rate_sequencing.xls + description: Summary of sequencing-based read duplication + - name: "--tin_output_summary" + type: file + direction: output + required: false + default: RSeQC/tin/txt/$id.tin_summary.txt + description: summary statistics (txt) of calculated TIN metrics + - name: "--tin_output_metrics" + type: file + direction: output + required: false + default: RSeQC/tin/xls/$id.tin.xls + description: file with TIN metrics (xls) + + # DupRadar + - name: "--dupradar_output_dupmatrix" + type: file + direction: output + required: false + default: dupradar/gene_data/$id.dup_matrix.txt + description: path to output file (txt) of duplicate tag counts + - name: "--dupradar_output_dup_intercept_mqc" + type: file + direction: output + required: false + default: dupradar/mqc_intercept/$id.dup_intercept_mqc.txt + description: path to output file (txt) of multiqc intercept value DupRadar + - name: "--dupradar_output_duprate_exp_boxplot" + type: file + direction: output + required: false + default: dupradar/box_plot/$id.duprate_exp_boxplot.pdf + description: path to output file (pdf) of distribution of expression box plot + - name: "--dupradar_output_duprate_exp_densplot" + type: file + direction: output + required: false + default: dupradar/scatter_plot/$id.duprate_exp_densityplot.pdf + description: path to output file (pdf) of 2D density scatter plot of duplicate tag counts + - name: "--dupradar_output_duprate_exp_denscurve_mqc" + type: file + direction: output + required: false + default: dupradar/density_curve/$id.duprate_exp_density_curve_mqc.pdf + description: path to output file (pdf) of density curve of gene duplication multiqc + - name: "--dupradar_output_expression_histogram" + type: file + direction: output + required: false + default: dupradar/histogram/$id.expression_hist.pdf + description: path to output file (pdf) of distribution of RPK values per gene histogram + - name: "--dupradar_output_intercept_slope" + type: file + direction: output + required: false + default: dupradar/intercept_slope/$id.intercept_slope.txt + + # Qualimap + - name: "--qualimap_output_pdf" + type: file + direction: output + required: false + must_exist: false + default: qualimap/$id.qualimap_output.pdf + - name: "--qualimap_output_dir" + type: file + direction: output + required: false + default: qualimap/$id + + # DESeq2 + - name: "--deseq2_output" + type: file + direction: output + default: deseq2_qc + - name: "--deseq2_output_pseudo" + type: file + direction: output + default: deseq2_qc_pseudo + + # MultiQC + - name: "--multiqc_report" + type: file + direction: output + default: multiqc/multiqc_report.html + - name: "--multiqc_data" + type: file + direction: output + default: multiqc/multiqc_data + - name: "--multiqc_plots" + type: file + direction: output + default: multiqc/multiqc_plots + - name: "--multiqc_versions" + type: file + direction: output + + # Quantification (pseudo alignment) + - name: "--pseudo_counts_gene" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts.tsv + - name: "--pseudo_counts_gene_length_scaled" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts_length_scaled.tsv + - name: "--pseudo_counts_gene_scaled" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts_scaled.tsv + - name: "--pseudo_tpm_gene" + type: file + direction: output + default: pseudo_alignment_quantification/gene_tpm.tsv + - name: "--pseudo_tpm_transcript" + type: file + direction: output + default: pseudo_alignment_quantification/transcript_tpm.tsv + - name: "--pseudo_counts_transcript" + type: file + direction: output + default: pseudo_alignment_quantification/transcript_counts.tsv + - name: "--pseudo_quant_merged_summarizedexperiment" + type: file + direction: output + default: pseudo_alignment_quantification/quant_merged_summarizedexperiment + +resources: + - type: nextflow_script + path: main.nf + entrypoint: run_wf + +test_resources: + - type: nextflow_script + path: test.nf + entrypoint: test_wf + +dependencies: + - name: demultiplex + repository: demultiplex + - name: workflows/rnaseq + repository: rnaseq + +repositories: + - name: demultiplex + type: vsh + repo: demultiplex + tag: v0.1.1 + - name: rnaseq + type: vsh + repo: rnaseq + tag: main + +runners: + - type: nextflow diff --git a/src/meta_workflows/demultiplex_rnaseq/example.yaml b/src/meta_workflows/demultiplex_rnaseq/example.yaml new file mode 100644 index 0000000..a623d87 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/example.yaml @@ -0,0 +1,10 @@ +input: "https://github.com/nf-core/test-datasets/raw/refs/heads/demultiplex/testdata/NovaSeq6000/200624_A00834_0183_BHMTFYDRXX.tar.gz" +sample_sheet: "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/demultiplex/testdata/NovaSeq6000/SampleSheet.csv" +fasta: "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/858/895/GCA_009858895.3_ASM985889v3/GCA_009858895.3_ASM985889v3_genomic.fna.gz" +gtf: "http://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz" +skip_bbsplit: true +skip_alignment: true +skip_deseq2_qc: true +min_trimmed_reads: 5000 +rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication" +multiqc_custom_config: "https://raw.githubusercontent.com/viash-hub/rnaseq/refs/heads/main/src/assets/multiqc_config.yml" \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/integration_test.sh b/src/meta_workflows/demultiplex_rnaseq/integration_test.sh new file mode 100755 index 0000000..8209841 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/integration_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +# Make sure the workflow is built +viash ns build --setup cb --parallel + +# export NXF_VER=24.04.4 + +set -eo pipefail + +nextflow \ + run . \ + -main-script src/meta_workflows/demultiplex_rnaseq/test.nf \ + -config src/config/labels.config \ + -entry test_wf \ + -resume \ + -profile docker,local \ + --publish_dir output \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf new file mode 100644 index 0000000..2a1d41f --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -0,0 +1,131 @@ +// Helper function to create identity mappings for state + def createIdentityMap(keys) { + return keys.collectEntries { [(it): it] } + } + +workflow run_wf { + take: + input_ch + + main: + // Define the required input and output keys + def rnaseqInputKeys = [ + "id", "fastq_1", "fastq_2", "fasta", "gtf", "gff", "transcript_fasta", + "additional_fasta", "gene_bed", "gencode", "skip_fastqc", "skip_trimming", + "trimmer", "min_trimmed_reads", "skip_bbsplit", "bbsplit_fasta_list", + "remove_ribo_rna", "ribo_database_manifest", "with_umi", "umitools_extract_method", + "umitools_bc_pattern", "umi_discard_read", "skip_alignment", "skip_pseudo_alignment", + "aligner", "pseudo_aligner", "min_mapped_reads", "skip_qc", "skip_preseq", + "skip_deseq2_qc", "skip_dupradar", "skip_qualimap", "skip_rseqc", + "skip_multiqc", "rseqc_modules", "multiqc_custom_config" + ] + + def rnaseqOutputKeys = [ + "output_fasta", "output_gtf", "output_transcript_fasta", "output_gene_bed", + "output_bbsplit_index", "output_star_index", "output_salmon_index", + "output_kallisto_index", "fastqc_html_1", "fastqc_html_2", "fastqc_zip_1", + "fastqc_zip_2", "output_fastq_1", "output_fastq_2", "trim_log_1", "trim_log_2", + "trim_zip_1", "trim_zip_2", "trim_html_1", "trim_html_2", "sortmerna_log", + "star_log", "genome_bam_sorted", "genome_bam_index", "genome_bam_stats", + "genome_bam_flagstat", "genome_bam_idxstats", "transcriptome_bam", + "transcriptome_bam_index", "transcriptome_bam_stats", "transcriptome_bam_flagstat", + "transcriptome_bam_idxstats", "salmon_quant_results", "pseudo_quant_results", + "markduplicates_metrics", "stringtie_transcript_gtf", "stringtie_coverage_gtf", + "stringtie_abundance", "stringtie_ballgown", "featurecounts", + "featurecounts_summary", "featurecounts_multiqc", "featurecounts_rrna_multiqc", + "bedgraph_forward", "bedgraph_reverse", "bigwig_forward", "bigwig_reverse", + "preseq_output", "bamstat_output", "strandedness_output", + "inner_dist_output_stats", "inner_dist_output_dist", "inner_dist_output_freq", + "inner_dist_output_plot", "inner_dist_output_plot_r", + "junction_annotation_output_log", "junction_annotation_output_plot_r", + "junction_annotation_output_junction_bed", + "junction_annotation_output_junction_interact", + "junction_annotation_output_junction_sheet", + "junction_annotation_output_splice_events_plot", + "junction_annotation_output_splice_junctions_plot", + "junction_saturation_output_plot_r", "junction_saturation_output_plot", + "read_distribution_output", "read_duplication_output_duplication_rate_plot_r", + "read_duplication_output_duplication_rate_plot", + "read_duplication_output_duplication_rate_mapping", + "read_duplication_output_duplication_rate_sequence", "tin_output_summary", + "tin_output_metrics", "dupradar_output_dupmatrix", + "dupradar_output_dup_intercept_mqc", "dupradar_output_duprate_exp_boxplot", + "dupradar_output_duprate_exp_densplot", + "dupradar_output_duprate_exp_denscurve_mqc", + "dupradar_output_expression_histogram", "dupradar_output_intercept_slope", + "qualimap_output_dir", "qualimap_output_pdf", "tpm_gene", "counts_gene", + "counts_gene_length_scaled", "counts_gene_scaled", "tpm_transcript", + "counts_transcript", "quant_merged_summarizedexperiment", "deseq2_output", + "pseudo_tpm_gene", "pseudo_counts_gene", "pseudo_counts_gene_length_scaled", + "pseudo_counts_gene_scaled", "pseudo_tpm_transcript", "pseudo_counts_transcript", + "pseudo_lengths_gene", "pseudo_lengths_transcript", + "pseudo_quant_merged_summarizedexperiment", "deseq2_output_pseudo", + "multiqc_report", "multiqc_data", "multiqc_plots" + ] + + output_ch = input_ch + | demultiplex.run( + fromState: [ + "input": "input", + "sample_sheet": "sample_sheet" + ], + toState: [ + "output_demultiplexed": "output" + ] + ) + + | flatMap { id, state -> + println "Processing sample sheet: $state.sample_sheet" + def sample_sheet = file(state.sample_sheet) + def original_id = id + def lines = sample_sheet.readLines() + + // Extract Sample_IDs + if (lines.indexOf('[Data]') < 0) { + println "Data section not found" + return + } + def data_lines = lines.drop(lines.indexOf('[Data]') + 2) + def samples = data_lines.collect { line -> + def columns = line.split(',') + return columns.size() > 1 ? columns[1].trim() : null + }.findAll { it != null } + + println "Looking for fastq files in ${state.output_demultiplexed}." + processed_samples = samples.collect { sample_id -> + def forward_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R1_(\d+)\.fastq\.gz$/ + def reverse_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R2_(\d+)\.fastq\.gz$/ + def forward_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ forward_regex} + def reverse_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ reverse_regex} + reverse_fastq = !reverse_fastq.isEmpty() ? reverse_fastq[0] : null + def fastqs_state = [ + "id": sample_id, + "fastq_1": forward_fastq[0], + "fastq_2": reverse_fastq, + "_meta": [ "join_id": original_id ] + ] + [sample_id, fastqs_state + state] + } + println "Finished processing sample sheet." + return processed_samples + } + + | rnaseq.run( + fromState: createIdentityMap(rnaseqInputKeys), + toState: createIdentityMap(rnaseqOutputKeys) + ) + + | map { id, state -> + def mod_state = state.findAll { key, value -> + value instanceof java.nio.file.Path && value.exists() + } + // Get the original join_id from the _meta if it exists, otherwise use current id + def join_id = state._meta?.join_id ?: id + [ id, mod_state + [ _meta: [join_id: join_id] ] ] + } + + | setState(createIdentityMap(rnaseqOutputKeys + ["_meta"])) + + emit: + output_ch +} \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/test.nf b/src/meta_workflows/demultiplex_rnaseq/test.nf new file mode 100644 index 0000000..70a769d --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/test.nf @@ -0,0 +1,48 @@ +nextflow.enable.dsl=2 +targetDir = params.rootDir + "/target" + +include { demultiplex_rnaseq } from targetDir + "/nextflow/demultiplex_rnaseq/main.nf" + +params.resources_test = "gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta/" + +workflow test_wf { + + resources_test_file = file(params.resources_test) + + output_ch = Channel.fromList([ + [ + id: "_test", + input: resources_test_file.resolve("200624_A00834_0183_BHMTFYDRXX.tar.gz"), + sample_sheet: resources_test_file.resolve("SampleSheet.csv"), + fasta: resources_test_file.resolve("GCA_009858895.3_ASM985889v3_genomic.fna.gz"), + gtf: resources_test_file.resolve("Sars_cov_2.ASM985889v3.101.gtf.gz"), + skip_bbsplit: true, + skip_alignment: true, + skip_deseq2_qc: true, + num_trimmed_reads: 5000, + rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication", + multiqc_custom_config: resources_test_file.resolve("multiqc_config.yml") + ] + ]) + + | map { state -> [state.id, state] } + + | demultiplex_rnaseq.run( + fromState: { id, state -> state }, + toState: { id, output, state -> output } + ) + + | view { output -> + assert output.size() == 2 : "Outputs should contain two elements; [id, state]" + + // check output + def state = output[1] + assert state instanceof Map : "State should be a map. Found: ${state}" + assert state.containsKey("output_fasta") : "Output should contain key 'output_fasta'." + assert state.output_fasta.isFile() : "'output_fasta' should be a file." + assert state.containsKey("output_gtf") : "Output should contain key 'output_gtf'." + assert state.output_gtf.isFile() : "'output_gtf' should be a file." + + "Output: $output" + } +} \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/test.sh b/src/meta_workflows/demultiplex_rnaseq/test.sh new file mode 100644 index 0000000..c280031 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/test.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +viash ns build --setup cb --parallel -q demultiplex_rnaseq + +nextflow run . \ + -main-script target/nextflow/demultiplex_rnaseq/main.nf \ + -params-file src/meta_workflows/demultiplex_rnaseq/example.yaml \ + -profile docker \ + --publish_dir test_results \ + --resume