From 2978359273acb141edacbf571715f291b3ce188c Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Wed, 13 Nov 2024 11:39:41 +0100 Subject: [PATCH 01/16] meta workflow (wip) --- .../demultiplex_rnaseq/config.vsh.yaml | 909 ++++++++++++++++++ src/meta_workflows/demultiplex_rnaseq/main.nf | 182 ++++ src/meta_workflows/demultiplex_rnaseq/test.sh | 15 + 3 files changed, 1106 insertions(+) create mode 100644 src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml create mode 100644 src/meta_workflows/demultiplex_rnaseq/main.nf create mode 100644 src/meta_workflows/demultiplex_rnaseq/test.sh diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml new file mode 100644 index 0000000..922f613 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -0,0 +1,909 @@ +name: demultiplex_rnaseq +description: Demultiplexing of raw sequencing data and RNA sequencing analysis. + +argument_groups: + + - name: Demultiplex Input arguments + arguments: + - name: --input + description: Directory containing raw sequencing data + type: file + required: true + - name: --sample_sheet + description: | + Sample sheet as input for BCL Convert. If not specified, + will try to autodetect the sample sheet in the input directory + type: file + required: false + + - name: Demultiplex Output arguments + arguments: + - name: --output_demultiplexed + description: Directory to write demultiplexed output to + type: file + direction: output + required: true + default: "$id/demultiplex" + - name: --output_falco + description: Directory to write Falco output to + type: file + direction: output + required: false + default: "$id/falco" + - name: --output_multiqc + description: Directory to write MultiQC output to + type: file + direction: output + required: false + default: "$id/multiqc_report.html" + + - name: Reference genome options + arguments: + - name: "--fasta" + type: file + description: Path to FASTA genome file. + required: true + - name: "--gtf" + type: file + description: Path to GTF annotation file. This parameter is *mandatory* if --genome is not specified. + required: false + - name: "--gff" + type: file + description: Path to GFF3 annotation file. Required if "--gtf" is not specified. + required: false + - name: "--additional_fasta" + type: file + description: FASTA file to concatenate to genome FASTA file e.g. containing spike-in sequences. + - name: "--transcript_fasta" + type: file + description: Path to FASTA transcriptome file. + - name: "--gene_bed" + type: file + description: Path to BED file containing gene intervals. This will be created from the GTF file if not specified. + - name: "--star_index" + type: file + description: Path to directory or tar.gz archive for pre-built STAR index. + - name: "--rsem_index" + type: file + description: Path to directory or tar.gz archive for pre-built RSEM index. + - name: "--salmon_index" + type: file + description: Path to directory or tar.gz archive for pre-built Salmon index. + - name: "--kallisto_index" + type: file + description: Path to directory or tar.gz archive for pre-built Kallisto index. + - name: "--gencode" + type: boolean_true + description: Specify if the GTF annotation is in GENCODE format. + - name: "--gtf_extra_attributes" + type: string + description: Additional gene identifiers from the input GTF file when running Salmon. More than one value can be specified separated by comma. + default: gene_name + - name: "--gtf_group_features" + type: string + description: Define the attribute type used to group features in the GTF file when running Salmon. + default: gene_id + - name: "--featurecounts_group_type" + type: string + description: The attribute type used to group feature types in the GTF file when generating the biotype plot with featureCounts. + default: gene_biotype + - name: "--featurecounts_feature_type" + type: string + description: By default, the pipeline assigns reads based on the 'exon' attribute within the GTF file. + default: exon + + - name: Read trimming options + arguments: + - name: "--trimmer" + type: string + description: Specify the trimming tool to use. + choices: ["trimgalore", "fastp"] + default: "trimgalore" + - name: "--extra_trimgalore_args" + type: string + description: Extra arguments to pass to Trim Galore! command in addition to defaults defined by the pipeline. + - name: "--extra_fastp_args" + type: string + description: Extra arguments to pass to fastp command in addition to defaults defined by the pipeline. + - name: "--min_trimmed_reads" + type: integer + description: Minimum number of trimmed reads below which samples are removed from further processing. Some downstream steps in the pipeline will fail if this threshold is too low. + default: 10000 + + - name: Read filtering options + arguments: + - name: "--bbsplit_fasta_list" + type: file + description: Path to comma-separated file containing a list of reference genomes to filter reads against with BBSplit. To use BBSplit, "--skip_bbsplit" must be explicitly set to "false". The file should contain 2 (comma separated) columns - short name and full path to reference genome(s) + - name: "--bbsplit_index" + type: file + description: Path to directory or tar.gz archive for pre-built BBSplit index. + - name: "--remove_ribo_rna" + type: boolean_true + description: Enable the removal of reads derived from ribosomal RNA using SortMeRNA. + - name: "--ribo_database_manifest" + type: file + description: Text file containing paths to fasta files (one per line) that will be used to create the database for SortMeRNA. + default: src/assets/rrna-db-defaults.txt + + - name: UMI options + arguments: + - name: "--with_umi" + type: boolean_true + description: Enable UMI-based read deduplication. + - name: "--umitools_extract_method" + type: string + description: UMI pattern to use. + default: "string" + choices: [string, regex] + - name: "--umitools_bc_pattern" + type: string + description: The UMI barcode pattern to use e.g. 'NNNNNN' indicates that the first 6 nucleotides of the read are from the UMI. + - name: "--umitools_bc_pattern2" + type: string + description: The UMI barcode pattern to use if the UMI is located in read 2. + - name: "--umi_discard_read" + type: integer + description: After UMI barcode extraction discard either R1 or R2 by setting this parameter to 1 or 2, respectively. + choices: [0, 1, 2] + default: 0 + - name: "--umitools_umi_separator" + type: string + description: The character that separates the UMI in the read name. Most likely a colon if you skipped the extraction with UMI-tools and used other software. + default: "_" + - name: "--umitools_grouping_method" + type: string + description: Method to use to determine read groups by subsuming those with similar UMIs. All methods start by identifying the reads with the same mapping position, but treat similar yet nonidentical UMIs differently. + choices: [ "unique", "percentile", "cluster", "adjacency", "directional" ] + default: "directional" + - name: "--umi_dedup_stats" + type: boolean_true + description: Generate output stats when running "umi_tools dedup". + + - name: Alignment options + arguments: + - name: "--aligner" + type: string + description: Specifies the alignment algorithm to use - available options are 'star_salmon', 'star_rsem' and 'hisat2'. + choices: [star_salmon, star_rsem, hisat2] + default: "star_salmon" + - name: "--pseudo_aligner" + type: string + description: Specifies the pseudo aligner to use - available options are 'salmon'. Runs in addition to '--aligner'. + choices: [salmon, kallisto] + default: "salmon" + - name: "--pseudo_aligner_kmer_size" + type: integer + description: Kmer length passed to indexing step of pseudoaligners. + default: 31 + - name: "--kallisto_quant_fragment_length" + type: integer + description: For single-end mode only, the estimated average fragment length to use for quantification with Kallisto. + - name: "--kallisto_quant_fragment_length_sd" + type: integer + description: For single-end mode only, the estimated standard deviation of the fragment length for quantification with Kallisto. + - name: "--bam_csi_index" + type: boolean_true + description: Create a CSI index for BAM files instead of the traditional BAI index. This will be required for genomes with larger chromosome sizes. + - name: "--salmon_quant_libtype" + type: string + description: Override Salmon library type inferred based on strandedness defined in meta object. + - name: "--min_mapped_reads" + type: integer + description: Minimum percentage of uniquely mapped reads below which samples are removed from further processing. + default: 5 + - name: "--stringtie_ignore_gtf" + type: boolean_true + description: Perform reference-guided de novo assembly of transcripts using StringTie, i.e. don't restrict to those in GTF file. + - name: "--save_unaligned" + type: boolean_true + description: Where possible, save unaligned reads from either STAR, HISAT2 or Salmon to the results directory. + - name: "--save_align_intermeds" + type: boolean_true + description: Save the intermediate BAM files from the alignment step. + - name: "--skip_alignment" + type: boolean_true + description: Skip all of the alignment-based processes within the pipeline. + - name: "--skip_pseudo_alignment" + type: boolean_true + description: Skip all of the pseudo-alignment-based processes within the pipeline. + - name: --extra_rsem_calculate_expression_args + type: string + description: Extra arguments to pass to rsem-calculate-expression command in addition to defaults defined by the pipeline. + default: '--star --star-output-genome-bam --star-gzipped-read-file --estimate-rspd --seed 1' + + - name: Process skipping options + arguments: + - name: "--skip_fastqc" + type: boolean + description: Skip FatQC step. + default: false + - name: "--skip_trimming" + type: boolean + description: Skip the adapter trimming step. + default: false + - name: "--skip_bbsplit" + type: boolean_true + description: Skip BBSplit for removal of non-reference genome reads. + - name: "--skip_umi_extract" + type: boolean + description: Skip umi_tools extract step. + default: false + - name: "--skip_qc" + type: boolean_true + description: Skip all QC steps except for MultiQC. + - name: "--skip_markduplicates" + type: boolean_true + description: Skip picard MarkDuplicates step. + - name: "--skip_stringtie" + type: boolean_true + description: Skip StringTie. + - name: "--skip_biotype_qc" + type: boolean_true + description: Skip additional featureCounts process for biotype QC. + - name: "--skip_bigwig" + type: boolean_true + description: Skip bigWig file creation. + - name: "--skip_preseq" + type: boolean_true + description: Skip Preseq. + - name: "--skip_deseq2_qc" + type: boolean_true + description: Skip DESeq2 PCA and heatmap plotting. + - name: skip_dupradar + type: boolean_true + description: Skip dupRadar. + - name: skip_rseqc + type: boolean_true + description: Skip RSeQC. + - name: skip_multiqc + type: boolean_true + description: Skip MultiQC. + + - name: Other process arguments + arguments: + - name: "--extra_fq_subsample_args" + type: string + default: ' --record-count 1000000 --seed 1' + description: Extra arguments to pass to fq subsample command in addition to defaults defined by the pipeline. + - name: "--extra_picard_args" + type: string + default: ' --ASSUME_SORTED true --REMOVE_DUPLICATES false --VALIDATION_STRINGENCY LENIENT --TMP_DIR tmp' + description: Extra arguments to pass to picard MarkDuplicates command in addition to defaults defined by the pipeline. + - name: "--extra_bedtools_args" + type: string + default: ' -split -du' + description: Extra arguments to pass to bedtools genomecov command in addition to defaults defined by the pipeline. + - name: "--extra_preseq_args" + type: string + description: Extra arguments to pass to preseq lc_extrap command in addition to defaults defined by the pipeline + default: '-verbose -seed 1 -seg_len 100000000' + - name: "--deseq2_vst" + type: boolean + default: true + description: Use vst transformation instead of rlog with DESeq2 + - name: "--rseqc_modules" + type: string + multiple: true + multiple_sep: "," + description: Specify the RSeQC modules to run_wf + default: bam_stat,inner_distance,infer_experiment,junction_annotation,junction_saturation,read_distribution,read_duplication + choices: [ "bam_stat", "inner_distance", "infer_experiment", "junction_annotation", "junction_saturation", "read_distribution", "read_duplication", "tin" ] + + - name: Multiqc paramenters + arguments: + - name: "--multiqc_custom_config" + type: file + default: src/assets/multiqc_config.yml + - name: "--multiqc_title" + type: string + - name: "--multiqc_methods_description" + type: file + default: src/assets/methods_description_template.yml + + - name: Output + arguments: + # Reference files + - name: "--output_fasta" + type: file + direction: output + default: reference/genome.fasta + - name: "--output_gtf" + type: file + direction: output + default: reference/gene_annotation.gtf + - name: "--output_transcript_fasta" + type: file + direction: output + default: reference/transcriptome.fasta + - name: "--output_gene_bed" + type: file + direction: output + default: reference/gene_annotation.bed + - name: "--output_star_index" + type: file + direction: output + description: Path to STAR index. + default: reference/index/STAR + - name: "--output_salmon_index" + type: file + direction: output + description: Path to Salmon index. + default: reference/index/Salmon + - name: "--output_bbsplit_index" + type: file + direction: output + description: Path to BBSplit index. + default: reference/index/BBSplit + - name: "--output_kallisto_index" + type: file + direction: output + description: Path to Kallisto index. + default: reference/index/Kallisto + + # fastq + - name: "--output_fastq_1" + type: file + direction: output + required: false + must_exist: false + description: Path to output directory + default: fastq/$id.read_1.fastq + - name: "--output_fastq_2" + type: file + direction: output + required: false + must_exist: false + description: Path to output directory + default: fastq/$id.read_2.fastq + + # FastQC + - name: "--fastqc_html_1" + type: file + direction: output + description: FastQC HTML report for read 1. + required: false + must_exist: false + default: fastqc_raw/$id.read_1.fastqc.html + - name: "--fastqc_html_2" + type: file + direction: output + description: FastQC HTML report for read 2. + required: false + must_exist: false + default: fastqc_raw/$id.read_2.fastqc.html + - name: "--fastqc_zip_1" + type: file + direction: output + description: FastQC report archive for read 1. + required: false + must_exist: false + default: fastqc_raw/$id.read_1.fastqc.zip + - name: "--fastqc_zip_2" + type: file + direction: output + description: FastQC report archive for read 2. + required: false + must_exist: false + default: fastqc_raw/$id.read_2.fastqc.zip + - name: "--trim_html_1" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_1.trimmed_fastqc.html + - name: "--trim_html_2" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_2.trimmed_fastqc.html + - name: "--trim_zip_1" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_1.trimmed_fastqc.zip + - name: "--trim_zip_2" + type: file + direction: output + required: false + must_exist: false + default: fastqc_trim/$id.read_2.trimmed_fastqc.zip + + # TrimGalore + - name: "--trim_log_1" + type: file + direction: output + required: false + must_exist: false + default: trimgalore/$id.read_1.trimming_report.txt + - name: "--trim_log_2" + type: file + direction: output + required: false + must_exist: false + default: trimgalore/$id.read_2.trimming_report.txt + + # fastp + - name: --fastp_trim_json + type: file + description: The fastp json format report file name + default: fastp/$id_out.json + direction: output + - name: --fastp_trim_html + type: file + description: The fastp html format report file name + default: fastp/$id_out.html + direction: output + + # SortMeRNA + - name: "--sortmerna_log" + type: file + direction: output + default: sortmerna/$id.log + required: false + must_exist: false + description: Sortmerna log file. + + # STAR + - name: "--star_alignment" + type: file + direction: output + default: STAR/$id + - name: "--genome_bam_sorted" + type: file + direction: output + default: STAR/genome_processed/$id.genome.bam + - name: "--genome_bam_index" + type: file + direction: output + default: STAR/genome_processed/$id.genome.bam.bai + - name: "--transcriptome_bam" + type: file + direction: output + default: STAR/transcriptome_processed/$id.transcriptome.bam + - name: "--transcriptome_bam_index" + type: file + direction: output + default: STAR/transcriptome_processed/$id.transcriptome.bam.bai + - name: "--star_log" + type: file + direction: output + default: STAR/log/$id.log + + # samtools + - name: "--genome_bam_stats" + type: file + direction: output + default: samtools_stats/$id.genome.stats + - name: "--genome_bam_flagstat" + type: file + direction: output + default: samtools_stats/$id.genome.flagstat + - name: "--genome_bam_idxstats" + type: file + direction: output + default: samtools_stats/$id.genome.idxstats + - name: "--transcriptome_bam_stats" + type: file + direction: output + default: samtools_stats/$id.transcriptome.stats + - name: "--transcriptome_bam_flagstat" + type: file + direction: output + default: samtools_stats/$id.transcriptome.flagstat + - name: "--transcriptome_bam_idxstats" + type: file + direction: output + default: samtools_stats/$id.transcriptome.idxstats + + # Transcript quantification + - name: "--salmon_quant_results" + type: file + direction: output + default: STAR_Salmon/$id + - name: "--salmon_quant_results_file" + type: file + direction: output + default: STAR_Salmon/$id/quant.sf + - name: "--pseudo_quant_results" + type: file + direction: output + default: Pseudo_align_quant/$id + + # RSEM + - name: "--rsem_counts_gene" + type: file + description: Expression counts on gene level + default: RSEM/$id.genes.results + direction: output + - name: "--rsem_counts_transcripts" + type: file + description: Expression counts on transcript level + default: RSEM/$id.isoforms.results + direction: output + - name: "--bam_star_rsem" + type: file + description: BAM file generated by STAR (from RSEM) + default: RSEM/$id.STAR.genome.bam + direction: output + - name: "--bam_genome_rsem" + type: file + description: Genome BAM file (from RSEM) + default: RSEM/$id.genome.bam + direction: output + - name: "--bam_transcript_rsem" + type: file + description: Transcript BAM file (from RSEM) + default: RSEM/$id.transcript.bam + direction: output + + # Quantification (alignment) + - name: "--tpm_gene" + type: file + direction: output + default: transcript_quantification/gene_tpm.tsv + - name: "--counts_gene" + type: file + direction: output + default: transcript_quantification/gene_counts.tsv + - name: "--counts_gene_length_scaled" + type: file + direction: output + default: transcript_quantification/gene_counts_length_scaled.tsv + - name: "--counts_gene_scaled" + type: file + direction: output + default: transcript_quantification/gene_counts_scaled.tsv + - name: "--tpm_transcript" + type: file + direction: output + default: transcript_quantification/transcript_tpm.tsv + - name: "--counts_transcript" + type: file + direction: output + default: transcript_quantification/transcript_counts.tsv + - name: "--quant_merged_summarizedexperiment" + type: file + direction: output + default: transcript_quantification/summarizedexperiment + + # MarkDuplicates + - name: "--markduplicates_metrics" + type: file + direction: output + default: picard/$id.MarkDuplicates.metrics.txt + + # StringTie + - name: "--stringtie_transcript_gtf" + type: file + direction: output + default: stringtie/$id.transcripts.gtf + - name: "--stringtie_coverage_gtf" + type: file + direction: output + default: stringtie/$id.coverage.gtf + - name: "--stringtie_abundance" + type: file + direction: output + default: stringtie/$id.gene_abundance.txt + - name: "--stringtie_ballgown" + type: file + direction: output + default: stringtie/$id.ballgown + + # featureCounts + - name: "--featurecounts" + type: file + direction: output + default: featurecounts/$id.featureCounts.txt + - name: "--featurecounts_summary" + type: file + direction: output + default: featurecounts/$id.featureCounts.txt.summary + - name: "--featurecounts_multiqc" + type: file + direction: output + must_exist: false + default: featurecounts/$id.featureCounts_mqc.tsv + - name: "--featurecounts_rrna_multiqc" + type: file + direction: output + must_exist: false + default: featurecounts/$id.featureCounts_rrna_mqc.tsv + + # bedGraph + - name: "--bedgraph_forward" + type: file + direction: output + default: bedgraph/$id.forward.bedgraph + - name: "--bedgraph_reverse" + type: file + direction: output + default: bedgraph/$id.reverse.bedgraph + + # bigWig + - name: "--bigwig_forward" + type: file + direction: output + default: bigwig/$id.forward.bigwig + - name: "--bigwig_reverse" + type: file + direction: output + default: bigwig/$id.reverse.bigwig + + # preseq lc_extrap + - name: "--preseq_output" + type: file + direction: output + default: preseq/$id.lc_extrap.txt + + # RSeQC + - name: "--bamstat_output" + type: file + direction: output + required: false + description: Path to output file (txt) of mapping quality statistics + default: RSeQC/bamstat/$id.mapping_quality.txt + - name: "--strandedness_output" + type: file + direction: output + required: false + default: RSeQC/inferexperiment/$id.strandedness.txt + description: Path to output report (txt) of inferred strandedness + - name: "--inner_dist_output_stats" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/$id.inner_distance.stats + description: output file (txt) with summary statistics of inner distances of paired reads + - name: "--inner_dist_output_dist" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/txt/$id.inner_distance.txt + description: output file (txt) with inner distances of all paired reads + - name: "--inner_dist_output_freq" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/txt/$id.inner_distance_freq.txt + description: output file (txt) with frequencies of inner distances of all paired reads + - name: "--inner_dist_output_plot" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/pdf/$id.inner_distance_plot.pdf + description: output file (pdf) with histogram plot of of inner distances of all paired reads + - name: "--inner_dist_output_plot_r" + type: file + direction: output + must_exist: false + default: RSeQC/innerdistance/rscript/$id.inner_distance_plot.r + description: output file (R) with script of histogram plot of of inner distances of all paired reads + - name: "--junction_annotation_output_log" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/log/$id.junction_annotation.log + description: output log of junction annotation script + - name: "--junction_annotation_output_plot_r" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/rscript/$id.junction_annotation_plot.r + description: R script to generate splice_junction and splice_events plot + - name: "--junction_annotation_output_junction_bed" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/bed/$id.junction_annotation.bed + description: junction annotation file (bed format) + - name: "--junction_annotation_output_junction_interact" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/bed/$id.junction_annotation.Interact.bed + description: interact file (bed format) of junctions. Can be uploaded to UCSC genome browser or converted to bigInteract (using bedToBigBed program) for visualization. + - name: "--junction_annotation_output_junction_sheet" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/xls/$id.junction_annotation.xls + description: junction annotation file (xls format) + - name: "--junction_annotation_output_splice_events_plot" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/pdf/$id.splice_events.pdf + description: plot of splice events (pdf) + - name: "--junction_annotation_output_splice_junctions_plot" + type: file + direction: output + required: false + default: RSeQC/junctionannotation/pdf/$id.splice_junctions_plot.pdf + description: plot of junctions (pdf) + - name: "--junction_saturation_output_plot_r" + type: file + direction: output + required: false + default: RSeQC/junctionsaturation/rscript/$id.junction_saturation_plot.r + description: r script to generate junction_saturation_plot plot + - name: "--junction_saturation_output_plot" + type: file + direction: output + required: false + default: RSeQC/junctionsaturation/pdf/$id.junction_saturation_plot.pdf + description: plot of junction saturation (pdf + - name: "--read_distribution_output" + type: file + direction: output + required: false + default: RSeQC/readdistribution/$id.read_distribution.txt + description: output file (txt) of read distribution analysis. + - name: "--read_duplication_output_duplication_rate_plot_r" + type: file + direction: output + required: false + default: RSeQC/readduplication/rscrpt/$id.duplication_rate_plot.r + description: R script for generating duplication rate plot + - name: "--read_duplication_output_duplication_rate_plot" + type: file + direction: output + required: false + default: RSeQC/readduplication/pdf/$id.duplication_rate_plot.pdf + description: duplication rate plot (pdf) + - name: "--read_duplication_output_duplication_rate_mapping" + type: file + direction: output + required: false + default: RSeQC/readduplication/xls/$id.duplication_rate_mapping.xls + description: Summary of mapping-based read duplication + - name: "--read_duplication_output_duplication_rate_sequence" + type: file + direction: output + required: false + default: RSeQC/readduplication/xls/$id.duplication_rate_sequencing.xls + description: Summary of sequencing-based read duplication + - name: "--tin_output_summary" + type: file + direction: output + required: false + default: RSeQC/tin/txt/$id.tin_summary.txt + description: summary statistics (txt) of calculated TIN metrics + - name: "--tin_output_metrics" + type: file + direction: output + required: false + default: RSeQC/tin/xls/$id.tin.xls + description: file with TIN metrics (xls) + + # DupRadar + - name: "--dupradar_output_dupmatrix" + type: file + direction: output + required: false + default: dupradar/gene_data/$id.dup_matrix.txt + description: path to output file (txt) of duplicate tag counts + - name: "--dupradar_output_dup_intercept_mqc" + type: file + direction: output + required: false + default: dupradar/mqc_intercept/$id.dup_intercept_mqc.txt + description: path to output file (txt) of multiqc intercept value DupRadar + - name: "--dupradar_output_duprate_exp_boxplot" + type: file + direction: output + required: false + default: dupradar/box_plot/$id.duprate_exp_boxplot.pdf + description: path to output file (pdf) of distribution of expression box plot + - name: "--dupradar_output_duprate_exp_densplot" + type: file + direction: output + required: false + default: dupradar/scatter_plot/$id.duprate_exp_densityplot.pdf + description: path to output file (pdf) of 2D density scatter plot of duplicate tag counts + - name: "--dupradar_output_duprate_exp_denscurve_mqc" + type: file + direction: output + required: false + default: dupradar/density_curve/$id.duprate_exp_density_curve_mqc.pdf + description: path to output file (pdf) of density curve of gene duplication multiqc + - name: "--dupradar_output_expression_histogram" + type: file + direction: output + required: false + default: dupradar/histogram/$id.expression_hist.pdf + description: path to output file (pdf) of distribution of RPK values per gene histogram + - name: "--dupradar_output_intercept_slope" + type: file + direction: output + required: false + default: dupradar/intercept_slope/$id.intercept_slope.txt + + # Qualimap + - name: "--qualimap_output_pdf" + type: file + direction: output + required: false + must_exist: false + default: qualimap/$id.qualimap_output.pdf + - name: "--qualimap_output_dir" + type: file + direction: output + required: false + default: qualimap/$id + + # DESeq2 + - name: "--deseq2_output" + type: file + direction: output + default: deseq2_qc + + # MultiQC + - name: "--multiqc_report" + type: file + direction: output + default: multiqc/multiqc_report.html + - name: "--multiqc_data" + type: file + direction: output + default: multiqc/multiqc_data + - name: "--multiqc_plots" + type: file + direction: output + default: multiqc/multiqc_plots + - name: "--multiqc_versions" + type: file + direction: output + + # Quantification (pseudo alignment) + - name: "--pseudo_counts_gene" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts.tsv + - name: "--pseudo_counts_gene_length_scaled" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts_length_scaled.tsv + - name: "--pseudo_counts_gene_scaled" + type: file + direction: output + default: pseudo_alignment_quantification/gene_counts_scaled.tsv + - name: "--pseudo_tpm_transcript" + type: file + direction: output + default: pseudo_alignment_quantification/transcript_tpm.tsv + - name: "--pseudo_counts_transcript" + type: file + direction: output + default: pseudo_alignment_quantification/transcript_counts.tsv + - name: "--pseudo_quant_merged_summarizedexperiment" + type: file + direction: output + default: pseudo_alignment_quantification/quant_merged_summarizedexperiment + +resources: + - type: nextflow_script + path: main.nf + entrypoint: run_wf + +dependencies: + - name: demultiplex + repository: demultiplex + - name: workflows/rnaseq + repository: rnaseq + +repositories: + - name: demultiplex + type: vsh + repo: demultiplex + tag: v0.1.1 + - name: rnaseq + type: vsh + repo: rnaseq + tag: main + +runners: + - type: nextflow diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf new file mode 100644 index 0000000..916296e --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -0,0 +1,182 @@ +workflow run_wf { + take: + input_ch + + main: + output_ch = input_ch + + | demultiplex.run( + fromState: [ + "input": "input", + "sample_sheet": "sample_sheet" + ], + toState: [ + "output_demultiplexed": "output", + // "output_falco": "output_falco", + // "output_multiqc": "output_multiqc" + ], + ) + + | flatMap { id, state -> + println "Processing sample sheet: $state.sample_sheet" + def sample_sheet = file(state.sample_sheet) + def samples = ["Undetermined"] + def original_id = id + def lines = sample_sheet.readLines() + + // Extract Sample_IDs + if (lines.indexOf('[Data]') < 0) { + println "Data section not found" + return + } + def data_lines = lines.drop(lines.indexOf('[Data]') + 2) + def sample_ids = data_lines.collect { line -> + def columns = line.split(',') + return columns.size() > 1 ? columns[1].trim() : null + }.findAll { it != null } // Remove nulls + + samples += sample_ids + println "Looking for fastq files in ${state.output_demultiplexed}." + def allfastqs = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ /^.+\.fastq.gz$/} + println "Found ${allfastqs.size()} fastq files, matching them to the following samples: ${samples}." + processed_samples = samples.collect { sample_id -> + def forward_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R1_(\d+)\.fastq\.gz$/ + def reverse_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R2_(\d+)\.fastq\.gz$/ + def forward_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ forward_regex} + def reverse_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ reverse_regex} + reverse_fastq = !reverse_fastq.isEmpty() ? reverse_fastq[0] : null + def fastqs_state = [ + "id": sample_id, + "fastq_1": forward_fastq[0], + "fastq_2": reverse_fastq, + "_meta": [ "join_id": original_id ] + ] + [sample_id, fastqs_state + state] + } + println "Finished processing sample sheet." + return processed_samples + } + | view + + | rnaseq.run( + fromState: [ + "id": "id", + "fastq_1": "fastq_1", + "fastq_2": "fastq_2", + "fasta": "fasta", + "gtf": "gtf", + "transcript_fasta": "transcript_fasta", + "skip_bbsplit": "skip_bbsplit", + "skip_alignment": "skip_alignment" + ], + toState: [ + "output_fasta": "output_fasta", + "output_gtf": "output_gtf", + "output_transcript_fasta": "output_transcript_fasta", + "output_gene_bed": "output_gene_bed", + "output_bbsplit_index": "output_bbsplit_index", + "output_star_index": "output_star_index", + "output_salmon_index": "output_salmon_index", + "output_kallisto_index": "output_kallisto_index", + "fastqc_html_1": "fastqc_html_1", + "fastqc_html_2": "fastqc_html_2", + "fastqc_zip_1": "fastqc_zip_1", + "fastqc_zip_2": "fastqc_zip_2", + "output_fastq_1": "output_fastq_1", + "output_fastq_2": "output_fastq_2", + "trim_log_1": "trim_log_1", + "trim_log_2": "trim_log_2", + "trim_zip_1": "trim_zip_1", + "trim_zip_2": "trim_zip_2", + "trim_html_1": "trim_html_1", + "trim_html_2": "trim_html_2", + "sortmerna_log": "sortmerna_log", + "star_log": "star_log", + "genome_bam_sorted": "genome_bam_sorted", + "genome_bam_index": "genome_bam_index", + "genome_bam_stats": "genome_bam_stats", + "genome_bam_flagstat": "genome_bam_flagstat", + "genome_bam_idxstats": "genome_bam_idxstats", + "transcriptome_bam": "transcriptome_bam", + "transcriptome_bam_index": "transcriptome_bam_index", + "transcriptome_bam_stats": "transcriptome_bam_stats", + "transcriptome_bam_flagstat": "transcriptome_bam_flagstat", + "transcriptome_bam_idxstats": "transcriptome_bam_idxstats", + "salmon_quant_results": "salmon_quant_results", + "pseudo_quant_results": "pseudo_quant_results", + "markduplicates_metrics": "markduplicates_metrics", + "stringtie_transcript_gtf": "stringtie_transcript_gtf", + "stringtie_coverage_gtf": "stringtie_coverage_gtf", + "stringtie_abundance": "stringtie_abundance", + "stringtie_ballgown": "stringtie_ballgown", + "featurecounts": "featurecounts", + "featurecounts_summary": "featurecounts_summary", + "featurecounts_multiqc": "featurecounts_multiqc", + "featurecounts_rrna_multiqc": "featurecounts_rrna_multiqc", + "bedgraph_forward": "bedgraph_forward", + "bedgraph_reverse": "bedgraph_reverse", + "bigwig_forward": "bigwig_forward", + "bigwig_reverse": "bigwig_reverse", + "preseq_output": "preseq_output", + "bamstat_output": "bamstat_output", + "strandedness_output": "strandedness_output", + "inner_dist_output_stats": "inner_dist_output_stats", + "inner_dist_output_dist": "inner_dist_output_dist", + "inner_dist_output_freq": "inner_dist_output_freq", + "inner_dist_output_plot": "inner_dist_output_plot", + "inner_dist_output_plot_r": "inner_dist_output_plot_r", + "junction_annotation_output_log": "junction_annotation_output_log", + "junction_annotation_output_plot_r": "junction_annotation_output_plot_r", + "junction_annotation_output_junction_bed": "junction_annotation_output_junction_bed", + "junction_annotation_output_junction_interact": "junction_annotation_output_junction_interact", + "junction_annotation_output_junction_sheet": "junction_annotation_output_junction_sheet", + "junction_annotation_output_splice_events_plot": "junction_annotation_output_splice_events_plot", + "junction_annotation_output_splice_junctions_plot": "junction_annotation_output_splice_junctions_plot", + "junction_saturation_output_plot_r": "junction_saturation_output_plot_r", + "junction_saturation_output_plot": "junction_saturation_output_plot", + "read_distribution_output": "read_distribution_output", + "read_duplication_output_duplication_rate_plot_r": "read_duplication_output_duplication_rate_plot_r", + "read_duplication_output_duplication_rate_plot": "read_duplication_output_duplication_rate_plot", + "read_duplication_output_duplication_rate_mapping": "read_duplication_output_duplication_rate_mapping", + "read_duplication_output_duplication_rate_sequence": "read_duplication_output_duplication_rate_sequence", + "tin_output_summary": "tin_output_summary", + "tin_output_metrics": "tin_output_metrics", + "dupradar_output_dupmatrix": "dupradar_output_dupmatrix", + "dupradar_output_dup_intercept_mqc": "dupradar_output_dup_intercept_mqc", + "dupradar_output_duprate_exp_boxplot": "dupradar_output_duprate_exp_boxplot", + "dupradar_output_duprate_exp_densplot": "dupradar_output_duprate_exp_densplot", + "dupradar_output_duprate_exp_denscurve_mqc": "dupradar_output_duprate_exp_denscurve_mqc", + "dupradar_output_expression_histogram": "dupradar_output_expression_histogram", + "dupradar_output_intercept_slope": "dupradar_output_intercept_slope", + "qualimap_output_dir": "qualimap_output_dir", + "qualimap_output_pdf": "qualimap_output_pdf", + "tpm_gene": "tpm_gene", + "counts_gene": "counts_gene", + "counts_gene_length_scaled": "counts_gene_length_scaled", + "counts_gene_scaled": "counts_gene_scaled", + "tpm_transcript": "tpm_transcript", + "counts_transcript": "counts_transcript", + "quant_merged_summarizedexperiment": "quant_merged_summarizedexperiment", + "deseq2_output": "deseq2_output", + "pseudo_tpm_gene": "pseudo_tpm_gene", + "pseudo_counts_gene": "pseudo_counts_gene", + "pseudo_counts_gene_length_scaled": "pseudo_counts_gene_length_scaled", + "pseudo_counts_gene_scaled": "pseudo_counts_gene_scaled", + "pseudo_tpm_transcript": "pseudo_tpm_transcript", + "pseudo_counts_transcript": "pseudo_counts_transcript", + "pseudo_lengths_gene": "pseudo_lengths_gene", + "pseudo_lengths_transcript": "pseudo_lengths_transcript", + "pseudo_quant_merged_summarizedexperiment": "pseudo_quant_merged_summarizedexperiment", + "deseq2_output_pseudo": "deseq2_output_pseudo", + "multiqc_report": "multiqc_report", + "multiqc_data": "multiqc_data", + "multiqc_plots": "multiqc_plots" + ] + ) + + // output_ch = demultiplex_ch + | setState(["output_demultiplexed", "output_falco", "output_multiqc"]) + + emit: + output_ch +} diff --git a/src/meta_workflows/demultiplex_rnaseq/test.sh b/src/meta_workflows/demultiplex_rnaseq/test.sh new file mode 100644 index 0000000..24cb27f --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +viash ns build --setup cb --parallel -q demultiplex_rnaseq + +nextflow run . \ + -main-script target/nextflow/demultiplex_rnaseq/main.nf \ + --input "https://github.com/nf-core/test-datasets/raw/refs/heads/demultiplex/testdata/NovaSeq6000/200624_A00834_0183_BHMTFYDRXX.tar.gz" \ + --sample_sheet "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/demultiplex/testdata/NovaSeq6000/SampleSheet.csv" \ + --fasta "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.fasta" \ + --gtf "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.gtf" \ + --transcript_fasta "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/transcriptome.fasta" \ + --skip_bbsplit \ + --skip_pseudo_alignment \ + -profile docker \ + --publish_dir test_results \ \ No newline at end of file From 04147961158e165f19fd4e8cbf8695ec73a2532a Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Mon, 18 Nov 2024 14:20:49 +0100 Subject: [PATCH 02/16] update output state --- .../demultiplex_rnaseq/config.vsh.yaml | 49 ++++++++++--------- src/meta_workflows/demultiplex_rnaseq/main.nf | 18 +++---- src/meta_workflows/demultiplex_rnaseq/test.sh | 2 +- 3 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml index 922f613..746ca49 100644 --- a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -5,38 +5,17 @@ argument_groups: - name: Demultiplex Input arguments arguments: - - name: --input + - name: "--input" description: Directory containing raw sequencing data type: file required: true - - name: --sample_sheet + - name: "--sample_sheet" description: | Sample sheet as input for BCL Convert. If not specified, will try to autodetect the sample sheet in the input directory type: file required: false - - name: Demultiplex Output arguments - arguments: - - name: --output_demultiplexed - description: Directory to write demultiplexed output to - type: file - direction: output - required: true - default: "$id/demultiplex" - - name: --output_falco - description: Directory to write Falco output to - type: file - direction: output - required: false - default: "$id/falco" - - name: --output_multiqc - description: Directory to write MultiQC output to - type: file - direction: output - required: false - default: "$id/multiqc_report.html" - - name: Reference genome options arguments: - name: "--fasta" @@ -301,7 +280,29 @@ argument_groups: type: file default: src/assets/methods_description_template.yml - - name: Output + + - name: Demultiplex Output arguments + arguments: + - name: --output_demultiplexed + description: Directory to write demultiplexed output to + type: file + direction: output + required: true + default: "$id/demultiplex" + - name: --output_falco + description: Directory to write Falco output to + type: file + direction: output + required: false + default: "$id/falco" + - name: --output_multiqc + description: Directory to write MultiQC output to + type: file + direction: output + required: false + default: "$id/multiqc_report.html" + + - name: RNAseq Output arguments: # Reference files - name: "--output_fasta" diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index 916296e..b7e6e9c 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -4,23 +4,22 @@ workflow run_wf { main: output_ch = input_ch - + | demultiplex.run( fromState: [ "input": "input", "sample_sheet": "sample_sheet" ], toState: [ - "output_demultiplexed": "output", // "output_falco": "output_falco", - // "output_multiqc": "output_multiqc" + // "output_multiqc": "output_multiqc", + "output_demultiplexed": "output" ], ) | flatMap { id, state -> println "Processing sample sheet: $state.sample_sheet" def sample_sheet = file(state.sample_sheet) - def samples = ["Undetermined"] def original_id = id def lines = sample_sheet.readLines() @@ -30,15 +29,12 @@ workflow run_wf { return } def data_lines = lines.drop(lines.indexOf('[Data]') + 2) - def sample_ids = data_lines.collect { line -> + def samples = data_lines.collect { line -> def columns = line.split(',') return columns.size() > 1 ? columns[1].trim() : null }.findAll { it != null } // Remove nulls - samples += sample_ids println "Looking for fastq files in ${state.output_demultiplexed}." - def allfastqs = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ /^.+\.fastq.gz$/} - println "Found ${allfastqs.size()} fastq files, matching them to the following samples: ${samples}." processed_samples = samples.collect { sample_id -> def forward_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R1_(\d+)\.fastq\.gz$/ def reverse_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R2_(\d+)\.fastq\.gz$/ @@ -174,7 +170,11 @@ workflow run_wf { ] ) - // output_ch = demultiplex_ch + | map { id, state -> + def mod_state = state.findAll { key, value -> value instanceof java.nio.file.Path && value.exists() } + [ id, mod_state ] + } + | setState(["output_demultiplexed", "output_falco", "output_multiqc"]) emit: diff --git a/src/meta_workflows/demultiplex_rnaseq/test.sh b/src/meta_workflows/demultiplex_rnaseq/test.sh index 24cb27f..d3d95c4 100644 --- a/src/meta_workflows/demultiplex_rnaseq/test.sh +++ b/src/meta_workflows/demultiplex_rnaseq/test.sh @@ -10,6 +10,6 @@ nextflow run . \ --gtf "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.gtf" \ --transcript_fasta "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/transcriptome.fasta" \ --skip_bbsplit \ - --skip_pseudo_alignment \ + --skip_alignment \ -profile docker \ --publish_dir test_results \ \ No newline at end of file From 013c85797e97383c30bbd7a6af51d9d6a136053c Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Mon, 18 Nov 2024 14:21:11 +0100 Subject: [PATCH 03/16] add readme --- .../demultiplex_rnaseq/README.qmd | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/meta_workflows/demultiplex_rnaseq/README.qmd diff --git a/src/meta_workflows/demultiplex_rnaseq/README.qmd b/src/meta_workflows/demultiplex_rnaseq/README.qmd new file mode 100644 index 0000000..f9e512e --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/README.qmd @@ -0,0 +1,18 @@ +--- +title: A meta workflow for demultiplexing and RNA sequencing analysis +format: gfm +--- + +# Introduction + +To perform demultiplexing and analysis of RNAseq data, we will create a meta workflow that includes the demultiplex and rnaseq workflows from Viash Hub. + +1. Start by creating a new workflow configuration with all the necessary inputs and outputs. +2. Add the demultiplex and rnaseq workflows as dependencies in the configuration. +3. Create the Nextflow script for the workflow. +4. Add the demultiplex and rnaseq workflows as steps in the Nextflow workflow. +5. Add any intermediate steps that are required to gather output fastq files from the demultiplex workflow and modify the state as required by the rnaseq workflow. + +# Test data +We will use the same test data that was used in the demultiplex workflow (TODO: add link). The genome reference files were obtained from . + From 6db25f3f0ac8c1a8f093520869879a7a1f70436b Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Wed, 27 Nov 2024 09:41:29 +0100 Subject: [PATCH 04/16] update arguments --- .../demultiplex_rnaseq/config.vsh.yaml | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml index 746ca49..c26934d 100644 --- a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -70,6 +70,9 @@ argument_groups: type: string description: By default, the pipeline assigns reads based on the 'exon' attribute within the GTF file. default: exon + - name: --star_sjdb_gtf_feature_exon + type: string + description: Feature type in GTF file to be used as exons for building transcripts - name: Read trimming options arguments: @@ -167,6 +170,10 @@ argument_groups: - name: "--salmon_quant_libtype" type: string description: Override Salmon library type inferred based on strandedness defined in meta object. + - name: "--extra_salmon_quant_args" + type: string + default: '-v' + description: Extra arguments to pass to salmon quant command in addition to defaults defined by the pipeline. - name: "--min_mapped_reads" type: integer description: Minimum percentage of uniquely mapped reads below which samples are removed from further processing. @@ -174,6 +181,10 @@ argument_groups: - name: "--stringtie_ignore_gtf" type: boolean_true description: Perform reference-guided de novo assembly of transcripts using StringTie, i.e. don't restrict to those in GTF file. + - name: "--extra_stringtie_args" + type: string + default: '-v' + description: Extra arguments to pass to stringtie command in addition to defaults defined by the pipeline. - name: "--save_unaligned" type: boolean_true description: Where possible, save unaligned reads from either STAR, HISAT2 or Salmon to the results directory. @@ -229,30 +240,25 @@ argument_groups: - name: "--skip_deseq2_qc" type: boolean_true description: Skip DESeq2 PCA and heatmap plotting. - - name: skip_dupradar + - name: --skip_dupradar type: boolean_true description: Skip dupRadar. - - name: skip_rseqc + - name: --skip_qualimap + type: boolean_true + description: Skip Qualimap. + - name: --skip_rseqc type: boolean_true description: Skip RSeQC. - - name: skip_multiqc + - name: --skip_multiqc type: boolean_true description: Skip MultiQC. - name: Other process arguments arguments: - - name: "--extra_fq_subsample_args" - type: string - default: ' --record-count 1000000 --seed 1' - description: Extra arguments to pass to fq subsample command in addition to defaults defined by the pipeline. - name: "--extra_picard_args" type: string default: ' --ASSUME_SORTED true --REMOVE_DUPLICATES false --VALIDATION_STRINGENCY LENIENT --TMP_DIR tmp' description: Extra arguments to pass to picard MarkDuplicates command in addition to defaults defined by the pipeline. - - name: "--extra_bedtools_args" - type: string - default: ' -split -du' - description: Extra arguments to pass to bedtools genomecov command in addition to defaults defined by the pipeline. - name: "--extra_preseq_args" type: string description: Extra arguments to pass to preseq lc_extrap command in addition to defaults defined by the pipeline @@ -264,12 +270,12 @@ argument_groups: - name: "--rseqc_modules" type: string multiple: true - multiple_sep: "," - description: Specify the RSeQC modules to run_wf - default: bam_stat,inner_distance,infer_experiment,junction_annotation,junction_saturation,read_distribution,read_duplication + multiple_sep: ";" + description: Specify the RSeQC modules to run_wf (comma-separated list) + default: bam_stat;inner_distance;infer_experiment;junction_annotation;junction_saturation;read_distribution;read_duplication choices: [ "bam_stat", "inner_distance", "infer_experiment", "junction_annotation", "junction_saturation", "read_distribution", "read_duplication", "tin" ] - - name: Multiqc paramenters + - name: MultiQC paramenters arguments: - name: "--multiqc_custom_config" type: file @@ -349,14 +355,14 @@ argument_groups: required: false must_exist: false description: Path to output directory - default: fastq/$id.read_1.fastq + default: fastq/$id.read_1.fastq.gz - name: "--output_fastq_2" type: file direction: output required: false must_exist: false description: Path to output directory - default: fastq/$id.read_2.fastq + default: fastq/$id.read_2.fastq.gz # FastQC - name: "--fastqc_html_1" @@ -841,6 +847,10 @@ argument_groups: type: file direction: output default: deseq2_qc + - name: "--deseq2_output_pseudo" + type: file + direction: output + default: deseq2_qc_pseudo # MultiQC - name: "--multiqc_report" @@ -872,6 +882,10 @@ argument_groups: type: file direction: output default: pseudo_alignment_quantification/gene_counts_scaled.tsv + - name: "--pseudo_tpm_gene" + type: file + direction: output + default: pseudo_alignment_quantification/gene_tpm.tsv - name: "--pseudo_tpm_transcript" type: file direction: output From 3da445503f0ed176c819d58bdd2dedc08d1072d1 Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Wed, 27 Nov 2024 09:41:47 +0100 Subject: [PATCH 05/16] update test --- src/meta_workflows/demultiplex_rnaseq/README.qmd | 4 ++-- src/meta_workflows/demultiplex_rnaseq/example.yaml | 8 ++++++++ src/meta_workflows/demultiplex_rnaseq/test.sh | 8 +------- 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 src/meta_workflows/demultiplex_rnaseq/example.yaml diff --git a/src/meta_workflows/demultiplex_rnaseq/README.qmd b/src/meta_workflows/demultiplex_rnaseq/README.qmd index f9e512e..565e7b2 100644 --- a/src/meta_workflows/demultiplex_rnaseq/README.qmd +++ b/src/meta_workflows/demultiplex_rnaseq/README.qmd @@ -14,5 +14,5 @@ To perform demultiplexing and analysis of RNAseq data, we will create a meta wor 5. Add any intermediate steps that are required to gather output fastq files from the demultiplex workflow and modify the state as required by the rnaseq workflow. # Test data -We will use the same test data that was used in the demultiplex workflow (TODO: add link). The genome reference files were obtained from . - +We will use the same test data from . The genome fasta was obtained from and the annotation file was obtained from . +The `example.yaml` parameter file can be used to test the meta-workflow. \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/example.yaml b/src/meta_workflows/demultiplex_rnaseq/example.yaml new file mode 100644 index 0000000..47d5ee6 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/example.yaml @@ -0,0 +1,8 @@ +input: "https://github.com/nf-core/test-datasets/raw/refs/heads/demultiplex/testdata/NovaSeq6000/200624_A00834_0183_BHMTFYDRXX.tar.gz" +sample_sheet: "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/demultiplex/testdata/NovaSeq6000/SampleSheet.csv" +fasta: "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/858/895/GCA_009858895.3_ASM985889v3/GCA_009858895.3_ASM985889v3_genomic.fna.gz" +gtf: "http://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz" +skip_bbsplit: true +skip_alignment: true +min_trimmed_reads: 5000 +rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication" diff --git a/src/meta_workflows/demultiplex_rnaseq/test.sh b/src/meta_workflows/demultiplex_rnaseq/test.sh index d3d95c4..d6c8b5e 100644 --- a/src/meta_workflows/demultiplex_rnaseq/test.sh +++ b/src/meta_workflows/demultiplex_rnaseq/test.sh @@ -4,12 +4,6 @@ viash ns build --setup cb --parallel -q demultiplex_rnaseq nextflow run . \ -main-script target/nextflow/demultiplex_rnaseq/main.nf \ - --input "https://github.com/nf-core/test-datasets/raw/refs/heads/demultiplex/testdata/NovaSeq6000/200624_A00834_0183_BHMTFYDRXX.tar.gz" \ - --sample_sheet "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/demultiplex/testdata/NovaSeq6000/SampleSheet.csv" \ - --fasta "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.fasta" \ - --gtf "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.gtf" \ - --transcript_fasta "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/transcriptome.fasta" \ - --skip_bbsplit \ - --skip_alignment \ + -params-file example.yaml \ -profile docker \ --publish_dir test_results \ \ No newline at end of file From 9e5f6391f8453a71e6ddcd5adb040b6e158f4e89 Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Wed, 27 Nov 2024 19:07:59 +0100 Subject: [PATCH 06/16] update output --- src/meta_workflows/demultiplex_rnaseq/main.nf | 107 +++++++++++++++++- src/meta_workflows/demultiplex_rnaseq/test.sh | 5 +- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index b7e6e9c..5171c9c 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -52,7 +52,6 @@ workflow run_wf { println "Finished processing sample sheet." return processed_samples } - | view | rnaseq.run( fromState: [ @@ -175,7 +174,111 @@ workflow run_wf { [ id, mod_state ] } - | setState(["output_demultiplexed", "output_falco", "output_multiqc"]) + | setState( + [ + "output_fasta": "output_fasta", + "output_gtf": "output_gtf", + "output_transcript_fasta": "output_transcript_fasta", + "output_gene_bed": "output_gene_bed", + "output_bbsplit_index": "output_bbsplit_index", + "output_star_index": "output_star_index", + "output_salmon_index": "output_salmon_index", + "output_kallisto_index": "output_kallisto_index", + "fastqc_html_1": "fastqc_html_1", + "fastqc_html_2": "fastqc_html_2", + "fastqc_zip_1": "fastqc_zip_1", + "fastqc_zip_2": "fastqc_zip_2", + "output_fastq_1": "output_fastq_1", + "output_fastq_2": "output_fastq_2", + "trim_log_1": "trim_log_1", + "trim_log_2": "trim_log_2", + "trim_zip_1": "trim_zip_1", + "trim_zip_2": "trim_zip_2", + "trim_html_1": "trim_html_1", + "trim_html_2": "trim_html_2", + "sortmerna_log": "sortmerna_log", + "star_log": "star_log", + "genome_bam_sorted": "genome_bam_sorted", + "genome_bam_index": "genome_bam_index", + "genome_bam_stats": "genome_bam_stats", + "genome_bam_flagstat": "genome_bam_flagstat", + "genome_bam_idxstats": "genome_bam_idxstats", + "transcriptome_bam": "transcriptome_bam", + "transcriptome_bam_index": "transcriptome_bam_index", + "transcriptome_bam_stats": "transcriptome_bam_stats", + "transcriptome_bam_flagstat": "transcriptome_bam_flagstat", + "transcriptome_bam_idxstats": "transcriptome_bam_idxstats", + "salmon_quant_results": "salmon_quant_results", + "pseudo_quant_results": "pseudo_quant_results", + "markduplicates_metrics": "markduplicates_metrics", + "stringtie_transcript_gtf": "stringtie_transcript_gtf", + "stringtie_coverage_gtf": "stringtie_coverage_gtf", + "stringtie_abundance": "stringtie_abundance", + "stringtie_ballgown": "stringtie_ballgown", + "featurecounts": "featurecounts", + "featurecounts_summary": "featurecounts_summary", + "featurecounts_multiqc": "featurecounts_multiqc", + "featurecounts_rrna_multiqc": "featurecounts_rrna_multiqc", + "bedgraph_forward": "bedgraph_forward", + "bedgraph_reverse": "bedgraph_reverse", + "bigwig_forward": "bigwig_forward", + "bigwig_reverse": "bigwig_reverse", + "preseq_output": "preseq_output", + "bamstat_output": "bamstat_output", + "strandedness_output": "strandedness_output", + "inner_dist_output_stats": "inner_dist_output_stats", + "inner_dist_output_dist": "inner_dist_output_dist", + "inner_dist_output_freq": "inner_dist_output_freq", + "inner_dist_output_plot": "inner_dist_output_plot", + "inner_dist_output_plot_r": "inner_dist_output_plot_r", + "junction_annotation_output_log": "junction_annotation_output_log", + "junction_annotation_output_plot_r": "junction_annotation_output_plot_r", + "junction_annotation_output_junction_bed": "junction_annotation_output_junction_bed", + "junction_annotation_output_junction_interact": "junction_annotation_output_junction_interact", + "junction_annotation_output_junction_sheet": "junction_annotation_output_junction_sheet", + "junction_annotation_output_splice_events_plot": "junction_annotation_output_splice_events_plot", + "junction_annotation_output_splice_junctions_plot": "junction_annotation_output_splice_junctions_plot", + "junction_saturation_output_plot_r": "junction_saturation_output_plot_r", + "junction_saturation_output_plot": "junction_saturation_output_plot", + "read_distribution_output": "read_distribution_output", + "read_duplication_output_duplication_rate_plot_r": "read_duplication_output_duplication_rate_plot_r", + "read_duplication_output_duplication_rate_plot": "read_duplication_output_duplication_rate_plot", + "read_duplication_output_duplication_rate_mapping": "read_duplication_output_duplication_rate_mapping", + "read_duplication_output_duplication_rate_sequence": "read_duplication_output_duplication_rate_sequence", + "tin_output_summary": "tin_output_summary", + "tin_output_metrics": "tin_output_metrics", + "dupradar_output_dupmatrix": "dupradar_output_dupmatrix", + "dupradar_output_dup_intercept_mqc": "dupradar_output_dup_intercept_mqc", + "dupradar_output_duprate_exp_boxplot": "dupradar_output_duprate_exp_boxplot", + "dupradar_output_duprate_exp_densplot": "dupradar_output_duprate_exp_densplot", + "dupradar_output_duprate_exp_denscurve_mqc": "dupradar_output_duprate_exp_denscurve_mqc", + "dupradar_output_expression_histogram": "dupradar_output_expression_histogram", + "dupradar_output_intercept_slope": "dupradar_output_intercept_slope", + "qualimap_output_dir": "qualimap_output_dir", + "qualimap_output_pdf": "qualimap_output_pdf", + "tpm_gene": "tpm_gene", + "counts_gene": "counts_gene", + "counts_gene_length_scaled": "counts_gene_length_scaled", + "counts_gene_scaled": "counts_gene_scaled", + "tpm_transcript": "tpm_transcript", + "counts_transcript": "counts_transcript", + "quant_merged_summarizedexperiment": "quant_merged_summarizedexperiment", + "deseq2_output": "deseq2_output", + "pseudo_tpm_gene": "pseudo_tpm_gene", + "pseudo_counts_gene": "pseudo_counts_gene", + "pseudo_counts_gene_length_scaled": "pseudo_counts_gene_length_scaled", + "pseudo_counts_gene_scaled": "pseudo_counts_gene_scaled", + "pseudo_tpm_transcript": "pseudo_tpm_transcript", + "pseudo_counts_transcript": "pseudo_counts_transcript", + "pseudo_lengths_gene": "pseudo_lengths_gene", + "pseudo_lengths_transcript": "pseudo_lengths_transcript", + "pseudo_quant_merged_summarizedexperiment": "pseudo_quant_merged_summarizedexperiment", + "deseq2_output_pseudo": "deseq2_output_pseudo", + "multiqc_report": "multiqc_report", + "multiqc_data": "multiqc_data", + "multiqc_plots": "multiqc_plots" + ] + ) emit: output_ch diff --git a/src/meta_workflows/demultiplex_rnaseq/test.sh b/src/meta_workflows/demultiplex_rnaseq/test.sh index d6c8b5e..c280031 100644 --- a/src/meta_workflows/demultiplex_rnaseq/test.sh +++ b/src/meta_workflows/demultiplex_rnaseq/test.sh @@ -4,6 +4,7 @@ viash ns build --setup cb --parallel -q demultiplex_rnaseq nextflow run . \ -main-script target/nextflow/demultiplex_rnaseq/main.nf \ - -params-file example.yaml \ + -params-file src/meta_workflows/demultiplex_rnaseq/example.yaml \ -profile docker \ - --publish_dir test_results \ \ No newline at end of file + --publish_dir test_results \ + --resume From 341e100adc9f0e6f44b192d7e2733015c8078321 Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Thu, 28 Nov 2024 16:04:27 +0100 Subject: [PATCH 07/16] update rnaseq arguments --- .../demultiplex_rnaseq/config.vsh.yaml | 14 --------- .../demultiplex_rnaseq/example.yaml | 1 + src/meta_workflows/demultiplex_rnaseq/main.nf | 30 ++++++++++++++++++- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml index c26934d..21379b3 100644 --- a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -81,12 +81,6 @@ argument_groups: description: Specify the trimming tool to use. choices: ["trimgalore", "fastp"] default: "trimgalore" - - name: "--extra_trimgalore_args" - type: string - description: Extra arguments to pass to Trim Galore! command in addition to defaults defined by the pipeline. - - name: "--extra_fastp_args" - type: string - description: Extra arguments to pass to fastp command in addition to defaults defined by the pipeline. - name: "--min_trimmed_reads" type: integer description: Minimum number of trimmed reads below which samples are removed from further processing. Some downstream steps in the pipeline will fail if this threshold is too low. @@ -170,10 +164,6 @@ argument_groups: - name: "--salmon_quant_libtype" type: string description: Override Salmon library type inferred based on strandedness defined in meta object. - - name: "--extra_salmon_quant_args" - type: string - default: '-v' - description: Extra arguments to pass to salmon quant command in addition to defaults defined by the pipeline. - name: "--min_mapped_reads" type: integer description: Minimum percentage of uniquely mapped reads below which samples are removed from further processing. @@ -197,10 +187,6 @@ argument_groups: - name: "--skip_pseudo_alignment" type: boolean_true description: Skip all of the pseudo-alignment-based processes within the pipeline. - - name: --extra_rsem_calculate_expression_args - type: string - description: Extra arguments to pass to rsem-calculate-expression command in addition to defaults defined by the pipeline. - default: '--star --star-output-genome-bam --star-gzipped-read-file --estimate-rspd --seed 1' - name: Process skipping options arguments: diff --git a/src/meta_workflows/demultiplex_rnaseq/example.yaml b/src/meta_workflows/demultiplex_rnaseq/example.yaml index 47d5ee6..1f62207 100644 --- a/src/meta_workflows/demultiplex_rnaseq/example.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/example.yaml @@ -4,5 +4,6 @@ fasta: "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/858/895/GCA_009858895.3 gtf: "http://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz" skip_bbsplit: true skip_alignment: true +skip_deseq2_qc: true min_trimmed_reads: 5000 rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication" diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index 5171c9c..0efe798 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -60,9 +60,37 @@ workflow run_wf { "fastq_2": "fastq_2", "fasta": "fasta", "gtf": "gtf", + "gff": "gff", "transcript_fasta": "transcript_fasta", + "additional_fasta": "additional_fasta", + "gene_bed": "gene_bed", + "gencode": "gencode", + "skip_fastqc": "skip_fastqc", + "skip_trimming": "skip_trimming", + "trimmer": "trimmer", + "min_trimmed_reads": "min_trimmed_reads", "skip_bbsplit": "skip_bbsplit", - "skip_alignment": "skip_alignment" + "bbsplit_fasta_list": "bbsplit_fasta_list", + "remove_ribo_rna": "remove_ribo_rna", + "ribo_database_manifest": "ribo_database_manifest", + "with_umi": "with_umi", + "umitools_extract_method": "umitools_extract_method", + "umitools_bc_pattern": "umitools_bc_pattern", + "umi_discard_read": "umi_discard_read", + "skip_alignment": "skip_alignment", + "skip_pseudo_alignment": "skip_pseudo_alignment", + "aligner": "aligner", + "pseudo_aligner": "pseudo_aligner", + "min_mapped_reads": "min_mapped_reads", + "skip_qc": "skip_qc", + "skip_preseq": "skip_preseq", + "skip_deseq2_qc": "skip_deseq2_qc", + "skip_dupradar": "skip_dupradar", + "skip_qualimap": "skip_qualimap", + "skip_rseqc": "skip_rseqc", + "skip_multiqc": "skip_multiqc", + "rseqc_modules": "rseqc_modules", + "multiqc_custom_config": "multiqc_custom_config" ], toState: [ "output_fasta": "output_fasta", From 50a6e58059b63f02c744aead6122c118b56f5fb4 Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Fri, 29 Nov 2024 11:05:26 +0100 Subject: [PATCH 08/16] add multiqc custom config --- src/meta_workflows/demultiplex_rnaseq/example.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/meta_workflows/demultiplex_rnaseq/example.yaml b/src/meta_workflows/demultiplex_rnaseq/example.yaml index 1f62207..a623d87 100644 --- a/src/meta_workflows/demultiplex_rnaseq/example.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/example.yaml @@ -7,3 +7,4 @@ skip_alignment: true skip_deseq2_qc: true min_trimmed_reads: 5000 rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication" +multiqc_custom_config: "https://raw.githubusercontent.com/viash-hub/rnaseq/refs/heads/main/src/assets/multiqc_config.yml" \ No newline at end of file From 912e97b63b65fdb55c4f5b76a2bd167609768def Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Fri, 29 Nov 2024 11:05:54 +0100 Subject: [PATCH 09/16] make demultiplex output optional --- src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml index 21379b3..57760dd 100644 --- a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -279,7 +279,7 @@ argument_groups: description: Directory to write demultiplexed output to type: file direction: output - required: true + required: false default: "$id/demultiplex" - name: --output_falco description: Directory to write Falco output to From 4acfdf3ba400725325df98aa459cc3a44e11dd9a Mon Sep 17 00:00:00 2001 From: Nirmayi Date: Sat, 30 Nov 2024 19:07:56 +0100 Subject: [PATCH 10/16] add _meta join id --- src/meta_workflows/demultiplex_rnaseq/main.nf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index 0efe798..7f6c7d0 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -199,7 +199,7 @@ workflow run_wf { | map { id, state -> def mod_state = state.findAll { key, value -> value instanceof java.nio.file.Path && value.exists() } - [ id, mod_state ] + [ id, mod_state + [ _meta: [join_id: "run"] ] ] } | setState( @@ -304,7 +304,8 @@ workflow run_wf { "deseq2_output_pseudo": "deseq2_output_pseudo", "multiqc_report": "multiqc_report", "multiqc_data": "multiqc_data", - "multiqc_plots": "multiqc_plots" + "multiqc_plots": "multiqc_plots", + "_meta": "_meta" ] ) From 3ff6449d7804dbe8709cb4967f10b196c58f1011 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Fri, 10 Jan 2025 14:32:01 +0100 Subject: [PATCH 11/16] updated demultiplex workflow --- src/meta_workflows/demultiplex_rnaseq/main.nf | 423 +++++------------- 1 file changed, 119 insertions(+), 304 deletions(-) diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index 7f6c7d0..2aaf36c 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -1,314 +1,129 @@ + // Helper function to create identity mappings for state + def createIdentityMap(keys) { + return keys.collectEntries { [(it): it] } + } + workflow run_wf { - take: - input_ch + take: + input_ch - main: - output_ch = input_ch + main: + // Define the required input and output keys + def rnaseqInputKeys = [ + "id", "fastq_1", "fastq_2", "fasta", "gtf", "gff", "transcript_fasta", + "additional_fasta", "gene_bed", "gencode", "skip_fastqc", "skip_trimming", + "trimmer", "min_trimmed_reads", "skip_bbsplit", "bbsplit_fasta_list", + "remove_ribo_rna", "ribo_database_manifest", "with_umi", "umitools_extract_method", + "umitools_bc_pattern", "umi_discard_read", "skip_alignment", "skip_pseudo_alignment", + "aligner", "pseudo_aligner", "min_mapped_reads", "skip_qc", "skip_preseq", + "skip_deseq2_qc", "skip_dupradar", "skip_qualimap", "skip_rseqc", + "skip_multiqc", "rseqc_modules", "multiqc_custom_config" + ] + + def rnaseqOutputKeys = [ + "output_fasta", "output_gtf", "output_transcript_fasta", "output_gene_bed", + "output_bbsplit_index", "output_star_index", "output_salmon_index", + "output_kallisto_index", "fastqc_html_1", "fastqc_html_2", "fastqc_zip_1", + "fastqc_zip_2", "output_fastq_1", "output_fastq_2", "trim_log_1", "trim_log_2", + "trim_zip_1", "trim_zip_2", "trim_html_1", "trim_html_2", "sortmerna_log", + "star_log", "genome_bam_sorted", "genome_bam_index", "genome_bam_stats", + "genome_bam_flagstat", "genome_bam_idxstats", "transcriptome_bam", + "transcriptome_bam_index", "transcriptome_bam_stats", "transcriptome_bam_flagstat", + "transcriptome_bam_idxstats", "salmon_quant_results", "pseudo_quant_results", + "markduplicates_metrics", "stringtie_transcript_gtf", "stringtie_coverage_gtf", + "stringtie_abundance", "stringtie_ballgown", "featurecounts", + "featurecounts_summary", "featurecounts_multiqc", "featurecounts_rrna_multiqc", + "bedgraph_forward", "bedgraph_reverse", "bigwig_forward", "bigwig_reverse", + "preseq_output", "bamstat_output", "strandedness_output", + "inner_dist_output_stats", "inner_dist_output_dist", "inner_dist_output_freq", + "inner_dist_output_plot", "inner_dist_output_plot_r", + "junction_annotation_output_log", "junction_annotation_output_plot_r", + "junction_annotation_output_junction_bed", + "junction_annotation_output_junction_interact", + "junction_annotation_output_junction_sheet", + "junction_annotation_output_splice_events_plot", + "junction_annotation_output_splice_junctions_plot", + "junction_saturation_output_plot_r", "junction_saturation_output_plot", + "read_distribution_output", "read_duplication_output_duplication_rate_plot_r", + "read_duplication_output_duplication_rate_plot", + "read_duplication_output_duplication_rate_mapping", + "read_duplication_output_duplication_rate_sequence", "tin_output_summary", + "tin_output_metrics", "dupradar_output_dupmatrix", + "dupradar_output_dup_intercept_mqc", "dupradar_output_duprate_exp_boxplot", + "dupradar_output_duprate_exp_densplot", + "dupradar_output_duprate_exp_denscurve_mqc", + "dupradar_output_expression_histogram", "dupradar_output_intercept_slope", + "qualimap_output_dir", "qualimap_output_pdf", "tpm_gene", "counts_gene", + "counts_gene_length_scaled", "counts_gene_scaled", "tpm_transcript", + "counts_transcript", "quant_merged_summarizedexperiment", "deseq2_output", + "pseudo_tpm_gene", "pseudo_counts_gene", "pseudo_counts_gene_length_scaled", + "pseudo_counts_gene_scaled", "pseudo_tpm_transcript", "pseudo_counts_transcript", + "pseudo_lengths_gene", "pseudo_lengths_transcript", + "pseudo_quant_merged_summarizedexperiment", "deseq2_output_pseudo", + "multiqc_report", "multiqc_data", "multiqc_plots" + ] - | demultiplex.run( - fromState: [ - "input": "input", - "sample_sheet": "sample_sheet" - ], - toState: [ - // "output_falco": "output_falco", - // "output_multiqc": "output_multiqc", - "output_demultiplexed": "output" - ], - ) + output_ch = input_ch + | demultiplex.run( + fromState: [ + "input": "input", + "sample_sheet": "sample_sheet" + ], + toState: [ + "output_demultiplexed": "output" + ] + ) - | flatMap { id, state -> - println "Processing sample sheet: $state.sample_sheet" - def sample_sheet = file(state.sample_sheet) - def original_id = id - def lines = sample_sheet.readLines() + | flatMap { id, state -> + println "Processing sample sheet: $state.sample_sheet" + def sample_sheet = file(state.sample_sheet) + def original_id = id + def lines = sample_sheet.readLines() - // Extract Sample_IDs - if (lines.indexOf('[Data]') < 0) { - println "Data section not found" - return - } - def data_lines = lines.drop(lines.indexOf('[Data]') + 2) - def samples = data_lines.collect { line -> - def columns = line.split(',') - return columns.size() > 1 ? columns[1].trim() : null - }.findAll { it != null } // Remove nulls + // Extract Sample_IDs + if (lines.indexOf('[Data]') < 0) { + println "Data section not found" + return + } + def data_lines = lines.drop(lines.indexOf('[Data]') + 2) + def samples = data_lines.collect { line -> + def columns = line.split(',') + return columns.size() > 1 ? columns[1].trim() : null + }.findAll { it != null } - println "Looking for fastq files in ${state.output_demultiplexed}." - processed_samples = samples.collect { sample_id -> - def forward_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R1_(\d+)\.fastq\.gz$/ - def reverse_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R2_(\d+)\.fastq\.gz$/ - def forward_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ forward_regex} - def reverse_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ reverse_regex} - reverse_fastq = !reverse_fastq.isEmpty() ? reverse_fastq[0] : null - def fastqs_state = [ - "id": sample_id, - "fastq_1": forward_fastq[0], - "fastq_2": reverse_fastq, - "_meta": [ "join_id": original_id ] - ] - [sample_id, fastqs_state + state] - } - println "Finished processing sample sheet." - return processed_samples - } + println "Looking for fastq files in ${state.output_demultiplexed}." + processed_samples = samples.collect { sample_id -> + def forward_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R1_(\d+)\.fastq\.gz$/ + def reverse_regex = ~/^${sample_id}_S(\d+)_(L(\d+)_)?R2_(\d+)\.fastq\.gz$/ + def forward_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ forward_regex} + def reverse_fastq = state.output_demultiplexed.listFiles().findAll{it.isFile() && it.name ==~ reverse_regex} + reverse_fastq = !reverse_fastq.isEmpty() ? reverse_fastq[0] : null + def fastqs_state = [ + "id": sample_id, + "fastq_1": forward_fastq[0], + "fastq_2": reverse_fastq, + "_meta": [ "join_id": original_id ] + ] + [sample_id, fastqs_state + state] + } + println "Finished processing sample sheet." + return processed_samples + } - | rnaseq.run( - fromState: [ - "id": "id", - "fastq_1": "fastq_1", - "fastq_2": "fastq_2", - "fasta": "fasta", - "gtf": "gtf", - "gff": "gff", - "transcript_fasta": "transcript_fasta", - "additional_fasta": "additional_fasta", - "gene_bed": "gene_bed", - "gencode": "gencode", - "skip_fastqc": "skip_fastqc", - "skip_trimming": "skip_trimming", - "trimmer": "trimmer", - "min_trimmed_reads": "min_trimmed_reads", - "skip_bbsplit": "skip_bbsplit", - "bbsplit_fasta_list": "bbsplit_fasta_list", - "remove_ribo_rna": "remove_ribo_rna", - "ribo_database_manifest": "ribo_database_manifest", - "with_umi": "with_umi", - "umitools_extract_method": "umitools_extract_method", - "umitools_bc_pattern": "umitools_bc_pattern", - "umi_discard_read": "umi_discard_read", - "skip_alignment": "skip_alignment", - "skip_pseudo_alignment": "skip_pseudo_alignment", - "aligner": "aligner", - "pseudo_aligner": "pseudo_aligner", - "min_mapped_reads": "min_mapped_reads", - "skip_qc": "skip_qc", - "skip_preseq": "skip_preseq", - "skip_deseq2_qc": "skip_deseq2_qc", - "skip_dupradar": "skip_dupradar", - "skip_qualimap": "skip_qualimap", - "skip_rseqc": "skip_rseqc", - "skip_multiqc": "skip_multiqc", - "rseqc_modules": "rseqc_modules", - "multiqc_custom_config": "multiqc_custom_config" - ], - toState: [ - "output_fasta": "output_fasta", - "output_gtf": "output_gtf", - "output_transcript_fasta": "output_transcript_fasta", - "output_gene_bed": "output_gene_bed", - "output_bbsplit_index": "output_bbsplit_index", - "output_star_index": "output_star_index", - "output_salmon_index": "output_salmon_index", - "output_kallisto_index": "output_kallisto_index", - "fastqc_html_1": "fastqc_html_1", - "fastqc_html_2": "fastqc_html_2", - "fastqc_zip_1": "fastqc_zip_1", - "fastqc_zip_2": "fastqc_zip_2", - "output_fastq_1": "output_fastq_1", - "output_fastq_2": "output_fastq_2", - "trim_log_1": "trim_log_1", - "trim_log_2": "trim_log_2", - "trim_zip_1": "trim_zip_1", - "trim_zip_2": "trim_zip_2", - "trim_html_1": "trim_html_1", - "trim_html_2": "trim_html_2", - "sortmerna_log": "sortmerna_log", - "star_log": "star_log", - "genome_bam_sorted": "genome_bam_sorted", - "genome_bam_index": "genome_bam_index", - "genome_bam_stats": "genome_bam_stats", - "genome_bam_flagstat": "genome_bam_flagstat", - "genome_bam_idxstats": "genome_bam_idxstats", - "transcriptome_bam": "transcriptome_bam", - "transcriptome_bam_index": "transcriptome_bam_index", - "transcriptome_bam_stats": "transcriptome_bam_stats", - "transcriptome_bam_flagstat": "transcriptome_bam_flagstat", - "transcriptome_bam_idxstats": "transcriptome_bam_idxstats", - "salmon_quant_results": "salmon_quant_results", - "pseudo_quant_results": "pseudo_quant_results", - "markduplicates_metrics": "markduplicates_metrics", - "stringtie_transcript_gtf": "stringtie_transcript_gtf", - "stringtie_coverage_gtf": "stringtie_coverage_gtf", - "stringtie_abundance": "stringtie_abundance", - "stringtie_ballgown": "stringtie_ballgown", - "featurecounts": "featurecounts", - "featurecounts_summary": "featurecounts_summary", - "featurecounts_multiqc": "featurecounts_multiqc", - "featurecounts_rrna_multiqc": "featurecounts_rrna_multiqc", - "bedgraph_forward": "bedgraph_forward", - "bedgraph_reverse": "bedgraph_reverse", - "bigwig_forward": "bigwig_forward", - "bigwig_reverse": "bigwig_reverse", - "preseq_output": "preseq_output", - "bamstat_output": "bamstat_output", - "strandedness_output": "strandedness_output", - "inner_dist_output_stats": "inner_dist_output_stats", - "inner_dist_output_dist": "inner_dist_output_dist", - "inner_dist_output_freq": "inner_dist_output_freq", - "inner_dist_output_plot": "inner_dist_output_plot", - "inner_dist_output_plot_r": "inner_dist_output_plot_r", - "junction_annotation_output_log": "junction_annotation_output_log", - "junction_annotation_output_plot_r": "junction_annotation_output_plot_r", - "junction_annotation_output_junction_bed": "junction_annotation_output_junction_bed", - "junction_annotation_output_junction_interact": "junction_annotation_output_junction_interact", - "junction_annotation_output_junction_sheet": "junction_annotation_output_junction_sheet", - "junction_annotation_output_splice_events_plot": "junction_annotation_output_splice_events_plot", - "junction_annotation_output_splice_junctions_plot": "junction_annotation_output_splice_junctions_plot", - "junction_saturation_output_plot_r": "junction_saturation_output_plot_r", - "junction_saturation_output_plot": "junction_saturation_output_plot", - "read_distribution_output": "read_distribution_output", - "read_duplication_output_duplication_rate_plot_r": "read_duplication_output_duplication_rate_plot_r", - "read_duplication_output_duplication_rate_plot": "read_duplication_output_duplication_rate_plot", - "read_duplication_output_duplication_rate_mapping": "read_duplication_output_duplication_rate_mapping", - "read_duplication_output_duplication_rate_sequence": "read_duplication_output_duplication_rate_sequence", - "tin_output_summary": "tin_output_summary", - "tin_output_metrics": "tin_output_metrics", - "dupradar_output_dupmatrix": "dupradar_output_dupmatrix", - "dupradar_output_dup_intercept_mqc": "dupradar_output_dup_intercept_mqc", - "dupradar_output_duprate_exp_boxplot": "dupradar_output_duprate_exp_boxplot", - "dupradar_output_duprate_exp_densplot": "dupradar_output_duprate_exp_densplot", - "dupradar_output_duprate_exp_denscurve_mqc": "dupradar_output_duprate_exp_denscurve_mqc", - "dupradar_output_expression_histogram": "dupradar_output_expression_histogram", - "dupradar_output_intercept_slope": "dupradar_output_intercept_slope", - "qualimap_output_dir": "qualimap_output_dir", - "qualimap_output_pdf": "qualimap_output_pdf", - "tpm_gene": "tpm_gene", - "counts_gene": "counts_gene", - "counts_gene_length_scaled": "counts_gene_length_scaled", - "counts_gene_scaled": "counts_gene_scaled", - "tpm_transcript": "tpm_transcript", - "counts_transcript": "counts_transcript", - "quant_merged_summarizedexperiment": "quant_merged_summarizedexperiment", - "deseq2_output": "deseq2_output", - "pseudo_tpm_gene": "pseudo_tpm_gene", - "pseudo_counts_gene": "pseudo_counts_gene", - "pseudo_counts_gene_length_scaled": "pseudo_counts_gene_length_scaled", - "pseudo_counts_gene_scaled": "pseudo_counts_gene_scaled", - "pseudo_tpm_transcript": "pseudo_tpm_transcript", - "pseudo_counts_transcript": "pseudo_counts_transcript", - "pseudo_lengths_gene": "pseudo_lengths_gene", - "pseudo_lengths_transcript": "pseudo_lengths_transcript", - "pseudo_quant_merged_summarizedexperiment": "pseudo_quant_merged_summarizedexperiment", - "deseq2_output_pseudo": "deseq2_output_pseudo", - "multiqc_report": "multiqc_report", - "multiqc_data": "multiqc_data", - "multiqc_plots": "multiqc_plots" - ] - ) + | rnaseq.run( + fromState: createIdentityMap(rnaseqInputKeys), + toState: createIdentityMap(rnaseqOutputKeys) + ) - | map { id, state -> - def mod_state = state.findAll { key, value -> value instanceof java.nio.file.Path && value.exists() } - [ id, mod_state + [ _meta: [join_id: "run"] ] ] - } + | map { id, state -> + def mod_state = state.findAll { key, value -> + value instanceof java.nio.file.Path && value.exists() + } + [ id, mod_state + [ _meta: [join_id: "run"] ] ] + } - | setState( - [ - "output_fasta": "output_fasta", - "output_gtf": "output_gtf", - "output_transcript_fasta": "output_transcript_fasta", - "output_gene_bed": "output_gene_bed", - "output_bbsplit_index": "output_bbsplit_index", - "output_star_index": "output_star_index", - "output_salmon_index": "output_salmon_index", - "output_kallisto_index": "output_kallisto_index", - "fastqc_html_1": "fastqc_html_1", - "fastqc_html_2": "fastqc_html_2", - "fastqc_zip_1": "fastqc_zip_1", - "fastqc_zip_2": "fastqc_zip_2", - "output_fastq_1": "output_fastq_1", - "output_fastq_2": "output_fastq_2", - "trim_log_1": "trim_log_1", - "trim_log_2": "trim_log_2", - "trim_zip_1": "trim_zip_1", - "trim_zip_2": "trim_zip_2", - "trim_html_1": "trim_html_1", - "trim_html_2": "trim_html_2", - "sortmerna_log": "sortmerna_log", - "star_log": "star_log", - "genome_bam_sorted": "genome_bam_sorted", - "genome_bam_index": "genome_bam_index", - "genome_bam_stats": "genome_bam_stats", - "genome_bam_flagstat": "genome_bam_flagstat", - "genome_bam_idxstats": "genome_bam_idxstats", - "transcriptome_bam": "transcriptome_bam", - "transcriptome_bam_index": "transcriptome_bam_index", - "transcriptome_bam_stats": "transcriptome_bam_stats", - "transcriptome_bam_flagstat": "transcriptome_bam_flagstat", - "transcriptome_bam_idxstats": "transcriptome_bam_idxstats", - "salmon_quant_results": "salmon_quant_results", - "pseudo_quant_results": "pseudo_quant_results", - "markduplicates_metrics": "markduplicates_metrics", - "stringtie_transcript_gtf": "stringtie_transcript_gtf", - "stringtie_coverage_gtf": "stringtie_coverage_gtf", - "stringtie_abundance": "stringtie_abundance", - "stringtie_ballgown": "stringtie_ballgown", - "featurecounts": "featurecounts", - "featurecounts_summary": "featurecounts_summary", - "featurecounts_multiqc": "featurecounts_multiqc", - "featurecounts_rrna_multiqc": "featurecounts_rrna_multiqc", - "bedgraph_forward": "bedgraph_forward", - "bedgraph_reverse": "bedgraph_reverse", - "bigwig_forward": "bigwig_forward", - "bigwig_reverse": "bigwig_reverse", - "preseq_output": "preseq_output", - "bamstat_output": "bamstat_output", - "strandedness_output": "strandedness_output", - "inner_dist_output_stats": "inner_dist_output_stats", - "inner_dist_output_dist": "inner_dist_output_dist", - "inner_dist_output_freq": "inner_dist_output_freq", - "inner_dist_output_plot": "inner_dist_output_plot", - "inner_dist_output_plot_r": "inner_dist_output_plot_r", - "junction_annotation_output_log": "junction_annotation_output_log", - "junction_annotation_output_plot_r": "junction_annotation_output_plot_r", - "junction_annotation_output_junction_bed": "junction_annotation_output_junction_bed", - "junction_annotation_output_junction_interact": "junction_annotation_output_junction_interact", - "junction_annotation_output_junction_sheet": "junction_annotation_output_junction_sheet", - "junction_annotation_output_splice_events_plot": "junction_annotation_output_splice_events_plot", - "junction_annotation_output_splice_junctions_plot": "junction_annotation_output_splice_junctions_plot", - "junction_saturation_output_plot_r": "junction_saturation_output_plot_r", - "junction_saturation_output_plot": "junction_saturation_output_plot", - "read_distribution_output": "read_distribution_output", - "read_duplication_output_duplication_rate_plot_r": "read_duplication_output_duplication_rate_plot_r", - "read_duplication_output_duplication_rate_plot": "read_duplication_output_duplication_rate_plot", - "read_duplication_output_duplication_rate_mapping": "read_duplication_output_duplication_rate_mapping", - "read_duplication_output_duplication_rate_sequence": "read_duplication_output_duplication_rate_sequence", - "tin_output_summary": "tin_output_summary", - "tin_output_metrics": "tin_output_metrics", - "dupradar_output_dupmatrix": "dupradar_output_dupmatrix", - "dupradar_output_dup_intercept_mqc": "dupradar_output_dup_intercept_mqc", - "dupradar_output_duprate_exp_boxplot": "dupradar_output_duprate_exp_boxplot", - "dupradar_output_duprate_exp_densplot": "dupradar_output_duprate_exp_densplot", - "dupradar_output_duprate_exp_denscurve_mqc": "dupradar_output_duprate_exp_denscurve_mqc", - "dupradar_output_expression_histogram": "dupradar_output_expression_histogram", - "dupradar_output_intercept_slope": "dupradar_output_intercept_slope", - "qualimap_output_dir": "qualimap_output_dir", - "qualimap_output_pdf": "qualimap_output_pdf", - "tpm_gene": "tpm_gene", - "counts_gene": "counts_gene", - "counts_gene_length_scaled": "counts_gene_length_scaled", - "counts_gene_scaled": "counts_gene_scaled", - "tpm_transcript": "tpm_transcript", - "counts_transcript": "counts_transcript", - "quant_merged_summarizedexperiment": "quant_merged_summarizedexperiment", - "deseq2_output": "deseq2_output", - "pseudo_tpm_gene": "pseudo_tpm_gene", - "pseudo_counts_gene": "pseudo_counts_gene", - "pseudo_counts_gene_length_scaled": "pseudo_counts_gene_length_scaled", - "pseudo_counts_gene_scaled": "pseudo_counts_gene_scaled", - "pseudo_tpm_transcript": "pseudo_tpm_transcript", - "pseudo_counts_transcript": "pseudo_counts_transcript", - "pseudo_lengths_gene": "pseudo_lengths_gene", - "pseudo_lengths_transcript": "pseudo_lengths_transcript", - "pseudo_quant_merged_summarizedexperiment": "pseudo_quant_merged_summarizedexperiment", - "deseq2_output_pseudo": "deseq2_output_pseudo", - "multiqc_report": "multiqc_report", - "multiqc_data": "multiqc_data", - "multiqc_plots": "multiqc_plots", - "_meta": "_meta" - ] - ) + | setState(createIdentityMap(rnaseqOutputKeys + ["_meta"])) - emit: - output_ch -} + emit: + output_ch +} \ No newline at end of file From 7997d68ea5f84899795970b83a3cc93741ce8cb2 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Fri, 14 Mar 2025 15:44:56 +0100 Subject: [PATCH 12/16] integration test added --- _viash.yaml | 2 + nextflow.config | 8 ++ src/config/labels.config | 98 +++++++++++++++++++ .../demultiplex_rnaseq/config.vsh.yaml | 5 + .../demultiplex_rnaseq/integration_test.sh | 23 +++++ src/meta_workflows/demultiplex_rnaseq/main.nf | 6 +- src/meta_workflows/demultiplex_rnaseq/test.nf | 48 +++++++++ 7 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/config/labels.config create mode 100755 src/meta_workflows/demultiplex_rnaseq/integration_test.sh create mode 100644 src/meta_workflows/demultiplex_rnaseq/test.nf diff --git a/_viash.yaml b/_viash.yaml index 4b90754..3b39045 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -11,3 +11,5 @@ viash_version: 0.9.0-RC6 config_mods: | .requirements.commands := ['ps'] + .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} + .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")' diff --git a/nextflow.config b/nextflow.config index 9889b4d..fa71ce3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,3 +1,11 @@ +params { + rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString() +} + + +// include common settings +includeConfig("${params.rootDir}/src/config/labels.config") + process.container = "nextflow/nextflow:21.04.3" docker { enabled = true diff --git a/src/config/labels.config b/src/config/labels.config new file mode 100644 index 0000000..a3a1cd3 --- /dev/null +++ b/src/config/labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 50.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml index 57760dd..1540425 100644 --- a/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml +++ b/src/meta_workflows/demultiplex_rnaseq/config.vsh.yaml @@ -890,6 +890,11 @@ resources: path: main.nf entrypoint: run_wf +test_resources: + - type: nextflow_script + path: test.nf + entrypoint: test_wf + dependencies: - name: demultiplex repository: demultiplex diff --git a/src/meta_workflows/demultiplex_rnaseq/integration_test.sh b/src/meta_workflows/demultiplex_rnaseq/integration_test.sh new file mode 100755 index 0000000..8209841 --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/integration_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +# Make sure the workflow is built +viash ns build --setup cb --parallel + +# export NXF_VER=24.04.4 + +set -eo pipefail + +nextflow \ + run . \ + -main-script src/meta_workflows/demultiplex_rnaseq/test.nf \ + -config src/config/labels.config \ + -entry test_wf \ + -resume \ + -profile docker,local \ + --publish_dir output \ No newline at end of file diff --git a/src/meta_workflows/demultiplex_rnaseq/main.nf b/src/meta_workflows/demultiplex_rnaseq/main.nf index 2aaf36c..2a1d41f 100644 --- a/src/meta_workflows/demultiplex_rnaseq/main.nf +++ b/src/meta_workflows/demultiplex_rnaseq/main.nf @@ -1,4 +1,4 @@ - // Helper function to create identity mappings for state +// Helper function to create identity mappings for state def createIdentityMap(keys) { return keys.collectEntries { [(it): it] } } @@ -119,7 +119,9 @@ workflow run_wf { def mod_state = state.findAll { key, value -> value instanceof java.nio.file.Path && value.exists() } - [ id, mod_state + [ _meta: [join_id: "run"] ] ] + // Get the original join_id from the _meta if it exists, otherwise use current id + def join_id = state._meta?.join_id ?: id + [ id, mod_state + [ _meta: [join_id: join_id] ] ] } | setState(createIdentityMap(rnaseqOutputKeys + ["_meta"])) diff --git a/src/meta_workflows/demultiplex_rnaseq/test.nf b/src/meta_workflows/demultiplex_rnaseq/test.nf new file mode 100644 index 0000000..70a769d --- /dev/null +++ b/src/meta_workflows/demultiplex_rnaseq/test.nf @@ -0,0 +1,48 @@ +nextflow.enable.dsl=2 +targetDir = params.rootDir + "/target" + +include { demultiplex_rnaseq } from targetDir + "/nextflow/demultiplex_rnaseq/main.nf" + +params.resources_test = "gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta/" + +workflow test_wf { + + resources_test_file = file(params.resources_test) + + output_ch = Channel.fromList([ + [ + id: "_test", + input: resources_test_file.resolve("200624_A00834_0183_BHMTFYDRXX.tar.gz"), + sample_sheet: resources_test_file.resolve("SampleSheet.csv"), + fasta: resources_test_file.resolve("GCA_009858895.3_ASM985889v3_genomic.fna.gz"), + gtf: resources_test_file.resolve("Sars_cov_2.ASM985889v3.101.gtf.gz"), + skip_bbsplit: true, + skip_alignment: true, + skip_deseq2_qc: true, + num_trimmed_reads: 5000, + rseqc_modules: "bam_stat;infer_experiment;junction_saturation;read_distribution;read_duplication", + multiqc_custom_config: resources_test_file.resolve("multiqc_config.yml") + ] + ]) + + | map { state -> [state.id, state] } + + | demultiplex_rnaseq.run( + fromState: { id, state -> state }, + toState: { id, output, state -> output } + ) + + | view { output -> + assert output.size() == 2 : "Outputs should contain two elements; [id, state]" + + // check output + def state = output[1] + assert state instanceof Map : "State should be a map. Found: ${state}" + assert state.containsKey("output_fasta") : "Output should contain key 'output_fasta'." + assert state.output_fasta.isFile() : "'output_fasta' should be a file." + assert state.containsKey("output_gtf") : "Output should contain key 'output_gtf'." + assert state.output_gtf.isFile() : "'output_gtf' should be a file." + + "Output: $output" + } +} \ No newline at end of file From c769fdfcdcacfccd0ce340faddf2491b809544be Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Wed, 19 Mar 2025 15:11:37 +0100 Subject: [PATCH 13/16] Add test resources to _viash.yaml --- _viash.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/_viash.yaml b/_viash.yaml index 3b39045..b229e18 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -9,6 +9,11 @@ links: viash_version: 0.9.0-RC6 +info: + test_resources: + - path: gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta/ + dest: resources_test + config_mods: | .requirements.commands := ['ps'] .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} From 531de03fa3dfa566821c074d83d25a785f34ef4e Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Wed, 19 Mar 2025 15:12:11 +0100 Subject: [PATCH 14/16] Update Viash version --- _viash.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_viash.yaml b/_viash.yaml index b229e18..71bbfd6 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -7,7 +7,7 @@ links: issue_tracker: https://github.com/viash-hub/playground/issues repository: https://github.com/viash-hub/playground -viash_version: 0.9.0-RC6 +viash_version: 0.9.2 info: test_resources: From 66e809e0518128b85c7a238932500fbba114987b Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Wed, 19 Mar 2025 17:06:51 +0100 Subject: [PATCH 15/16] Remove trailing slash --- _viash.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_viash.yaml b/_viash.yaml index 71bbfd6..c8c1bef 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -11,7 +11,7 @@ viash_version: 0.9.2 info: test_resources: - - path: gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta/ + - path: gs://viash-hub-resources/rnaseq/demultiplex_rnaseq_meta dest: resources_test config_mods: | From c857f54ca10eafa781ee5870f49218de6ae7931e Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Thu, 20 Mar 2025 17:41:40 +0100 Subject: [PATCH 16/16] Remove (global) pointer to labels.config --- nextflow.config | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nextflow.config b/nextflow.config index fa71ce3..84625cf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -2,10 +2,6 @@ params { rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString() } - -// include common settings -includeConfig("${params.rootDir}/src/config/labels.config") - process.container = "nextflow/nextflow:21.04.3" docker { enabled = true