diff --git a/.agents/skills/monsda/reference/config-guide.md b/.agents/skills/monsda/reference/config-guide.md index ed823c83..4cc8079a 100644 --- a/.agents/skills/monsda/reference/config-guide.md +++ b/.agents/skills/monsda/reference/config-guide.md @@ -109,6 +109,20 @@ Each step listed in `WORKFLOWS` needs: - `ENV`/`BIN` at the condition level override the step-level default (useful to benchmark two installs of the same tool, or point at a local binary for tools without a conda package, e.g. `guppy`, `dorado`). +- `SEQUENCING` can likewise be set at the step level (or nested inside a + tool's own block) to override the global `SETTINGS.SEQUENCING` for just + that step, same `"single"/"paired"/"singlecell"[,rf|fr]` syntax. Useful for + e.g. QuantSeq-style 3' protocols: keep the global value `"paired,rf"` so + TRIMMING/DEDUP see both reads (read2 carries the UMI), but set + `"SEQUENCING": "single,rf"` under `MAPPING` (and `COUNTING` if needed) so + only read1 is actually mapped/counted. Steps downstream of MAPPING (PEAKS, + TRACKS, COUNTING, CIRCS, FUSIONS, DE, DEU, DAS) default to the resolved + MAPPING pairedness/strandedness, not the global one, when they don't set + their own — a MAPPING-level override already propagates to DE/DEU/DAS + automatically, e.g. so their featureCounts calls use the right `-p`/`-s` + flags. DTU is the exception: its default salmon path quantifies from the + trimmed FASTQ directly, independent of MAPPING, so it falls back to the + global value instead. - Inside each tool's block, `OPTIONS` holds one key per sub-step (e.g. `INDEX` vs `MAP` for a mapper) with a **raw command-line string** of extra flags for that tool — exactly as you'd type them, minus anything MONSDA diff --git a/.agents/skills/monsda/reference/workflows.md b/.agents/skills/monsda/reference/workflows.md index fa6ec3c4..d93d0e2b 100644 --- a/.agents/skills/monsda/reference/workflows.md +++ b/.agents/skills/monsda/reference/workflows.md @@ -10,7 +10,7 @@ can put under that step's `TOOLS` key. Env names match a `.yaml` in `envs/`. | `BASECALL` | FAST5/POD5 -> FASTQ | `guppy: $PATH_TO_LOCAL_BIN` (no conda pkg), `dorado: $PATH_TO_LOCAL_BIN` (no conda pkg) | | `QC` | QC of FASTQ/BAM (runs standalone pre-processing, or again post-processing for any later step) | `fastqc: fastqc` (FASTQ/BAM, includes MultiQC), `rustqc: rustqc` (BAM only, includes MultiQC, has a strandedness guard — aborts the run if inferred strandedness conflicts with `SEQUENCING`) | | `TRIMMING` | Adapter/quality trimming | `trimgalore: trim_galore`, `cutadapt: cutadapt`, `bbduk: bbduk`, `fastp: fastp` | -| `MAPPING` | Align to reference | `hisat2: hisat2`, `star: STAR` (also used for `STARsolo` single-cell mode), `rustar: rustar` (Rust STAR reimplementation), `segemehl2`/`segemehl3`/`segemehl2bisulfite`/`segemehl3bisulfite: segemehl.x`, `bwa: bwa mem`, `bwa2: bwa2-mem`, `bwameth: bwameth.py`, `minimap: minimap2` (long-read, use `-x map-ont`/`-x map-pb` in OPTIONS), `salmonalign: salmon` (alignment mode), `rammap: rammap` (long-read, not on bioconda), `piscem: piscem` (single-cell only, emits RAD for `alevinfry`) | +| `MAPPING` | Align to reference | `hisat2: hisat2`, `star: STAR` (also used for `STARsolo` single-cell mode), `rustar: rustar` (Rust STAR reimplementation), `segemehl2`/`segemehl3`/`segemehl2bisulfite`/`segemehl3bisulfite: segemehl.x`, `bwa: bwa mem`, `bwa2: bwa2-mem`, `bwameth: bwameth.py`, `bowtie2: bowtie2` (no native stranded-protocol flag, so the strandedness part of `SEQUENCING` is not forwarded to it), `minimap: minimap2` (long-read, use `-x map-ont`/`-x map-pb` in OPTIONS), `salmonalign: salmon` (alignment mode), `rammap: rammap` (long-read, not on bioconda), `piscem: piscem` (single-cell only, emits RAD for `alevinfry`) | | `DEDUP` | Remove/mark duplicates | `umitools: umi_tools`, `fgumi: fgumi` (UMI extraction + dedup), `picard: picard` (position-based) | | `COUNTING` | Quantify reads/transcripts | `countreads: featureCounts`, `salmon: salmon` (FASTQ or, via `OPTIONS.BAM = "transcriptome"\|"genome"`, directly from BAM), `kallisto: kallisto`, `oarfish: oarfish` (long-read, from transcriptome BAM), `alevinfry: alevin-fry` (needs a piscem RAD file), `simpleaf: simpleaf` (self-contained single-cell index+map+quant) | | `TRACKS` | trackdb.txt + BIGWIG for UCSC/genome browsers | `ucsc: ucsc` | diff --git a/AGENTS.md b/AGENTS.md index 2876616e..3e00b0f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ State clearly what was actually verified and what was not. `snakemake` and `nextflow` live in the `monsda` conda env, they are *not* on the default PATH: - export PATH=/home/fall/anaconda3/envs/monsda/bin:$PATH + export PATH=${HOME}/anaconda3/envs/monsda/bin:$PATH Confirmed there: snakemake 9.20.0, nextflow 26.04.0.12031. `multiqc` is **not** installed in that env, MultiQC behaviour cannot be verified locally. diff --git a/MONSDA/Params.py b/MONSDA/Params.py index 77be14a6..d8b57e50 100644 --- a/MONSDA/Params.py +++ b/MONSDA/Params.py @@ -1628,6 +1628,36 @@ def checkstranded(sample: list, config: dict) -> str: return stranded +def resolve_sequencing(seqval: str, default_paired: str, default_stranded: str): + """Resolve a per-workflow-step SEQUENCING override (e.g. 'single,rf') into + a (paired, stranded) tuple, falling back to the given defaults (usually + the global SETTINGS.SEQUENCING derived values, or the resolved MAPPING + values for steps downstream of MAPPING) when the step itself does not + define its own override. This allows e.g. a quantseq MAPPING step to run + in 'single' mode while TRIMMING/DEDUP still see the global 'paired' mode + needed for UMI extraction from read2. + + Parameters + ---------- + seqval : str + SEQUENCING value configured for the step, e.g. 'single,rf', or falsy + if not configured for this step. + default_paired : str + paired value to fall back to when seqval is not set. + default_stranded : str + stranded value to fall back to when seqval is not set. + + Returns + ------- + tuple(str, str) + (paired, stranded) + """ + if seqval and seqval != "": + parts = str(seqval).split(",") + return parts[0], (parts[1] if len(parts) > 1 else "") + return default_paired, default_stranded + + @check_run def set_pairing(samples: list, config: dict) -> list: """get pairs of samples diff --git a/MONSDA/Workflows.py b/MONSDA/Workflows.py index 1eef410b..3199acdf 100755 --- a/MONSDA/Workflows.py +++ b/MONSDA/Workflows.py @@ -652,11 +652,25 @@ def create_subworkflow(config, subwork, conditions, envs=None, stage=None): tempconf["RUNDEDUP"] = "enabled" continue if mu.sub_dict(config[subwork], condition).get("ANNOTATION"): - tempconf[subwork]["ANNOTATION"] = mu.sub_dict( + subwork_annotation = mu.sub_dict( config[subwork], condition )["ANNOTATION"] elif config[subwork].get("ANNOTATION"): - tempconf[subwork]["ANNOTATION"] = config[subwork]["ANNOTATION"] + subwork_annotation = config[subwork]["ANNOTATION"] + else: + subwork_annotation = None + # per-step SEQUENCING override (e.g. 'single,rf'), same + # condition-level lookup as ANNOTATION above; a tool-nested + # override (alongside a tool's OPTIONS) is already carried + # through via the per-tool subset_dict merge below. + if mu.sub_dict(config[subwork], condition).get("SEQUENCING"): + subwork_sequencing = mu.sub_dict( + config[subwork], condition + )["SEQUENCING"] + elif config[subwork].get("SEQUENCING"): + subwork_sequencing = config[subwork]["SEQUENCING"] + else: + subwork_sequencing = None if ("TOOLS" in config[key] and env == "" and exe == "") and len( mu.get_from_dict(config[key], condition) ) != 0: # env and exe overrule TOOLS @@ -674,6 +688,14 @@ def create_subworkflow(config, subwork, conditions, envs=None, stage=None): tempconf[key] = mu.subset_dict(config[key], condition) else: tempconf[key] = mu.subset_dict(config[key], condition) + # (Re-)apply the ANNOTATION/SEQUENCING overrides: the TOOLS + # handling above can replace tempconf[key] wholesale (e.g. + # when no envs filter is given, as for DE/DEU/DAS/DTU), + # which would otherwise silently drop them again. + if subwork_annotation: + tempconf[subwork]["ANNOTATION"] = subwork_annotation + if subwork_sequencing: + tempconf[subwork]["SEQUENCING"] = subwork_sequencing if stage and stage == "POST": tempconf["SAMPLES"] = mp.get_samples_postprocess(config, key) @@ -1968,6 +1990,9 @@ def nf_fetch_params( retconf["PAIRED"] = paired if stranded: retconf["STRANDED"] = stranded + # Per-step pairedness/strandedness, default to the global values above and + # overridden below wherever a workflow step defines its own SEQUENCING key. + MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded if rundedup: retconf["RUNDEDUP"] = rundedup if prededup: @@ -1986,6 +2011,10 @@ def nf_fetch_params( REF = mapper_dict.get("REFERENCE", MAPCONF.get("REFERENCE")) MANNO = mapper_dict.get("ANNOTATION", MAPCONF.get("ANNOTATION")) MDECOY = mapper_dict.get("DECOY", MAPCONF.get("DECOY")) + MSEQ = mapper_dict.get("SEQUENCING", MAPCONF.get("SEQUENCING")) + MAPPING_PAIRED, MAPPING_STRANDED = mp.resolve_sequencing( + MSEQ, paired, stranded + ) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -2070,6 +2099,8 @@ def nf_fetch_params( retconf["MAPPINGUIDXNAME"] = UIDXNAME retconf["MAPPINGUIDX2NAME"] = UIDX2NAME retconf["MAPPINGPREFIX"] = PREFIX + retconf["MAPPINGPAIRED"] = MAPPING_PAIRED + retconf["MAPPINGSTRANDED"] = MAPPING_STRANDED # Peak Calling Variables if "PEAKS" in config: @@ -2085,6 +2116,13 @@ def nf_fetch_params( ANNOTATION = ( ANNO.get("GTF") if "GTF" in ANNO else ANNO.get("GFF") ) # by default GTF forma + # downstream steps consume the MAPPING output, so default to its pairedness + PSEQ = PEAKCONF.get("SEQUENCING") + PEAKS_PAIRED, PEAKS_STRANDED = mp.resolve_sequencing( + PSEQ, MAPPING_PAIRED, MAPPING_STRANDED + ) + retconf["PEAKPAIRED"] = PEAKS_PAIRED + retconf["PEAKSTRANDED"] = PEAKS_STRANDED if not IP: IP = mp.check_IP(SAMPLES, config) log.debug(logid + "Running Peak finding for " + IP + " protocol") @@ -2133,9 +2171,29 @@ def nf_fetch_params( XCONF = mu.sub_dict(config[x], SETUP) XENV = XENV.split("_")[0] log.debug(logid + "XCONFIG: " + str(SETUP) + "\t" + str(XCONF)) - REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE")) - XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION")) - XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY")) + # Subconfigs from the non-combo nf_make_sub path have the toolenv + # key stripped out (same as MAPPING above); fall back to XCONF + # itself when the toolenv key is absent, otherwise REFERENCE/ + # ANNOTATION/DECOY/SEQUENCING would silently raise KeyError/be lost. + x_dict = XCONF.get(XENV) if mu.dict_inst(XCONF) else None + if not mu.dict_inst(x_dict): + x_dict = XCONF + REF = x_dict.get( + "REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE")) + ) + XANNO = x_dict.get( + "ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION")) + ) + XDECOY = x_dict.get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = x_dict.get( + "SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING")) + ) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + x_paired, x_stranded = mp.resolve_sequencing( + XSEQ, MAPPING_PAIRED, MAPPING_STRANDED + ) + retconf[x + "PAIRED"] = x_paired + retconf[x + "STRANDED"] = x_stranded if XANNO: ANNOTATION = XANNO else: @@ -2194,9 +2252,36 @@ def nf_fetch_params( XBIN, XENV = mp.env_bin_from_config(config, x) XENV = XENV.split("_")[0] log.debug(logid + "XCONFIG: " + str(SETUP) + "\t" + str(XCONF)) - REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE")) - XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION")) - XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY")) + # Subconfigs from the non-combo nf_make_sub path have the toolenv + # key stripped out (same as MAPPING/TRACKS/COUNTING above); fall + # back to XCONF itself when the toolenv key is absent, otherwise + # REFERENCE/ANNOTATION/DECOY/SEQUENCING would silently raise + # KeyError/be lost. + x_dict = XCONF.get(XENV) if mu.dict_inst(XCONF) else None + if not mu.dict_inst(x_dict): + x_dict = XCONF + REF = x_dict.get( + "REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE")) + ) + XANNO = x_dict.get( + "ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION")) + ) + XDECOY = x_dict.get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = x_dict.get( + "SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING")) + ) + if x == "DTU": + # DTU's default salmon path quantifies directly from + # TRIMMED_FASTQ, independent of the MAPPING step, so fall + # back to the global layout + x_paired, x_stranded = mp.resolve_sequencing(XSEQ, paired, stranded) + else: + # DE/DEU/DAS consume the MAPPING output, so default to its pairedness + x_paired, x_stranded = mp.resolve_sequencing( + XSEQ, MAPPING_PAIRED, MAPPING_STRANDED + ) + retconf[x + "PAIRED"] = x_paired + retconf[x + "STRANDED"] = x_stranded if XANNO: ANNOTATION = XANNO else: @@ -2319,9 +2404,16 @@ def nf_fetch_params( ANNO.get("GTF") if "GTF" in ANNO else ANNO.get("GFF") ) # by default GTF format will be used + CSEQ = CIRCCONF.get("SEQUENCING") + CIRCS_PAIRED, CIRCS_STRANDED = mp.resolve_sequencing( + CSEQ, MAPPING_PAIRED, MAPPING_STRANDED + ) + retconf["CIRCSREF"] = REFERENCE retconf["CIRCSREFDIR"] = REFDIR retconf["CIRCSANNO"] = ANNOTATION + retconf["CIRCSPAIRED"] = CIRCS_PAIRED + retconf["CIRCSSTRANDED"] = CIRCS_STRANDED # FUSIONS Variables if "FUSIONS" in config: @@ -2340,9 +2432,16 @@ def nf_fetch_params( ANNO.get("GTF") if "GTF" in ANNO else ANNO.get("GFF") ) # by default GTF format will be used + FSEQ = FUSCONF.get("SEQUENCING") + FUSIONS_PAIRED, FUSIONS_STRANDED = mp.resolve_sequencing( + FSEQ, MAPPING_PAIRED, MAPPING_STRANDED + ) + retconf["FUSIONSREF"] = REFERENCE retconf["FUSIONSREFDIR"] = REFDIR retconf["FUSIONSANNO"] = ANNOTATION + retconf["FUSIONSPAIRED"] = FUSIONS_PAIRED + retconf["FUSIONSSTRANDED"] = FUSIONS_STRANDED retconf["FUSIONSLIB"] = FUSCONF.get(FUSENV, {}).get("OPTIONS", {}).get( "INDEX", "" ) or os.path.join(REFDIR, "CTAT", FUSENV) diff --git a/configs/template.json b/configs/template.json index 1ed68f3d..3d55d57d 100644 --- a/configs/template.json +++ b/configs/template.json @@ -163,6 +163,7 @@ "hisat2": "hisat2", "bwa": "bwa mem", "bwameth": "bwameth", + "bowtie2": "bowtie2", "minimap": "minimap2" }, "id": { @@ -219,6 +220,15 @@ "MAP": "", "EXTENSION" : "" } + }, + "bowtie2": { + "OPTIONS": + { + "INDEX" : "", + "MAP": "", + "EXTENSION" : "" + }, + "SEQUENCING": "single,rf" # per-step override, e.g. map only read1 for quantseq-style UMI protocols while TRIMMING/DEDUP keep the global paired SEQUENCING from SETTINGS } } } diff --git a/configs/template_base_commented.json b/configs/template_base_commented.json index 61c766c9..f761a030 100644 --- a/configs/template_base_commented.json +++ b/configs/template_base_commented.json @@ -212,6 +212,7 @@ "hisat2": "hisat2", "bwa": "bwa mem", "bwameth": "bwameth.py", + "bowtie2": "bowtie2", "minimap": "minimap2", "salmonalign": "salmon", "rammap": "rammap", @@ -291,6 +292,19 @@ "EXTENSION" : "" } }, + "bowtie2": { + "comment": { + "INDEX" : "Indexing options", + "MAP": "Mapping options", + "EXTENSION" : "Prefix for index name, can be left empty.", + "SEQUENCING": "OPTIONAL per-step override of SETTINGS.SEQUENCING, e.g. 'single,rf' to map only read1 (e.g. for QuantSeq-style protocols where read2 carries the UMI used by DEDUP/TRIMMING). Bowtie2 has no native stranded-protocol flag, so only the paired/single part has an effect here." + }, + "OPTIONS": { + "INDEX" : "", + "MAP": "", + "EXTENSION" : "" + } + }, "minimap": { "comment": { "INDEX" : "Indexing options", @@ -332,7 +346,7 @@ "INDEX" : "", "MAP": "" } - }, + } }, "COUNTING": { "TOOLS": { @@ -486,7 +500,7 @@ "COUNTTOOL": "Gene-level counting tool used for DE, like the TOOLS key but for the counting step. 'featureCounts' for short-read BAMs (default). 'oarfish' aggregates long-read (PacBio/Nanopore) transcript-level quantification from the genome-mapped BAM to gene level.", "COUNT": "Options for read counting, independent of COUNTS workflows, utilizes FeatureCounts", "QUANT": "oarfish quant options, only used when COUNTTOOL is 'oarfish'", - "DE": "Options for DE Analysis, e.g. path to spike-in" + "DE": "Options for DE Analysis as key=value pairs. 'controlgenes=' is a file with one spike-in/control gene ID per line. Setting it does not change the standard analysis, it adds two more readings of every contrast: '_norm' uses the control genes as size factors (absolute, per cell fold changes) and '_ruv' adds the RUVg factors of unwanted variation W_1..W_k to the design while keeping the standard normalization (RUVSeq vignette). Control genes are never tested themselves. 'ruvk=auto|' sets the number of RUVg factors, default 'auto' estimates it from the control genes by a permutation test on the variance spectrum, 'ruvk=0' drops the W covariates so that '_ruv' reproduces the standard branch. The per-k diagnostics are written to *_table_ruv_k_selection.tsv and *_figure_RUV_k_selection.png. 'design=' sets the model formula per contrast, e.g. 'design=batch+condition' (the leading ~ is optional). It may use condition, type and batch, which are filled from GROUPS, TYPES and BATCHES, must contain condition and must keep the intercept. Without it MONSDA derives the design from the types and batches present in the two groups. Terms that are not estimable for a given contrast are dropped with a warning" }, "OPTIONS": { "COUNTTOOL": "featureCounts", @@ -500,7 +514,7 @@ "COUNTTOOL": "Gene-level counting tool used for DE, like the TOOLS key but for the counting step. 'featureCounts' for short-read BAMs (default). 'oarfish' aggregates long-read (PacBio/Nanopore) transcript-level quantification from the genome-mapped BAM to gene level.", "COUNT": "Options for read counting, independent of COUNTS workflows, utilizes FeatureCounts", "QUANT": "oarfish quant options, only used when COUNTTOOL is 'oarfish'", - "DE": "Options for DE Analysis, e.g. path to spike-in" + "DE": "Options for DE Analysis as key=value pairs. 'controlgenes=' is a file with one spike-in/control gene ID per line. Setting it does not change the standard analysis, it adds two more readings of every contrast: '_norm' uses the control genes as size factors (absolute, per cell fold changes) and '_ruv' adds the RUVg factors of unwanted variation W_1..W_k to the design while keeping the standard normalization (RUVSeq vignette). Control genes are never tested themselves. 'ruvk=auto|' sets the number of RUVg factors, default 'auto' estimates it from the control genes by a permutation test on the variance spectrum, 'ruvk=0' drops the W covariates so that '_ruv' reproduces the standard branch. The per-k diagnostics are written to *_table_ruv_k_selection.tsv and *_figure_RUV_k_selection.png. 'design=' sets the model formula per contrast, e.g. 'design=batch+condition' (the leading ~ is optional). It may use condition, type and batch, which are filled from GROUPS, TYPES and BATCHES, must contain condition and must keep the intercept. Without it MONSDA derives the design from the types and batches present in the two groups. Terms that are not estimable for a given contrast are dropped with a warning" }, "OPTIONS": { "COUNTTOOL": "featureCounts", diff --git a/containers/apptainer/LoveSonesonPatro.def b/containers/apptainer/LoveSonesonPatro.def index b73001c6..57ea22a2 100644 --- a/containers/apptainer/LoveSonesonPatro.def +++ b/containers/apptainer/LoveSonesonPatro.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/LoveSonesonPatro.yaml /opt/envs/ + ${HOME}/MONSDA/envs/LoveSonesonPatro.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/base.def b/containers/apptainer/base.def index 1b27c8f2..debd3975 100644 --- a/containers/apptainer/base.def +++ b/containers/apptainer/base.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/base.yaml /opt/envs/ + ${HOME}/MONSDA/envs/base.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bbduk.def b/containers/apptainer/bbduk.def index c6a48816..35f1494f 100644 --- a/containers/apptainer/bbduk.def +++ b/containers/apptainer/bbduk.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bbduk.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bbduk.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bbmap.def b/containers/apptainer/bbmap.def index c2c72ac9..149945cc 100644 --- a/containers/apptainer/bbmap.def +++ b/containers/apptainer/bbmap.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bbmap.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bbmap.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bedtools.def b/containers/apptainer/bedtools.def index c50fbcd5..a4024c31 100644 --- a/containers/apptainer/bedtools.def +++ b/containers/apptainer/bedtools.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bedtools.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bedtools.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bowtie2.def b/containers/apptainer/bowtie2.def new file mode 100644 index 00000000..4f5702d2 --- /dev/null +++ b/containers/apptainer/bowtie2.def @@ -0,0 +1,24 @@ +Bootstrap: docker +From: continuumio/miniconda3:latest + +%files + ${HOME}/MONSDA/envs/bowtie2.yaml /opt/envs/ + ${HOME}/MONSDA/scripts /opt/MONSDA/ + +%environment + +%post + find /opt/MONSDA/scripts -type f \( -name '*.py' -o -name '*.R' -o -name '*.sh' -o -name '*.pl' \) -exec chmod a+x {} + + ls -alrt /opt/envs + chmod -R +x /opt/envs/bowtie2.yaml + + ENV_NAME=bowtie2 + echo ". /opt/conda/etc/profile.d/conda.sh" >> $APPTAINER_ENVIRONMENT + echo "conda activate $ENV_NAME" >> $APPTAINER_ENVIRONMENT + + . /opt/conda/etc/profile.d/conda.sh + conda env create -f /opt/envs/bowtie2.yaml -p /opt/conda/envs/$ENV_NAME + conda clean --all + +%runscript + exec "$@" diff --git a/containers/apptainer/bwa.def b/containers/apptainer/bwa.def index b4dbb047..4581b357 100644 --- a/containers/apptainer/bwa.def +++ b/containers/apptainer/bwa.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bwa.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bwa.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bwa2.def b/containers/apptainer/bwa2.def index 7461bb89..24ace263 100644 --- a/containers/apptainer/bwa2.def +++ b/containers/apptainer/bwa2.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bwa2.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bwa2.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/bwameth.def b/containers/apptainer/bwameth.def index 920807fe..2eec04fc 100644 --- a/containers/apptainer/bwameth.def +++ b/containers/apptainer/bwameth.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/bwameth.yaml /opt/envs/ + ${HOME}/MONSDA/envs/bwameth.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/ciri2.def b/containers/apptainer/ciri2.def index 7a1173e3..a23ba2ed 100644 --- a/containers/apptainer/ciri2.def +++ b/containers/apptainer/ciri2.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/ciri2.yaml /opt/envs/ + ${HOME}/MONSDA/envs/ciri2.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/countreads.def b/containers/apptainer/countreads.def index 540a9786..eff1e399 100644 --- a/containers/apptainer/countreads.def +++ b/containers/apptainer/countreads.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/countreads.yaml /opt/envs/ + ${HOME}/MONSDA/envs/countreads.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/countreads_de.def b/containers/apptainer/countreads_de.def index 00620c4f..520c72cc 100644 --- a/containers/apptainer/countreads_de.def +++ b/containers/apptainer/countreads_de.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/countreads_de.yaml /opt/envs/ + ${HOME}/MONSDA/envs/countreads_de.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/cutadapt.def b/containers/apptainer/cutadapt.def index 2121f754..52f120c6 100644 --- a/containers/apptainer/cutadapt.def +++ b/containers/apptainer/cutadapt.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/cutadapt.yaml /opt/envs/ + ${HOME}/MONSDA/envs/cutadapt.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/deseq2_DE.def b/containers/apptainer/deseq2_DE.def index 2f82ce09..98c68aee 100644 --- a/containers/apptainer/deseq2_DE.def +++ b/containers/apptainer/deseq2_DE.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/deseq2_DE.yaml /opt/envs/ + ${HOME}/MONSDA/envs/deseq2_DE.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/dexseq_DEU.def b/containers/apptainer/dexseq_DEU.def index 9b18be79..758503f2 100644 --- a/containers/apptainer/dexseq_DEU.def +++ b/containers/apptainer/dexseq_DEU.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/dexseq_DEU.yaml /opt/envs/ + ${HOME}/MONSDA/envs/dexseq_DEU.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/dexseq_DTU.def b/containers/apptainer/dexseq_DTU.def index 59e10b8e..74aff87c 100644 --- a/containers/apptainer/dexseq_DTU.def +++ b/containers/apptainer/dexseq_DTU.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/dexseq_DTU.yaml /opt/envs/ + ${HOME}/MONSDA/envs/dexseq_DTU.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/diego_DAS.def b/containers/apptainer/diego_DAS.def index 1d18b300..56fc477b 100644 --- a/containers/apptainer/diego_DAS.def +++ b/containers/apptainer/diego_DAS.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/diego_DAS.yaml /opt/envs/ + ${HOME}/MONSDA/envs/diego_DAS.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/dorado.def b/containers/apptainer/dorado.def index 6d02bdfe..add7c5f3 100644 --- a/containers/apptainer/dorado.def +++ b/containers/apptainer/dorado.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/dorado.yaml /opt/envs/ + ${HOME}/MONSDA/envs/dorado.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/drimseq_DTU.def b/containers/apptainer/drimseq_DTU.def index 9c5b2273..02d7a2e0 100644 --- a/containers/apptainer/drimseq_DTU.def +++ b/containers/apptainer/drimseq_DTU.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/drimseq_DTU.yaml /opt/envs/ + ${HOME}/MONSDA/envs/drimseq_DTU.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/edger_DAS.def b/containers/apptainer/edger_DAS.def index 93b365ae..1355a3d9 100644 --- a/containers/apptainer/edger_DAS.def +++ b/containers/apptainer/edger_DAS.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/edger_DAS.yaml /opt/envs/ + ${HOME}/MONSDA/envs/edger_DAS.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/edger_DE.def b/containers/apptainer/edger_DE.def index c9f5a3b9..1394a8b9 100644 --- a/containers/apptainer/edger_DE.def +++ b/containers/apptainer/edger_DE.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/edger_DE.yaml /opt/envs/ + ${HOME}/MONSDA/envs/edger_DE.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/edger_DEU.def b/containers/apptainer/edger_DEU.def index ec66e905..cfed09fe 100644 --- a/containers/apptainer/edger_DEU.def +++ b/containers/apptainer/edger_DEU.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/edger_DEU.yaml /opt/envs/ + ${HOME}/MONSDA/envs/edger_DEU.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/fastp.def b/containers/apptainer/fastp.def index 5dca52c8..56277e0a 100644 --- a/containers/apptainer/fastp.def +++ b/containers/apptainer/fastp.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/fastp.yaml /opt/envs/ + ${HOME}/MONSDA/envs/fastp.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/fastqc.def b/containers/apptainer/fastqc.def index 51095502..521bc583 100644 --- a/containers/apptainer/fastqc.def +++ b/containers/apptainer/fastqc.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/fastqc.yaml /opt/envs/ + ${HOME}/MONSDA/envs/fastqc.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/fgumi.def b/containers/apptainer/fgumi.def index 4e8add5e..f13a9abd 100644 --- a/containers/apptainer/fgumi.def +++ b/containers/apptainer/fgumi.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/fgumi.yaml /opt/envs/ + ${HOME}/MONSDA/envs/fgumi.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/guppy.def b/containers/apptainer/guppy.def index 3472fee8..5198213e 100644 --- a/containers/apptainer/guppy.def +++ b/containers/apptainer/guppy.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/guppy.yaml /opt/envs/ + ${HOME}/MONSDA/envs/guppy.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/hisat2.def b/containers/apptainer/hisat2.def index 00ab1bb6..40dadb2f 100644 --- a/containers/apptainer/hisat2.def +++ b/containers/apptainer/hisat2.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/hisat2.yaml /opt/envs/ + ${HOME}/MONSDA/envs/hisat2.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/index.def b/containers/apptainer/index.def index 8705ea85..ccb7f86f 100644 --- a/containers/apptainer/index.def +++ b/containers/apptainer/index.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/index.yaml /opt/envs/ + ${HOME}/MONSDA/envs/index.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/isoformswitchanalyzer.def b/containers/apptainer/isoformswitchanalyzer.def index 8d337291..411977c8 100644 --- a/containers/apptainer/isoformswitchanalyzer.def +++ b/containers/apptainer/isoformswitchanalyzer.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/isoformswitchanalyzer.yaml /opt/envs/ + ${HOME}/MONSDA/envs/isoformswitchanalyzer.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/kallisto.def b/containers/apptainer/kallisto.def index 90cd3ba5..16eccc1e 100644 --- a/containers/apptainer/kallisto.def +++ b/containers/apptainer/kallisto.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/kallisto.yaml /opt/envs/ + ${HOME}/MONSDA/envs/kallisto.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/macs.def b/containers/apptainer/macs.def index 795b2946..55672190 100644 --- a/containers/apptainer/macs.def +++ b/containers/apptainer/macs.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/macs.yaml /opt/envs/ + ${HOME}/MONSDA/envs/macs.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/minimap.def b/containers/apptainer/minimap.def index fdb79d96..3ac17f78 100644 --- a/containers/apptainer/minimap.def +++ b/containers/apptainer/minimap.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/minimap.yaml /opt/envs/ + ${HOME}/MONSDA/envs/minimap.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/monsda.def b/containers/apptainer/monsda.def index c4ee84be..939ef988 100644 --- a/containers/apptainer/monsda.def +++ b/containers/apptainer/monsda.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/monsda.yaml /opt/envs/ + ${HOME}/MONSDA/envs/monsda.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/peaks.def b/containers/apptainer/peaks.def index cc2fd306..2511ec02 100644 --- a/containers/apptainer/peaks.def +++ b/containers/apptainer/peaks.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/peaks.yaml /opt/envs/ + ${HOME}/MONSDA/envs/peaks.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/perl.def b/containers/apptainer/perl.def index 1cb93b79..f697d330 100644 --- a/containers/apptainer/perl.def +++ b/containers/apptainer/perl.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/perl.yaml /opt/envs/ + ${HOME}/MONSDA/envs/perl.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/picard.def b/containers/apptainer/picard.def index 778718af..f4421a8a 100644 --- a/containers/apptainer/picard.def +++ b/containers/apptainer/picard.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/picard.yaml /opt/envs/ + ${HOME}/MONSDA/envs/picard.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/piranha.def b/containers/apptainer/piranha.def index cf2eda39..8dfbff06 100644 --- a/containers/apptainer/piranha.def +++ b/containers/apptainer/piranha.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/piranha.yaml /opt/envs/ + ${HOME}/MONSDA/envs/piranha.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/qc.def b/containers/apptainer/qc.def index 2797dfec..0c4fbf2f 100644 --- a/containers/apptainer/qc.def +++ b/containers/apptainer/qc.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/qc.yaml /opt/envs/ + ${HOME}/MONSDA/envs/qc.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/rammap.def b/containers/apptainer/rammap.def index f23a56b5..b8b54c7d 100644 --- a/containers/apptainer/rammap.def +++ b/containers/apptainer/rammap.def @@ -2,8 +2,8 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/Gits/Monsda/envs/rammap.yaml /opt/envs/ - /home/fall/Gits/Monsda/scripts /opt/MONSDA/ + ${HOME}/Gits/Monsda/envs/rammap.yaml /opt/envs/ + ${HOME}/Gits/Monsda/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/rastqc.def b/containers/apptainer/rastqc.def index 065107b8..cf220d27 100644 --- a/containers/apptainer/rastqc.def +++ b/containers/apptainer/rastqc.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/rastqc.yaml /opt/envs/ + ${HOME}/MONSDA/envs/rastqc.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/rustar.def b/containers/apptainer/rustar.def index 2a7d1afd..eebb1da2 100644 --- a/containers/apptainer/rustar.def +++ b/containers/apptainer/rustar.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/rustar.yaml /opt/envs/ + ${HOME}/MONSDA/envs/rustar.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/rustqc.def b/containers/apptainer/rustqc.def index 841fc35c..b2932b09 100644 --- a/containers/apptainer/rustqc.def +++ b/containers/apptainer/rustqc.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3:latest %files - /home/fall/MONSDA/envs/rustqc.yaml /opt/envs/ + ${HOME}/MONSDA/envs/rustqc.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/salmon.def b/containers/apptainer/salmon.def index 26cc15f4..bcc04f57 100644 --- a/containers/apptainer/salmon.def +++ b/containers/apptainer/salmon.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/salmon.yaml /opt/envs/ + ${HOME}/MONSDA/envs/salmon.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/samtools.def b/containers/apptainer/samtools.def index 9182a7e5..6adffb6e 100644 --- a/containers/apptainer/samtools.def +++ b/containers/apptainer/samtools.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/samtools.yaml /opt/envs/ + ${HOME}/MONSDA/envs/samtools.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/scyphy.def b/containers/apptainer/scyphy.def index 4d960b53..cdd3b097 100644 --- a/containers/apptainer/scyphy.def +++ b/containers/apptainer/scyphy.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/scyphy.yaml /opt/envs/ + ${HOME}/MONSDA/envs/scyphy.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/segemehl.def b/containers/apptainer/segemehl.def index 306399ee..ba56c638 100644 --- a/containers/apptainer/segemehl.def +++ b/containers/apptainer/segemehl.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/segemehl.yaml /opt/envs/ + ${HOME}/MONSDA/envs/segemehl.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/segemehl3.def b/containers/apptainer/segemehl3.def index ac6b63fc..8c978b0f 100644 --- a/containers/apptainer/segemehl3.def +++ b/containers/apptainer/segemehl3.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/segemehl3.yaml /opt/envs/ + ${HOME}/MONSDA/envs/segemehl3.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/spit_DTU.def b/containers/apptainer/spit_DTU.def index 81b329cb..9ff63c1c 100644 --- a/containers/apptainer/spit_DTU.def +++ b/containers/apptainer/spit_DTU.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/spit_DTU.yaml /opt/envs/ + ${HOME}/MONSDA/envs/spit_DTU.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/sra.def b/containers/apptainer/sra.def index 17bc2896..628183c8 100644 --- a/containers/apptainer/sra.def +++ b/containers/apptainer/sra.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/sra.yaml /opt/envs/ + ${HOME}/MONSDA/envs/sra.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/star.def b/containers/apptainer/star.def index daeac763..c408992d 100644 --- a/containers/apptainer/star.def +++ b/containers/apptainer/star.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/star.yaml /opt/envs/ + ${HOME}/MONSDA/envs/star.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/starfusion.def b/containers/apptainer/starfusion.def index 013cf5d3..7089a96a 100644 --- a/containers/apptainer/starfusion.def +++ b/containers/apptainer/starfusion.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/starfusion.yaml /opt/envs/ + ${HOME}/MONSDA/envs/starfusion.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/summary.def b/containers/apptainer/summary.def index fb144944..176c1942 100644 --- a/containers/apptainer/summary.def +++ b/containers/apptainer/summary.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/summary.yaml /opt/envs/ + ${HOME}/MONSDA/envs/summary.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/trimgalore.def b/containers/apptainer/trimgalore.def index 78090536..77904aa3 100644 --- a/containers/apptainer/trimgalore.def +++ b/containers/apptainer/trimgalore.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/trimgalore.yaml /opt/envs/ + ${HOME}/MONSDA/envs/trimgalore.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/trimm.def b/containers/apptainer/trimm.def index 466646bd..e28f541b 100644 --- a/containers/apptainer/trimm.def +++ b/containers/apptainer/trimm.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/trimm.yaml /opt/envs/ + ${HOME}/MONSDA/envs/trimm.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/trimmomatic.def b/containers/apptainer/trimmomatic.def index 965b59dc..c5ce7294 100644 --- a/containers/apptainer/trimmomatic.def +++ b/containers/apptainer/trimmomatic.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/trimmomatic.yaml /opt/envs/ + ${HOME}/MONSDA/envs/trimmomatic.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/trnascan.def b/containers/apptainer/trnascan.def index 53e73e1c..80e341bf 100644 --- a/containers/apptainer/trnascan.def +++ b/containers/apptainer/trnascan.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/trnascan.yaml /opt/envs/ + ${HOME}/MONSDA/envs/trnascan.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/ucsc.def b/containers/apptainer/ucsc.def index 67b98789..4ddabcab 100644 --- a/containers/apptainer/ucsc.def +++ b/containers/apptainer/ucsc.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/ucsc.yaml /opt/envs/ + ${HOME}/MONSDA/envs/ucsc.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/umitools.def b/containers/apptainer/umitools.def index c3844b78..dbcdfd99 100644 --- a/containers/apptainer/umitools.def +++ b/containers/apptainer/umitools.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/umitools.yaml /opt/envs/ + ${HOME}/MONSDA/envs/umitools.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/containers/apptainer/zip.def b/containers/apptainer/zip.def index c6602523..2d8ef14c 100644 --- a/containers/apptainer/zip.def +++ b/containers/apptainer/zip.def @@ -2,7 +2,7 @@ Bootstrap: docker From: continuumio/miniconda3 %files - /home/fall/MONSDA/envs/zip.yaml /opt/envs/ + ${HOME}/MONSDA/envs/zip.yaml /opt/envs/ ${HOME}/MONSDA/scripts /opt/MONSDA/ %environment diff --git a/docs/source/config.rst b/docs/source/config.rst index 9e4f8949..72c44b49 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -32,6 +32,8 @@ Now the actual workflow section begins, where you can define which environments You can always define differing ENV/BIN keys for each condition-tree leaf separately, which will overwrite the ENV/BIN set under 'TOOLS'. This is intended to be used to benchmark different versions of the same tool for example. Furthermore, you can define keys like 'ANNOTATION' and 'REFERENCE' similarly, overwriting the ones defined under 'SETTINGS', which is useful for example, in case you run workflows like 'COUNTING' with salmon and want to quantify transcript isoforms while still being able to map and trim and count reads on genome level. +The *SEQUENCING* key can likewise be overridden per workflow step (and per tool within a step), following the same ``"paired"``/``"single"``/``"singlecell"`` plus optional strandedness syntax as under 'SETTINGS'. The global value from 'SETTINGS' remains in effect for every step unless a step defines its own 'SEQUENCING' key. This is useful e.g. for QuantSeq-style 3' protocols, where read2 carries the UMI needed for DEDUP/UMI extraction but only read1 should actually be mapped and counted: set the global 'SEQUENCING' to ``"paired,rf"`` so TRIMMING/DEDUP see both reads, and set 'SEQUENCING' under 'MAPPING' (and, if needed, 'COUNTING') to ``"single,rf"`` so mapping and downstream steps only use read1. Steps downstream of MAPPING (PEAKS, TRACKS, COUNTING, CIRCS, FUSIONS, DE, DEU, DAS) fall back to the resolved MAPPING pairedness/strandedness rather than the global 'SETTINGS' value when they do not define their own override, since they consume the MAPPING output (e.g. their featureCounts calls). This means an override set under 'MAPPING' already propagates automatically to DE/DEU/DAS, but it can also be overridden again specifically for one of these steps if needed. DTU is the exception: its default salmon-based quantification works from the trimmed FASTQ files directly, independent of MAPPING, so it falls back to the global 'SETTINGS' value instead. + The next key-level is the *OPTIONS* key which is where you can define additional parameters for each tool. It is not needed to define anything related to *single-/paired-* end or *singlecell* sequencing, this is done automatically. To add parameters simply add the *OPTION* key which defines a dict where you can set parameters for each defined subworkflow-step. Parameters are here defined as key/value pairs corresponding to the subworkflow-step, e.g. 'INDEX' to generate an index file for mapping and all settings similar to a command line call as values. This should become clear having a look at the different processing steps in the template json. If there are no options just leave the 'OPTIONS' dict empty. .. note:: diff --git a/docs/source/workflows.rst b/docs/source/workflows.rst index 4c2e4a25..ebbaf771 100644 --- a/docs/source/workflows.rst +++ b/docs/source/workflows.rst @@ -187,6 +187,8 @@ Maps sequences to reference genomes or transcriptomes +-----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+------------+------------------------------------------------------------------------------+---------------------+------------+ | BWA2 | BWA is a software package for mapping low-divergent sequences against a large reference genome | bwa2 | bwa2-mem | `bwa2 `_ | FASTQ/TRIMMED_FASTQ | SAM.gz/BAM | +-----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+------------+------------------------------------------------------------------------------+---------------------+------------+ + | Bowtie2 | Bowtie2 is a fast and memory-efficient tool for aligning short reads to reference sequences. No native stranded-protocol flag, so per-step SEQUENCING strandedness is not forwarded to it. | bowtie2 | bowtie2 | `bowtie2 `_ | FASTQ/TRIMMED_FASTQ | SAM.gz/BAM | + +-----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+------------+------------------------------------------------------------------------------+---------------------+------------+ | BWA-Meth | BWA-meth, Fast and accurante alignment of BS-Seq reads. | bwameth | bwameth.py | `bwa-meth `_ | FASTQ/TRIMMED_FASTQ | SAM.gz/BAM | +-----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+------------+------------------------------------------------------------------------------+---------------------+------------+ | Minimap2 | Minimap2 is a versatile sequence alignment program that aligns DNA or mRNA sequences against a large reference database. Use ``-x map-ont``/``-x map-pb`` for long-read (Nanopore/PacBio) mapping. | minimap | minimap2 | `minimap `_ | FASTQ/TRIMMED_FASTQ | SAM.gz/BAM | @@ -288,6 +290,16 @@ Includes DE, DEU, DAS and DTU DEU (DEXSeq/edgeR) and DAS (DIEGO/edgeR) accept the same ``COUNTTOOL`` option for consistency; currently only the default ``featureCounts`` is supported. + The model formula used for each DE contrast can be set with the ``DE`` option ``design=``, for example ``design=batch+condition`` (the leading ``~`` is optional and may be omitted, which avoids quoting issues when the option string is passed on unquoted). It may reference ``condition``, ``type`` and ``batch``, the three columns MONSDA fills from ``GROUPS``, ``TYPES`` and ``BATCHES``. The formula must contain a ``condition`` main effect, since that is the term the pairwise contrast is read off, and it must keep the intercept: both engines take the contrast from the treatment-contrast coefficient (``condition__vs_`` in DESeq2, ``condition`` in edgeR), which does not exist in a ``~ 0 + condition`` parameterisation. Without the option MONSDA keeps deriving the design per contrast as before, adding ``type`` and ``batch`` only where they vary inside the two groups being compared. In either case the design is reduced to the terms that are estimable for the contrast at hand: a term with only one level left in the subset, a term confounded with the rest of the design, or a term that would consume the last residual degree of freedom is dropped and warned about, so one degenerate contrast does not abort the others. An interaction such as ``design=condition*batch`` is accepted but warned about, since both engines then report the condition effect at the reference level of the interacting variable rather than an effect averaged over it. The design is logged per contrast, and the ``_ruv`` branch adds its ``W_1..W_k`` on top of whichever design was used. + + DE (DESeq2/edgeR) can make use of spike-ins with the ``DE`` option ``controlgenes=`` (one control gene ID per line). The control genes are never tested themselves, they are removed from the count matrix before fitting. Setting the option does not replace the standard analysis, it adds two further readings of the same contrast, because spike-in size factors and RUVg answer different questions and must not be stacked on top of each other. Each contrast is therefore reported three times, distinguished by a suffix on the output files: + + * no suffix -- the standard analysis, unchanged: DESeq2 median of ratios resp. edgeR TMM over the endogenous genes, design as configured, spike-ins not involved beyond being dropped. Fold changes are per library, i.e. relative to total RNA. + * ``_norm`` -- the spike-ins set the scale. Size factors are the median of ratios over the control genes (identical to DESeq2's ``estimateSizeFactors(controlGenes=)``, installed as offsets on the edgeR side so both engines use the same scale), the design is left unchanged and no ``W`` covariates are added. This is the branch that reports absolute, per-cell fold changes and the only one that can see a global shift in RNA content. + * ``_ruv`` -- RUVg as prescribed by the RUVSeq vignette. The factors of unwanted variation ``W_1..W_k`` estimated from the control genes enter the design, and normalization is the engine's own (median of ratios resp. TMM) over the endogenous genes; the spike-in size factors are deliberately *not* applied underneath. Fold changes are again per library, but with the unwanted variation the control genes expose removed. With ``ruvk=0`` this branch reproduces the standard branch exactly. + + ``k`` is chosen automatically (``ruvk=auto``, the default) by a permutation test on the control gene variance spectrum (parallel analysis, Buja & Eyuboglu 1992, the procedure ``sva::num.sv`` uses), capped so that at least one residual degree of freedom is left for dispersion estimation. Set ``ruvk=`` to pin it, or ``ruvk=0`` to switch the ``W`` covariates off. Asking for more factors than the data can estimate is an error, not a silent truncation. Every run writes the per-k diagnostics to ``Tables/DE____table_ruv_k_selection.tsv`` and ``Figures/DE____figure_RUV_k_selection.png``: variance explained per factor with its permutation p-value, the correlation of each factor with the spike-in size factors, its association with the condition under test, and what every additional factor does to the RLE spread, the within-group variance and the global log2 shift between the groups. Because the spike-in size factors and ``W`` are estimated from the same control genes they encode the same quantity whenever the spike-in was added per cell: a ``W`` strongly correlated with the size factors cancels the spike-in scale inside ``_ruv``, which is why the log warns about it and the ``global_log2_shift`` column quantifies it per k. The spike-anchored reading then lives in ``_norm`` and only there. A ``W`` that is associated with the condition under test is absorbing biology rather than nuisance and makes ``_ruv`` conservative; this is reported as well, never acted upon silently. + PostDE (GO/GSEA/GSVA/decoupleR/dream) ##################################### diff --git a/envs/bowtie2.yaml b/envs/bowtie2.yaml new file mode 100644 index 00000000..13e1a357 --- /dev/null +++ b/envs/bowtie2.yaml @@ -0,0 +1,10 @@ +name: bowtie2 +channels: + - conda-forge + - bioconda + - nodefaults +dependencies: + - bowtie2 =2.5.4 + - samtools =1.21 + - pigz =2.6 + - rename =1.601 diff --git a/scripts/Analysis/DE/DESEQ2.R b/scripts/Analysis/DE/DESEQ2.R index d8a694ea..7d992164 100755 --- a/scripts/Analysis/DE/DESEQ2.R +++ b/scripts/Analysis/DE/DESEQ2.R @@ -26,7 +26,10 @@ outdir <- args[4] cmp <- args[5] combi <- args[6] availablecores <- as.integer(args[7]) -spike <- if (argsLen > 7) args[8] else "" +## everything beyond the fixed arguments is the free-form DE option string +## ("controlgenes= ruvk=auto|"), which snakemake passes as one +## argument and nextflow as one argument per whitespace separated token +deopts_raw <- if (argsLen > 7) args[8:argsLen] else character(0) print(args) @@ -42,6 +45,16 @@ if (!file.exists(libp)) { } source(libp) +de_opts <- parse_de_options(deopts_raw) +spike <- de_option(de_opts, "controlgenes") +## number of RUVg factors of unwanted variation: "auto" estimates it from the +## control genes (permutation test on the variance spectrum), an integer pins +## it, 0 switches the W covariates off and normalizes on the spike-in scale only +ruvk <- de_option(de_opts, "ruvk", "auto") +## model formula for the per-contrast fit, over condition/type/batch, e.g. +## design=batch+condition. Empty means MONSDA derives it from the annotation. +design_spec <- de_option(de_opts, "design") + postde_enabled <- Sys.getenv("MONSDA_POSTDE") == "1" if (postde_enabled) { postde_args <- commandArgs(trailingOnly = FALSE) @@ -111,7 +124,7 @@ write.table(add_gene_coordinates(tpm_matrix, rownames(tpm_matrix), gtf_gene), gz # Normalize by spike in if available if (spike != "") { print("Spike-in used, data will be normalized to spike in separately") - spiken <- strsplit(spike, "=")[[1]][2] + spiken <- spike setwd(WD) ctrlgenes <- trimws(readLines(spiken)) ctrlgenes <- ctrlgenes[nzchar(ctrlgenes)] @@ -124,20 +137,31 @@ if (spike != "") { if (length(ctrlgenes) == 0) { stop(paste0("None of the control genes in ", spiken, " are present in the count table, cannot normalize by spike-in")) } - counts_norm_set <- RUVg(newSeqExpressionSet(as.matrix(countData_all)), ctrlgenes, k = 1) - sampleData_norm <- cbind(sampleData_all, pData(counts_norm_set)) - ctrl_idx_all <- rownames(counts(counts_norm_set)) %in% ctrlgenes # for spike-in-derived size factors + ctrl_idx_all <- rownames(countData_all) %in% ctrlgenes # for spike-in-derived size factors # Derive size factors from the spike-ins themselves (controlGenes); normCounts() from RUVg - # only removes unwanted variation (W_1) and does not carry the spike-in scale. - dds_sf <- DESeqDataSetFromMatrix(countData = counts(counts_norm_set), colData = sampleData_norm, design = ~1) + # only removes unwanted variation (W_i) and does not carry the spike-in scale. + dds_sf <- DESeqDataSetFromMatrix(countData = as.matrix(countData_all), colData = sampleData_all, design = ~1) dds_sf <- estimateSizeFactors(dds_sf, controlGenes = ctrl_idx_all) + # how many factors of unwanted variation the control genes actually support + ruv_all <- ruv_estimate_k(countData_all, ctrlgenes, model.matrix(~condition, data = sampleData_all), + condition = sampleData_all$condition, size_factors = sizeFactors(dds_sf), k = ruvk, label = "DataSet") + ruv_write_k_report(ruv_all, paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "ruv_k_selection.tsv", sep = "_")) + ruv_plot_k(ruv_all, paste("Figures/DE", "DESEQ2", combi, "DataSet", "figure", "RUV_k_selection.png", sep = "_")) + if (ruv_all$k > 0) { + ruv_set_all <- RUVg(newSeqExpressionSet(as.matrix(countData_all)), ctrlgenes, k = ruv_all$k) + sampleData_ruv <- cbind(sampleData_all, pData(ruv_set_all)) + } else { + sampleData_ruv <- sampleData_all + } + # counts_norm is the spike-in size factor normalized matrix (the _norm branch); the RUVg + # factors live in sampleData_ruv and enter the model as covariates, not as a count scale counts_norm <- as.data.frame(counts(dds_sf, normalized = TRUE)) countData_clean <- countData_all %>% subset(!row.names(countData_all) %in% ctrlgenes) # removing spike-ins for standard analysis counts_norm_clean <- counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes) # removing spike-ins for standard analysis write.table(add_gene_coordinates(counts_norm, rownames(counts_norm), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "counts_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) write.table(add_gene_coordinates(counts_norm_clean, rownames(counts_norm_clean), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "counts_norm_clean.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) write.table(add_gene_coordinates(countData_clean, rownames(countData_clean), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "counts_clean.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) - write.table(sampleData_norm, gzfile(paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "sampleData_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) + write.table(sampleData_ruv, gzfile(paste("Tables/DE", "DESEQ2", combi, "DataSet", "table", "sampleData_ruv.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) # Calculate CPM and TPM cpm_norm_matrix <- calc_cpm(counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes)) tpm_norm_matrix <- calc_tpm(counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes), gtf_gene) @@ -245,26 +269,74 @@ for (contrast in comparison) { sampleData <- sel$sampleData countData <- sel$countData - ## Create design-table considering different types (paired, unpaired) and batches - if (length(unique(subset(sampleData, A == condition)$type)) > 1 | length(unique(subset(sampleData, B == condition)$type)) > 1) { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - design <- ~ type + batch + condition - } else { - design <- ~ type + condition + ## Design for this contrast: `design=` from the config if set, otherwise + ## derived from the types and batches present in the two groups + design <- build_de_design(design_spec, sampleData, A, B, contrast_name) + print(design) + + ## Fit one spike-in branch (_norm or _ruv) and write its QC outputs. Identical to the + ## plain branch below except for the suffix, the optional externally supplied size + ## factors and the design, so the three readings stay directly comparable. + fit_spike_branch <- function(counts_b, meta_b, design_b, size_factors, suffix) { + d <- DESeqDataSetFromMatrix(countData = counts_b, colData = meta_b, design = design_b) + if (!is.null(size_factors)) { + sizeFactors(d) <- size_factors[colnames(d)] } - } else { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - design <- ~ batch + condition - } else { - design <- ~ condition + d <- d[rowSums(counts(d) >= 10) >= min_group_size(meta_b$condition), ] + d$condition <- droplevels(d$condition) + d$condition <- relevel(d$condition, ref = B[[1]]) + d <- DESeq(d, parallel = TRUE, BPPARAM = BPPARAM, betaPrior = FALSE) + rld_b <- rlogTransformation(d, blind = FALSE) + vsd_b <- varianceStabilizingTransformation(d, blind = FALSE) + png(paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure", paste0("PCA", suffix, ".png"), sep = "_"), width=1900, height=1200, res=75) + print(DESeq2::plotPCA(vsd_b, intgroup = c("condition")) + geom_text_repel(aes(label = name), arrow = arrow(length = unit(0.02, "npc")), box.padding = .5) + theme_bw()) # requires ggrepel) + dev.off() + write.table(add_gene_coordinates(as.data.frame(assay(rld_b)), rownames(assay(rld_b)), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", paste0("rld", suffix, ".tsv.gz"), sep = "_")), sep = "\t", col.names = NA) + write.table(add_gene_coordinates(as.data.frame(assay(vsd_b)), rownames(assay(vsd_b)), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", paste0("vsd", suffix, ".tsv.gz"), sep = "_")), sep = "\t", col.names = NA) + list(dds = d, vsd = vsd_b) + } + + ## Results, tables and figures of one spike-in branch. Byte-identical in structure to + ## the plain branch below, only the suffix and the fitted object differ. + emit_spike_branch <- function(dds_b, meta_b, counts_b, vsd_b, design_b, suffix) { + listname <- paste0(contrast_name, suffix) + res <- results(dds_b, contrast = c("condition", A, B), parallel = TRUE, BPPARAM = BPPARAM) + resn <- res + res_shrink <- lfcShrink(dds = dds_b, coef = paste("condition", A, "vs", B, sep = "_"), res = res, type = "apeglm") + check_shrinkage(res, res_shrink, label = listname) + + if (postde_enabled) { + postde_capture(engine = "deseq2", id = listname, A = A, B = B, normalized = TRUE, metadata = meta_b, counts = counts_b, expression = assay(vsd_b), formula = design_b, results = postde_results_deseq2(res), mean_scale = "baseMean") } + + resOrdered <- format_deseq2_results(res_shrink[order(res_shrink$log2FoldChange), ], gtf_gene, shrink = TRUE, center = TRUE) + pdf(file = paste("Figures/DE", "DESEQ2", combi, contrast_name, paste0("figure_Volcano", suffix, ".pdf"), sep = "_"), width = 15, height = 10) + print(EnhancedVolcano(resOrdered, + lab = resOrdered$Gene, x = "log2FoldChange", y = "padj", + title = paste0(contrast_name, suffix, "_p005_lfc15"), + pCutoff = 0.05, FCcutoff = 1.5, pointSize = 3.0, labSize = 5.0, colAlpha = .3, + legendLabels = c("Not sig.", "Log (base 2) FC", "p-value", "p-value & Log (base 2) FC"), + legendPosition = "right", legendLabSize = 10, legendIconSize = 5.0, + drawConnectors = TRUE, widthConnectors = 0.75 + )) + dev.off() + write.table(as.data.frame(apply(resOrdered, 2, as.character)), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", paste0("results", suffix, ".tsv.gz"), sep = "_")), sep = "\t", row.names = FALSE, quote = F) + + res <- resn[order(resn$log2FoldChange), ] + res$Gene_ID <- rownames(res) + res <- format_deseq2_results(res, gtf_gene, shrink = FALSE, center = TRUE) + write.table(as.data.frame(apply(res, 2, as.character)), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", paste0("results", suffix, "_noshrink.tsv.gz"), sep = "_")), sep = "\t", row.names = FALSE, quote = F) + + png(paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure", paste0("MA", suffix, ".png"), sep = "_"), width=1900, height=1200, res=300) + DESeq2::plotMA(res_shrink) + dev.off() + resn } - print(design) # Normalize by spike in if available if (spike != "") { print("Spike-in used, data will be normalized to spike in separately") - spiken <- strsplit(spike, "=")[[1]][2] + spiken <- spike setwd(WD) ctrlgenes <- trimws(readLines(spiken)) ctrlgenes <- ctrlgenes[nzchar(ctrlgenes)] @@ -278,46 +350,48 @@ for (contrast in comparison) { stop(paste0("None of the control genes in ", spiken, " are present in the count table, cannot normalize by spike-in")) } - counts_norm <- RUVg(newSeqExpressionSet(as.matrix(countData)), ctrlgenes, k = 1) - ctrl_idx <- rownames(counts(counts_norm)) %in% ctrlgenes # for spike-in-derived size factors - counts_norm_mat <- counts(counts_norm)[!ctrl_idx, , drop = FALSE] # removing spike-ins for actual DE testing + ctrl_idx <- rownames(countData) %in% ctrlgenes # spike-ins are never tested themselves + counts_endo <- as.matrix(countData)[!ctrl_idx, , drop = FALSE] + + # Spike-in derived size factors (median of ratios over the control genes): the + # normalization of the _norm branch, and the reference the k diagnostics compare the + # factors of unwanted variation against. + sf_spike <- spikein_size_factors(as.matrix(countData)[ctrl_idx, , drop = FALSE]) + write_scaling_log(sf_spike, colnames(countData), paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "scaling.log", sep = "_")) + + # how many factors of unwanted variation the control genes support for this contrast + ruv_sel <- ruv_estimate_k(countData, ctrlgenes, stats::model.matrix(design, data = sampleData), + condition = sampleData$condition, size_factors = sf_spike, k = ruvk, label = contrast_name, contrast = c(A, B)) + ruv_write_k_report(ruv_sel, paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "ruv_k_selection.tsv", sep = "_")) + ruv_plot_k(ruv_sel, paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure", "RUV_k_selection.png", sep = "_")) + + if (ruv_sel$k > 0) { + ruv_set <- RUVg(newSeqExpressionSet(as.matrix(countData)), ctrlgenes, k = ruv_sel$k) + sampleData_ruv <- cbind(sampleData, pData(ruv_set)) + design_ruv <- ruv_add_w_terms(design, colnames(pData(ruv_set))) # Last argument is variable of interest + } else { + sampleData_ruv <- sampleData + design_ruv <- design + } countData <- countData %>% subset(!row.names(countData) %in% ctrlgenes) # removing spike-ins for standard analysis - sampleData_norm <- cbind(sampleData, pData(counts_norm)) - design_norm <- as.formula(paste(gsub("~", "~ W_1 +", deparse(design)), collapse = "")) # Last argument is variable of interest - #design_norm <- as.formula(paste(deparse(design), " + W_1"), collapse = "") - print(paste0("Design with spike-in normalization: ", paste(all.vars(design_norm), collapse = ", "))) - - # Build with the full (incl. spike-in) counts so that size factors can be derived from the - # spike-ins themselves (controlGenes); this is what puts the spike-in scale into the - # normalized results, the W_1 covariate alone only adjusts for unwanted variation, not scale. - dds_norm <- DESeqDataSetFromMatrix(countData = counts(counts_norm), colData = sampleData_norm, design = design_norm) - dds_norm <- estimateSizeFactors(dds_norm, controlGenes = ctrl_idx) - write_scaling_log(sizeFactors(dds_norm), colnames(dds_norm), paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "scaling.log", sep = "_")) - dds_norm <- dds_norm[!ctrl_idx, ] # drop spike-ins from testing, size factors are retained - - # filter low counts - smallestGroupSize <- min_group_size(sampleData_norm$condition) - keep_norm <- rowSums(counts(dds_norm) >= 10) >= smallestGroupSize - dds_norm <- dds_norm[keep_norm, ] - - # drop unused samples - dds_norm$condition <- droplevels(dds_norm$condition) - - # relevel to base condition B - dds_norm$condition <- relevel(dds_norm$condition, ref = B[[1]]) - - dds_norm <- DESeq(dds_norm, parallel = TRUE, BPPARAM = BPPARAM, betaPrior = FALSE) - rld_norm <- rlogTransformation(dds_norm, blind = FALSE) - vsd_norm <- varianceStabilizingTransformation(dds_norm, blind = FALSE) - - png(paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure", "PCA_norm.png", sep = "_"), width=1900, height=1200, res=75) - print(DESeq2::plotPCA(vsd_norm, intgroup = c("condition")) + geom_text_repel(aes(label = name), arrow = arrow(length = unit(0.02, "npc")), box.padding = .5) + theme_bw()) # requires ggrepel) - # DESeq2::plotPCA(rld_norm, intgroup=c('condition')) + geom_text(aes(label = name), position = position_nudge(y = 2)) - dev.off() - - # We also write the normalized counts to file - write.table(add_gene_coordinates(as.data.frame(assay(rld_norm)), rownames(assay(rld_norm)), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "rld_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA) - write.table(add_gene_coordinates(as.data.frame(assay(vsd_norm)), rownames(assay(vsd_norm)), gtf_gene), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "vsd_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA) + print(paste0("Design of the _ruv branch: ", paste(all.vars(design_ruv), collapse = ", "))) + + # _norm: the spike-ins set the scale. Size factors from the control genes, design + # unchanged, so its fold changes are absolute (per cell) rather than per library. + print("Spike-in branch _norm: size factors from the control genes, no RUV covariates") + branch_norm <- fit_spike_branch(counts_endo, sampleData, design, sf_spike, "_norm") + dds_norm <- branch_norm$dds + vsd_norm <- branch_norm$vsd + + # _ruv: RUVg exactly as the RUVSeq vignette prescribes it. The W_i covariates enter + # the design and normalization is DESeq2's own median of ratios over the endogenous + # genes; the spike-in size factors are deliberately NOT applied on top. Both are + # estimated from the same control genes, so combining them would correct the same + # quantity twice and leave the result depending on which of the two wins. + print("Spike-in branch _ruv: RUVg factors as covariates, standard median-of-ratios normalization") + branch_ruv <- fit_spike_branch(counts_endo, sampleData_ruv, design_ruv, NULL, "_ruv") + dds_ruv <- branch_ruv$dds + vsd_ruv <- branch_ruv$vsd } # Create DESeqDataSet @@ -359,6 +433,7 @@ for (contrast in comparison) { res <- results(dds, contrast = c("condition", A, B), parallel = TRUE, BPPARAM = BPPARAM) resn <- res res_shrink <- lfcShrink(dds = dds, coef = paste("condition", A, "vs", B, sep = "_"), res = res, type = "apeglm") + check_shrinkage(res, res_shrink, label = contrast_name) # add comp object to list for image if (postde_enabled) { @@ -406,9 +481,9 @@ for (contrast in comparison) { # sort and output res <- resn[order(resn$log2FoldChange), ] - res$Gene <- unlist(lapply(rownames(res), function(x) { + res$Gene <- vapply(rownames(res), function(x) { get_gene_name(x, gtf_gene) - })) + }, FUN.VALUE = character(1), USE.NAMES = FALSE) res$Gene_ID <- rownames(res) res <- format_deseq2_results(res, gtf_gene, shrink = FALSE) res <- as.data.frame(apply(res, 2, as.character)) @@ -420,75 +495,11 @@ for (contrast in comparison) { DESeq2::plotMA(res_shrink) dev.off() - if (spike != "") { # DE run for spike-in normalized data - - # initialize empty objects - res <- "" - resOrdered <- "" - res <- results(dds_norm, contrast = c("condition", A, B), parallel = TRUE, BPPARAM = BPPARAM) - resn <- res - res_shrink <- lfcShrink(dds = dds_norm, coef = paste("condition", A, "vs", B, sep = "_"), res = res, type = "apeglm") - - # add comp object to list for image - listname <- paste(contrast_name, "_norm", sep = "") - if (postde_enabled) { - postde_capture(engine = "deseq2", id = listname, A = A, B = B, normalized = TRUE, metadata = sampleData_norm, counts = counts_norm_mat, expression = assay(vsd_norm), formula = design_norm, results = postde_results_deseq2(res), mean_scale = "baseMean") - } - comparison_objs[[listname]] <- res - - # sort and output - resOrdered <- res_shrink[order(res_shrink$log2FoldChange), ] - - # # Add gene names (check how gene_id col is named ) - resOrdered <- format_deseq2_results(resOrdered, gtf_gene, shrink = TRUE, center = TRUE) - - # plot Volcano - pdf( - file = paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure_Volcano_norm.pdf", sep = "_"), width = 15, height = 10 - ) - print(EnhancedVolcano(resOrdered, - lab = resOrdered$Gene, - x = "log2FoldChange", - y = "padj", - title = paste0(contrast_name, "_p005_lfc15", sep = ""), - pCutoff = 0.05, - FCcutoff = 1.5, - pointSize = 3.0, - labSize = 5.0, - colAlpha = .3, - legendLabels = c( - "Not sig.", "Log (base 2) FC", "p-value", - "p-value & Log (base 2) FC" - ), - legendPosition = "right", - legendLabSize = 10, - legendIconSize = 5.0, - drawConnectors = TRUE, - widthConnectors = 0.75 - )) - dev.off() - resOrdered <- as.data.frame(apply(resOrdered, 2, as.character)) - - # write the table to a tsv file - write.table(as.data.frame(resOrdered), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "results_norm.tsv.gz", sep = "_")), sep = "\t", row.names = FALSE, quote = F) - - - # sort and output - res <- resn[order(resn$log2FoldChange), ] - - res$Gene <- unlist(lapply(rownames(res), function(x) { - get_gene_name(x, gtf_gene) - })) - res$Gene_ID <- rownames(res) - res <- format_deseq2_results(res, gtf_gene, shrink = FALSE, center = TRUE) - res <- as.data.frame(apply(res, 2, as.character)) - - write.table(as.data.frame(res), gzfile(paste("Tables/DE", "DESEQ2", combi, contrast_name, "table", "results_norm_noshrink.tsv.gz", sep = "_")), sep = "\t", row.names = FALSE, quote = F) - - # plotMA - png(paste("Figures/DE", "DESEQ2", combi, contrast_name, "figure", "MA_norm.png", sep = "_"), width=1900, height=1200, res=300) - DESeq2::plotMA(res_shrink) - dev.off() + if (spike != "") { # DE runs for the two spike-in branches + # _norm: spike-in size factors, plain design -> absolute (per cell) fold changes + comparison_objs[[paste0(contrast_name, "_norm")]] <- emit_spike_branch(dds_norm, sampleData, counts_endo, vsd_norm, design, "_norm") + # _ruv: RUVg covariates, standard normalization -> unwanted variation removed + comparison_objs[[paste0(contrast_name, "_ruv")]] <- emit_spike_branch(dds_ruv, sampleData_ruv, counts_endo, vsd_ruv, design_ruv, "_ruv") } # cleanup rm(res, resOrdered) diff --git a/scripts/Analysis/DE/EDGER.R b/scripts/Analysis/DE/EDGER.R index 52bc69f9..99fdcb4f 100755 --- a/scripts/Analysis/DE/EDGER.R +++ b/scripts/Analysis/DE/EDGER.R @@ -21,7 +21,10 @@ outdir <- args[4] cmp <- args[5] combi <- args[6] availablecores <- as.integer(args[7]) -spike <- if (argsLen > 7) args[8] else "" +## everything beyond the fixed arguments is the free-form DE option string +## ("controlgenes= ruvk=auto|"), which snakemake passes as one +## argument and nextflow as one argument per whitespace separated token +deopts_raw <- if (argsLen > 7) args[8:argsLen] else character(0) print(args) @@ -37,6 +40,16 @@ if (!file.exists(libp)) { } source(libp) +de_opts <- parse_de_options(deopts_raw) +spike <- de_option(de_opts, "controlgenes") +## number of RUVg factors of unwanted variation: "auto" estimates it from the +## control genes (permutation test on the variance spectrum), an integer pins +## it, 0 switches the W covariates off and normalizes on the spike-in scale only +ruvk <- de_option(de_opts, "ruvk", "auto") +## model formula for the per-contrast fit, over condition/type/batch, e.g. +## design=batch+condition. Empty means MONSDA derives it from the annotation. +design_spec <- de_option(de_opts, "design") + postde_enabled <- Sys.getenv("MONSDA_POSTDE") == "1" if (postde_enabled) { postde_args <- commandArgs(trailingOnly = FALSE) @@ -128,7 +141,7 @@ write.table(add_gene_coordinates(tpm_matrix, rownames(tpm_matrix), gtf_gene), gz ## Normalize by spike in if available if (spike != "") { print("Spike-in used, data will be normalized to spike in separately") - spiken <- strsplit(spike, "=")[[1]][2] + spiken <- spike setwd(WD) ctrlgenes <- trimws(readLines(spiken)) ctrlgenes <- ctrlgenes[nzchar(ctrlgenes)] @@ -141,22 +154,29 @@ if (spike != "") { if (length(ctrlgenes) == 0) { stop(paste0("None of the control genes in ", spiken, " are present in the count table, cannot normalize by spike-in")) } - counts_norm_set <- RUVg(newSeqExpressionSet(as.matrix(countData_all)), ctrlgenes, k = 1) - sampleData_norm <- cbind(sampleData_all, pData(counts_norm_set)) - ctrl_idx_all <- rownames(counts(counts_norm_set)) %in% ctrlgenes # for spike-in-derived normalization factors - # Derive the scale from the spike-ins (TMM on control genes) rather than RUVg's normCounts(), - # which only removes unwanted variation (W_1) and does not carry the spike-in scale. - spike_counts_all <- counts(counts_norm_set)[ctrl_idx_all, , drop = FALSE] - dge_spike_all <- calcNormFactors(DGEList(counts = spike_counts_all), method = "TMM") - eff_lib_all <- dge_spike_all$samples$lib.size * dge_spike_all$samples$norm.factors - size_factor_all <- eff_lib_all / exp(mean(log(eff_lib_all))) - counts_norm <- as.data.frame(sweep(counts(counts_norm_set), 2, size_factor_all, FUN = "/")) + ctrl_idx_all <- rownames(countData_all) %in% ctrlgenes # for spike-in-derived normalization factors + # Derive the scale from the spike-ins themselves (median of ratios over the control + # genes); counts_norm below is that _norm scale. RUVg's factors are covariates, not a + # count scale, and live in sampleData_ruv. + size_factor_all <- spikein_size_factors(as.matrix(countData_all)[ctrl_idx_all, , drop = FALSE]) + # how many factors of unwanted variation the control genes actually support + ruv_all <- ruv_estimate_k(countData_all, ctrlgenes, model.matrix(~condition, data = sampleData_all), + condition = sampleData_all$condition, size_factors = size_factor_all, k = ruvk, label = "DataSet") + ruv_write_k_report(ruv_all, paste("Tables/DE", "EDGER", combi, "DataSet", "table", "ruv_k_selection.tsv", sep = "_")) + ruv_plot_k(ruv_all, paste("Figures/DE", "EDGER", combi, "DataSet", "figure", "RUV_k_selection.png", sep = "_")) + if (ruv_all$k > 0) { + ruv_set_all <- RUVg(newSeqExpressionSet(as.matrix(countData_all)), ctrlgenes, k = ruv_all$k) + sampleData_ruv <- cbind(sampleData_all, pData(ruv_set_all)) + } else { + sampleData_ruv <- sampleData_all + } + counts_norm <- as.data.frame(sweep(as.matrix(countData_all), 2, size_factor_all, FUN = "/")) countData_clean <- countData_all %>% subset(!row.names(countData_all) %in% ctrlgenes) # removing spike-ins for standard analysis counts_norm_clean <- counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes) # removing spike-ins for standard analysis write.table(add_gene_coordinates(counts_norm, rownames(counts_norm), gtf_gene), gzfile(paste("Tables/DE", "EDGER", combi, "DataSet", "table", "counts_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) write.table(add_gene_coordinates(counts_norm_clean, rownames(counts_norm_clean), gtf_gene), gzfile(paste("Tables/DE", "EDGER", combi, "DataSet", "table", "counts_norm_clean.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) write.table(add_gene_coordinates(countData_clean, rownames(countData_clean), gtf_gene), gzfile(paste("Tables/DE", "EDGER", combi, "DataSet", "table", "counts_clean.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) - write.table(sampleData_norm, gzfile(paste("Tables/DE", "EDGER", combi, "DataSet", "table", "sampleData_norm.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) + write.table(sampleData_ruv, gzfile(paste("Tables/DE", "EDGER", combi, "DataSet", "table", "sampleData_ruv.tsv.gz", sep = "_")), sep = "\t", col.names = NA, quote = FALSE) # Calculate CPM and TPM cpm_norm_matrix <- calc_cpm(counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes)) tpm_norm_matrix <- calc_tpm(counts_norm %>% subset(!row.names(counts_norm) %in% ctrlgenes), gtf_gene) @@ -254,34 +274,100 @@ for (contrast in comparison) { bl <- sapply("batch", paste0, levels(sampleData$batch)[1:length(levels(sampleData$batch)) - 1]) tl <- sapply("type", paste0, levels(sampleData$type)[1:length(levels(sampleData$type)) - 1]) - ## Create design-table considering different types (paired, unpaired) and batches - if (length(unique(subset(sampleData, A == condition)$type)) > 1 | length(unique(subset(sampleData, B == condition)$type)) > 1) { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - des <- ~ type + batch + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(sampleData$condition), tl, bl) - } else { - des <- ~ type + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(condition), tl) + ## Design for this contrast: `design=` from the config if set, otherwise + ## derived from the types and batches present in the two groups + des <- build_de_design(design_spec, sampleData, A, B, contrast_name) + design <- model.matrix(des, data = sampleData) + print(design) + + ## Results, tables and figures of one spike-in branch. Byte-identical in structure to + ## the plain branch below, only the suffix and the fitted object differ. + emit_spike_branch <- function(branch, meta_b, counts_b, des_b, suffix) { + contrast_col <- paste0("condition", A) + if (!contrast_col %in% colnames(branch$design)) { + stop(paste0("Spike-in design ", suffix, " has no column '", contrast_col, "' (columns: ", paste(colnames(branch$design), collapse = ", "), ")")) } - } else { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - des <- ~ batch + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(sampleData$condition), bl) + AvsB <- setNames(numeric(ncol(branch$design)), colnames(branch$design)) + AvsB[contrast_col] <- 1 + qlf <- glmQLFTest(branch$fit, contrast = AvsB) ## glm quasi-likelihood-F-Test + + if (postde_enabled) { + postde_capture(engine = "edger", id = paste0(contrast_name, suffix), A = A, B = B, normalized = TRUE, metadata = meta_b, counts = counts_b, expression = cpm(branch$dge, log = TRUE), formula = des_b, results = postde_results_edger(qlf), mean_scale = "logCPM") + } + + res <- format_edger_results(qlf$table, gtf_gene, center = TRUE) + pdf(file = paste("Figures/DE", "EDGER", combi, contrast_name, paste0("figure_Volcano", suffix, ".pdf"), sep = "_"), width = 15, height = 10) + print(EnhancedVolcano(res, + lab = res$Gene, x = "log2FoldChange", y = "padj", + title = paste0(contrast_name, suffix, "_p005_lfc15"), + pCutoff = 0.05, FCcutoff = 1.5, pointSize = 3.0, labSize = 5.0, colAlpha = .3, + legendLabels = c("Not sig.", "Log (base 2) FC", "p-value", "p-value & Log (base 2) FC"), + legendPosition = "right", legendLabSize = 10, legendIconSize = 5.0, + drawConnectors = TRUE, widthConnectors = 0.75 + )) + dev.off() + write.table(as.data.frame(apply(res, 2, as.character)), gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", paste0("results", suffix, ".tsv.gz"), sep = "_")), sep = "\t", quote = F, row.names = FALSE) + + for (sorting in c("logFC", "PValue")) { + tops <- format_edger_results(topTags(qlf, n = nrow(qlf$table), sort.by = sorting)$table, gtf_gene, center = TRUE) + label <- if (sorting == "logFC") "resultsLogFCsorted" else "resultsPValueSorted" + write.table(as.data.frame(apply(tops, 2, as.character)), gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", paste0(label, suffix, ".tsv.gz"), sep = "_")), sep = "\t", quote = F, row.names = FALSE) + } + + png(paste("Figures/DE", "EDGER", combi, contrast_name, "figure", paste0("MD", suffix, ".png"), sep = "_"), width=1900, height=1200, res=300) + print(plotMD(qlf, main = paste0(contrast_name, suffix))) + abline(h = c(-1, 1), col = "blue") + dev.off() + qlf + } + + ## Fit one spike-in branch (_norm or _ruv) and write its QC outputs. Identical to the + ## plain branch below except for the suffix, the normalization and the design, so the + ## three readings stay directly comparable. + fit_spike_branch <- function(counts_b, meta_b, des_b, size_factors, suffix) { + design_b <- model.matrix(des_b, data = meta_b) + print(paste0("Design of the ", suffix, " branch: ", paste(colnames(design_b), collapse = ", "))) + d <- DGEList(counts = counts_b, group = meta_b$condition, samples = samples, genes = rownames(counts_b)) + d <- d[filterByExpr(d), , keep.lib.sizes = FALSE] + d$samples$group <- relevel(d$samples$group, ref = B[[1]]) + if (is.null(size_factors)) { + d <- calcNormFactors(d, method = "TMM", BPPARAM = BPPARAM) } else { - des <- ~ condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- levels(sampleData$condition) + d <- set_spikein_offsets(d, size_factors[colnames(d)]) + } + + tmm_b <- as.data.frame(cpm(d)) + colnames(tmm_b) <- t(d$samples$samples) + tmm_b$ID <- d$genes$genes + tmm_b <- tmm_b[c(ncol(tmm_b), 1:ncol(tmm_b) - 1)] + tmm_b <- add_gene_coordinates(tmm_b, tmm_b$ID, gtf_gene, after = "ID") + write.table(as.data.frame(tmm_b), gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", paste0("Normalized", suffix, ".tsv.gz"), sep = "_")), sep = "\t", quote = F, row.names = FALSE) + + png(paste("Figures/DE", "EDGER", combi, contrast_name, "figure", paste0("MDS", suffix, ".png"), sep = "_"), width=1900, height=1200, res=300) + print(safe_plotMDS(d, col = as.numeric(d$samples$group), cex = 1)) + dev.off() + + if (no_residual_df(design_b)) { + stop(paste0("Spike-in design ", suffix, " for contrast ", contrast_name, " has no residual degrees of freedom (", nrow(design_b), " samples, design rank ", qr(design_b)$rank, "). The nuisance-adjusted model cannot be fit; refusing to fall back to an unadjusted test that ignores the covariates.")) } + d <- estimateDisp(d, design_b, robust = TRUE) + + png(paste("Figures/DE", "EDGER", combi, contrast_name, "figure", paste0("BCV", suffix, ".png"), sep = "_"), width=1900, height=1200, res=300) + print(plotBCV(d)) + dev.off() + + f <- glmQLFit(d, design_b, robust = TRUE) + + png(paste("Figures/DE", "EDGER", combi, contrast_name, "figure", paste0("QLDisp", suffix, ".png"), sep = "_"), width=1900, height=1200, res=300) + print(plotQLDisp(f)) + dev.off() + list(dge = d, fit = f, design = design_b) } - print(design) ## check genes and spike-ins if (spike != "") { print("Spike-in used, data will be normalized to spike in separately") - spiken <- strsplit(spike, "=")[[1]][2] + spiken <- spike setwd(WD) ctrlgenes <- trimws(readLines(spiken)) ctrlgenes <- ctrlgenes[nzchar(ctrlgenes)] @@ -294,72 +380,49 @@ for (contrast in comparison) { if (length(ctrlgenes) == 0) { stop(paste0("None of the control genes in ", spiken, " are present in the count table, cannot normalize by spike-in")) } - counts_norm <- RUVg(newSeqExpressionSet(as.matrix(countData)), ctrlgenes, k = 1) - ctrl_idx <- rownames(counts(counts_norm)) %in% ctrlgenes # for spike-in-derived normalization factors - counts_norm_mat <- counts(counts_norm)[!ctrl_idx, , drop = FALSE] # removing spike-ins for actual DE testing - genes <- rownames(counts_norm_mat) - countData <- countData %>% subset(!row.names(countData) %in% ctrlgenes) # removing spike-ins for standard analysis - sampleData_norm <- cbind(sampleData, pData(counts_norm)) - des_norm <- as.formula(paste(gsub("~", "~ W_1 +", deparse(des)), collapse = "")) # Last argument is variable of interest - # des_norm <- as.formula(paste(deparse(des), " + W_1"), collapse = "") - design_norm <- model.matrix(des_norm, data = sampleData_norm) - print(paste0("Design with spike-in normalization: ", paste(colnames(design_norm), collapse = ", "))) - - dge_norm <- DGEList(counts = counts_norm_mat, group = sampleData$condition, samples = samples, genes = genes) - ## filter low counts; keep original (pre-filter) lib sizes since the spike-in-derived - ## normalization factors below are only valid relative to them - keep <- filterByExpr(dge_norm) - dge_norm <- dge_norm[keep, , keep.lib.sizes = TRUE] - - # relevel to base condition B - dge_norm$samples$group <- relevel(dge_norm$samples$group, ref = B[[1]]) - - ## normalize using the spike-in (control gene) counts rather than TMM on the endogenous - ## genes: this is what actually puts the spike-in scale into the normalized results, the - ## W_1 covariate alone only adjusts for unwanted variation, not scale. - spike_counts <- counts(counts_norm)[ctrl_idx, , drop = FALSE] - dge_spike <- calcNormFactors(DGEList(counts = spike_counts), method = "TMM") - dge_norm$samples$norm.factors <- dge_spike$samples$norm.factors - - eff_lib_norm <- dge_norm$samples$lib.size * dge_norm$samples$norm.factors - scaling_factors <- eff_lib_norm / exp(mean(log(eff_lib_norm))) - write_scaling_log(scaling_factors, dge_norm$samples$samples, paste("Tables/DE", "EDGER", combi, contrast_name, "table", "scaling.log", sep = "_")) - - ## create file normalized table - tmm_norm <- as.data.frame(cpm(dge_norm)) - colnames(tmm_norm) <- t(dge_norm$samples$samples) - tmm_norm$ID <- dge_norm$genes$genes - tmm_norm <- tmm_norm[c(ncol(tmm_norm), 1:ncol(tmm_norm) - 1)] - - tmm_norm <- add_gene_coordinates(tmm_norm, tmm_norm$ID, gtf_gene, after = "ID") - write.table(as.data.frame(tmm_norm), gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", "Normalized_norm.tsv.gz", sep = "_")), sep = "\t", quote = F, row.names = FALSE) - - ## create file MDS-plot with and without summarized replicates - out <- paste("Figures/DE", "EDGER", combi, contrast_name, "figure", "MDS_norm.png", sep = "_") - png(out, width=1900, height=1200, res=300) - print(safe_plotMDS(dge_norm, col = as.numeric(dge_norm$samples$group), cex = 1)) - dev.off() - - ## estimate Dispersion - if (no_residual_df(design_norm)) { - stop(paste0("Spike-in normalized design for contrast ", contrast_name, " has no residual degrees of freedom (", nrow(design_norm), " samples, design rank ", qr(design_norm)$rank, "). The nuisance-adjusted model cannot be fit; refusing to fall back to an unadjusted test that ignores the covariates.")) + ctrl_idx <- rownames(countData) %in% ctrlgenes # spike-ins are never tested themselves + counts_endo <- as.matrix(countData)[!ctrl_idx, , drop = FALSE] + + ## Spike-in derived scale (median of ratios over the control genes): the normalization + ## of the _norm branch, and the reference the k diagnostics compare the factors of + ## unwanted variation against. + size_factor_spike <- spikein_size_factors(as.matrix(countData)[ctrl_idx, , drop = FALSE]) + write_scaling_log(size_factor_spike, colnames(countData), paste("Tables/DE", "EDGER", combi, contrast_name, "table", "scaling.log", sep = "_")) + + ## how many factors of unwanted variation the control genes support for this contrast + ruv_sel <- ruv_estimate_k(countData, ctrlgenes, design, + condition = sampleData$condition, size_factors = size_factor_spike, k = ruvk, label = contrast_name, contrast = c(A, B)) + ruv_write_k_report(ruv_sel, paste("Tables/DE", "EDGER", combi, contrast_name, "table", "ruv_k_selection.tsv", sep = "_")) + ruv_plot_k(ruv_sel, paste("Figures/DE", "EDGER", combi, contrast_name, "figure", "RUV_k_selection.png", sep = "_")) + + if (ruv_sel$k > 0) { + ruv_set <- RUVg(newSeqExpressionSet(as.matrix(countData)), ctrlgenes, k = ruv_sel$k) + sampleData_ruv <- cbind(sampleData, pData(ruv_set)) + des_ruv <- ruv_add_w_terms(des, colnames(pData(ruv_set))) # Last argument is variable of interest + } else { + sampleData_ruv <- sampleData + des_ruv <- des } - dge_norm <- estimateDisp(dge_norm, design_norm, robust = TRUE) - - ## create file BCV-plot - visualizing estimated dispersions - out <- paste("Figures/DE", "EDGER", combi, contrast_name, "figure", "BCV_norm.png", sep = "_") - png(out, width=1900, height=1200, res=300) - print(plotBCV(dge_norm)) - dev.off() - - ## fitting a quasi-likelihood negative binomial generalized log-linear model to counts - fit_norm <- glmQLFit(dge_norm, design_norm, robust = TRUE) + countData <- countData %>% subset(!row.names(countData) %in% ctrlgenes) # removing spike-ins for standard analysis - ## create file quasi-likelihood-dispersion-plot - out <- paste("Figures/DE", "EDGER", combi, contrast_name, "figure", "QLDisp_norm.png", sep = "_") - png(out, width=1900, height=1200, res=300) - print(plotQLDisp(fit_norm)) - dev.off() + ## _norm: the spike-ins set the scale. Offsets from the control genes, design + ## unchanged, so its fold changes are absolute (per cell) rather than per library. + print("Spike-in branch _norm: offsets from the control genes, no RUV covariates") + branch_norm <- fit_spike_branch(counts_endo, sampleData, des, size_factor_spike, "_norm") + dge_norm <- branch_norm$dge + fit_norm <- branch_norm$fit + design_norm <- branch_norm$design + + ## _ruv: RUVg exactly as the RUVSeq vignette prescribes it. The W_i covariates enter + ## the design and normalization is edgeR's own TMM over the endogenous genes; the + ## spike-in offsets are deliberately NOT applied on top. Both are estimated from the + ## same control genes, so combining them would correct the same quantity twice and + ## leave the result depending on which of the two wins. + print("Spike-in branch _ruv: RUVg factors as covariates, standard TMM normalization") + branch_ruv <- fit_spike_branch(counts_endo, sampleData_ruv, des_ruv, NULL, "_ruv") + dge_ruv <- branch_ruv$dge + fit_ruv <- branch_ruv$fit + design_ruv <- branch_ruv$design } # Same without spike-in normalization @@ -507,89 +570,11 @@ for (contrast in comparison) { abline(h = c(-1, 1), col = "blue") dev.off() - if (spike != "") { # Same for spike-in normalized - rm(qlf, tops, res) - - # determine contrast, only for complex cases, not needed for our pairwise comparisons now - # A <- strsplit(contrast_groups[[1]][1], "\\+") - # B <- strsplit(contrast_groups[[1]][2], "\\+") - # minus <- 1/length(A[[1]])*(-1) - # plus <- 1/length(B[[1]]) - # contrast <- cbind(integer(dim(design)[2]), colnames(design)) - # for(i in A[[1]]){ - # contrast[which(contrast[,2]==i)]<- minus - # } - # for(i in B[[1]]){ - # contrast[which(contrast[,2]==i)]<- plus - # } - # contrast <- as.numeric(contrast[,1]) - - ## Testing - # qlf <- glmQLFTest(fit, contrast=contrast) ## glm quasi-likelihood-F-Test - contrast_col <- paste0("condition", A) - if (!contrast_col %in% colnames(design_norm)) { - stop(paste0("Spike-in design has no column '", contrast_col, "' (columns: ", paste(colnames(design_norm), collapse = ", "), ")")) - } - AvsB <- setNames(numeric(ncol(design_norm)), colnames(design_norm)) - AvsB[contrast_col] <- 1 - qlf <- glmQLFTest(fit_norm, contrast = AvsB) ## glm quasi-likelihood-F-Test - # add comp object to list for image - if (postde_enabled) { - postde_capture(engine = "edger", id = paste0(contrast_name, "_norm"), A = A, B = B, normalized = TRUE, metadata = sampleData_norm, counts = counts_norm_mat, expression = cpm(dge_norm, log = TRUE), formula = des_norm, results = postde_results_edger(qlf), mean_scale = "logCPM") - } - comparison_objs[[paste0(contrast_name, "_norm")]] <- qlf - - # # Add gene names (check how gene_id col is named ) - res <- format_edger_results(qlf$table, gtf_gene, center = TRUE) - - # plotVolcano - pdf( - file = paste("Figures/DE", "EDGER", combi, contrast_name, "figure_Volcano_norm.pdf", sep = "_"), width = 15, height = 10 - ) - print(EnhancedVolcano(res, - lab = res$Gene, - x = "log2FoldChange", - y = "padj", - title = paste0(contrast_name, "_p005_lfc15", sep = ""), - pCutoff = 0.05, - FCcutoff = 1.5, - pointSize = 3.0, - labSize = 5.0, - colAlpha = .3, - legendLabels = c( - "Not sig.", "Log (base 2) FC", "p-value", - "p-value & Log (base 2) FC" - ), - legendPosition = "right", - legendLabSize = 10, - legendIconSize = 5.0, - drawConnectors = TRUE, - widthConnectors = 0.75 - )) - dev.off() - - res <- as.data.frame(apply(res, 2, as.character)) - - # create results table - write.table(as.data.frame(res), gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", "results_norm.tsv.gz", sep = "_")), sep = "\t", quote = F, row.names = FALSE) - - # create sorted results Tables - tops <- topTags(qlf, n = nrow(qlf$table), sort.by = "logFC") - tops <- format_edger_results(tops$table, gtf_gene, center = TRUE) - tops <- as.data.frame(apply(tops, 2, as.character)) - write.table(tops, gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", "resultsLogFCsorted_norm.tsv.gz", sep = "_")), sep = "\t", quote = F, row.names = FALSE) - - tops <- topTags(qlf, n = nrow(qlf$table), sort.by = "PValue") - tops <- format_edger_results(tops$table, gtf_gene, center = TRUE) - tops <- as.data.frame(apply(tops, 2, as.character)) - write.table(tops, gzfile(paste("Tables/DE", "EDGER", combi, contrast_name, "table", "resultsPValueSorted_norm.tsv.gz", sep = "_")), sep = "\t", quote = F, row.names = FALSE) - - ## plot lFC vs CPM - out <- paste("Figures/DE", "EDGER", combi, contrast_name, "figure", "MD_norm.png", sep = "_") - png(out, width=1900, height=1200, res=300) - print(plotMD(qlf, main = contrast_name)) - abline(h = c(-1, 1), col = "blue") - dev.off() + if (spike != "") { # DE runs for the two spike-in branches + # _norm: spike-in offsets, plain design -> absolute (per cell) fold changes + comparison_objs[[paste0(contrast_name, "_norm")]] <- emit_spike_branch(branch_norm, sampleData, counts_endo, des, "_norm") + # _ruv: RUVg covariates, TMM normalization -> unwanted variation removed + comparison_objs[[paste0(contrast_name, "_ruv")]] <- emit_spike_branch(branch_ruv, sampleData_ruv, counts_endo, des_ruv, "_ruv") } # cleanup diff --git a/scripts/Analysis/DE/EDGER_no_rep.R b/scripts/Analysis/DE/EDGER_no_rep.R index 11ae5519..fdf2e64a 100755 --- a/scripts/Analysis/DE/EDGER_no_rep.R +++ b/scripts/Analysis/DE/EDGER_no_rep.R @@ -21,19 +21,54 @@ outdir <- args[4] cmp <- args[5] combi <- args[6] availablecores <- as.integer(args[7]) -bcv <- if (argsLen > 7) as.numeric(args[8]) else 0.2 -spike <- if (argsLen > 8) args[9] else "" -if (!exists(quote(bcv))){ - bcv <- 0.2 +deopts_raw <- if (argsLen > 7) args[8:argsLen] else character(0) +## Legacy call form passed the BCV as a bare positional number before the +## options, accept it so existing invocations keep working. +if (length(deopts_raw) > 0 && !grepl("^[A-Za-z_][A-Za-z0-9_.]*=", deopts_raw[1])) { + deopts_raw <- c(paste0("bcv=", deopts_raw[1]), deopts_raw[-1]) } print(args) -print(paste0("Typical values for the common BCV (square-root-dispersion) for datasets arising from well-controlled experiments are 0.4 for human data, 0.1 for data on genetically identical model organisms or 0.01 for technical replicates. You selected ", bcv, sep = "")) ## FUNCS libp <- paste0(gsub("/bin/conda", "/envs/monsda", Sys.getenv("CONDA_EXE")), "/share/MONSDA/scripts/lib/_lib.R") source(libp) +de_opts <- parse_de_options(deopts_raw) +bcv_raw <- de_option(de_opts, "bcv", "0.2") +bcv <- suppressWarnings(as.numeric(bcv_raw)) +if (is.na(bcv) || bcv <= 0) { + stop(paste0("DE option bcv must be a positive number, got '", bcv_raw, "'")) +} +spike <- de_option(de_opts, "controlgenes") +## model formula for the per-contrast fit, over condition/type/batch, e.g. +## design=batch+condition. Empty means MONSDA derives it from the annotation. +design_spec <- de_option(de_opts, "design") + +print(paste0("Typical values for the common BCV (square-root-dispersion) for datasets arising from well-controlled experiments are 0.4 for human data, 0.1 for data on genetically identical model organisms or 0.01 for technical replicates. You selected ", bcv, sep = "")) + +## With a fixed BCV edgeR cannot run glmQLFTest, but it can still fit a GLM, and that +## is the only way to honour covariates: exactTest compares two groups and ignores the +## design completely. Use the GLM whenever the design carries more than the condition, +## and keep the exact test when it does not, or when the design is not fittable for +## this contrast, which is the usual case with a single sample per group. +fixed_bcv_test <- function(dge_obj, design_mat, A, B, bcv, label) { + coef_name <- paste0("condition", A) + extra <- setdiff(colnames(design_mat), c("(Intercept)", coef_name)) + if (length(extra) == 0) { + return(exactTest(dge_obj, pair = c(B, A), dispersion = bcv^2, prior.count = 2)) + } + if (!coef_name %in% colnames(design_mat)) { + stop(paste0("Design for ", label, " has no column '", coef_name, "' (columns: ", paste(colnames(design_mat), collapse = ", "), ")")) + } + if (ncol(design_mat) > nrow(design_mat) || qr(design_mat)$rank < ncol(design_mat)) { + warning(paste0("Design for ", label, " is not fittable with ", nrow(design_mat), " samples and ", ncol(design_mat), " coefficients, falling back to the exact test, which ignores ", paste(extra, collapse = ", ")), call. = FALSE) + return(exactTest(dge_obj, pair = c(B, A), dispersion = bcv^2, prior.count = 2)) + } + print(paste0("Testing ", label, " with a fixed-BCV GLM (glmFit/glmLRT) on coefficient ", coef_name, ", covariates: ", paste(extra, collapse = ", "))) + glmLRT(glmFit(dge_obj, design_mat, dispersion = bcv^2), coef = coef_name) +} + ### SCRIPT print(paste("Run EdgeR DE with ", availablecores, " cores", sep = "")) @@ -138,34 +173,18 @@ for (contrast in comparison) { bl <- sapply("batch", paste0, levels(sampleData$batch)[1:length(levels(sampleData$batch)) - 1]) tl <- sapply("type", paste0, levels(sampleData$type)[1:length(levels(sampleData$type)) - 1]) - ## Create design-table considering different types (paired, unpaired) and batches - if (length(unique(subset(sampleData, A == condition)$type)) > 1 | length(unique(subset(sampleData, B == condition)$type)) > 1) { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - des <- ~ type + batch + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(sampleData$condition), tl, bl) - } else { - des <- ~ type + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(condition), tl) - } - } else { - if (length(unique(subset(sampleData, A == condition)$batch)) > 1 | length(unique(subset(sampleData, B == condition)$batch)) > 1) { - des <- ~ batch + condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- c(levels(sampleData$condition), bl) - } else { - des <- ~condition - design <- model.matrix(des, data = sampleData) - # colnames(design) <- levels(sampleData$condition) - } - } + ## Design for this contrast: `design=` from the config if set, otherwise derived + ## from the types and batches present in the two groups. The dispersion is fixed + ## here rather than estimated, so a term only has to be fittable and needs no + ## residual degree of freedom of its own. + des <- build_de_design(design_spec, sampleData, A, B, contrast_name, min_residual_df = 0) + design <- model.matrix(des, data = sampleData) print(design) ## check genes and spike-ins if (spike != "") { print("Spike-in used, data will be normalized to spike in separately") - spiken <- strsplit(spike, "=")[[1]][2] + spiken <- spike setwd(WD) ctrlgenes <- trimws(readLines(spiken)) ctrlgenes <- ctrlgenes[nzchar(ctrlgenes)] @@ -185,7 +204,7 @@ for (contrast in comparison) { countData <- countData %>% subset(!row.names(countData) %in% ctrlgenes) # removing spike-ins for standard analysis sampleData_norm <- cbind(sampleData, pData(counts_norm)) - design_norm <- model.matrix(as.formula(paste(deparse(des), colnames(pData(counts_norm))[1], sep = " + ")), data = sampleData_norm) + design_norm <- model.matrix(ruv_add_w_terms(des, colnames(pData(counts_norm))), data = sampleData_norm) # colnames(design_norm) <- c(colnames(design),"W_1") dge_norm <- DGEList(counts = counts_norm_mat, group = sampleData$condition, samples = samples, genes = genes) @@ -294,7 +313,7 @@ for (contrast in comparison) { # dge <- estimateDisp(dge, design, robust = TRUE) # AS WE HAVE TO SET BCV MANUALLY WITHOUT REPLICATES => no glmQLFTest possible # qlf <- glmQLFTest(fit, contrast = AvsB) ## glm quasi-likelihood-F-Test - qlf <- exactTest(dge, pair = c(B, A), dispersion = bcv^2, prior.count = 2) + qlf <- fixed_bcv_test(dge, design, A, B, bcv, contrast_name) # add comp object to list for image comparison_objs[[contrast_name]] <- qlf @@ -373,7 +392,7 @@ for (contrast in comparison) { #AvsB <- makeContrasts(TreatvsUntreat = paste("condition", A, sep = ""), levels = design) # THIS IS SKIPPED AS WE HAVE TO SET BCV MANUALLY WITHOUT REPLICATES, no glmQLFTest possible #qlf <- glmQLFTest(fit, contrast = AvsB) ## glm quasi-likelihood-F-Test - qlf <- exactTest(dge_norm, pair = c(B, A), dispersion = bcv^2, prior.count = 2) + qlf <- fixed_bcv_test(dge_norm, design_norm, A, B, bcv, paste0(contrast_name, " spike-in branch")) # add comp object to list for image comparison_objs[[paste0(contrast_name, "_norm")]] <- qlf diff --git a/scripts/lib/_lib.R b/scripts/lib/_lib.R index eae96f09..64a5f3b3 100644 --- a/scripts/lib/_lib.R +++ b/scripts/lib/_lib.R @@ -9,12 +9,15 @@ get_gene_name <- function(id, df) { df$gene_name <- df$gene } name_list <- unique(df$gene_name[df["type"] == "gene" & df["gene_id"] == id]) + name_list <- name_list[!is.na(name_list) & nzchar(name_list)] + ## Must ALWAYS return exactly one string: callers assign the result into a + ## column, so a zero-length return silently shortens the vector (unlist drops + ## it) and a multi-element return lengthens it. Both crash the assignment. if (length(name_list) == 0) { - message(paste("WARNING: gene id not found in annotation, using id as name: ", id)) return(id) } if (length(name_list) == 1) { - return(name_list) + return(name_list[1]) } message(paste("WARNING: ambigous gene id: ", id)) return(paste(name_list, collapse = "|")) @@ -218,9 +221,9 @@ write_scaling_log <- function(scaling_factors, samples, path) { ## the deviation from the contrast's global shift. It is appended rather than ## inserted because the significance filters address columns by position. format_deseq2_results <- function(res, gtf_gene, shrink = TRUE, center = FALSE) { - res$Gene <- unlist(lapply(rownames(res), function(x) { + res$Gene <- vapply(rownames(res), function(x) { get_gene_name(x, gtf_gene) - })) + }, FUN.VALUE = character(1), USE.NAMES = FALSE) res$Gene_ID <- rownames(res) if (shrink) { res <- res[, c("Gene_ID", "Gene", "baseMean", "log2FoldChange", "lfcSE", "pvalue", "padj")] @@ -253,9 +256,9 @@ format_edger_results <- function(tbl, gtf_gene, center = FALSE) { gene_ids <- rownames(tbl) out <- data.frame( Gene_ID = gene_ids, - Gene = unlist(lapply(gene_ids, function(x) { + Gene = vapply(gene_ids, function(x) { get_gene_name(x, gtf_gene) - })), + }, FUN.VALUE = character(1), USE.NAMES = FALSE), logCPM = tbl$logCPM, log2FoldChange = tbl$logFC, pvalue = tbl$PValue, @@ -361,3 +364,646 @@ plot_sample_correlation_heatmap <- function(expr_mat, outfile, group = NULL, tit graphics::mtext("Pearson r", side = 3, line = 0.5, cex = 0.8) invisible(cmat) } + + +## --------------------------------------------------------------------------- +## Spike-in / RUV normalization: option parsing, selection of the number of +## factors k and diagnostics. Nothing here changes a result silently: every +## choice is written to the k-report table and announced in the log. +## --------------------------------------------------------------------------- + +## Parse the free-form DE option string into a named list with lowercase keys. +## Snakemake hands the whole string over as a single argument while nextflow +## splits it on whitespace, so all remaining argv elements are folded back into +## one string before parsing. Known keys: controlgenes=, ruvk=auto|. +parse_de_options <- function(args_tail) { + opts <- list() + tokens <- unlist(strsplit(trimws(paste(args_tail, collapse = " ")), "[[:space:]]+")) + tokens <- tokens[nzchar(tokens)] + key <- NULL + for (tok in tokens) { + if (grepl("^[A-Za-z_][A-Za-z0-9_.]*=", tok)) { + key <- tolower(sub("=.*$", "", tok)) + opts[[key]] <- sub("^[^=]*=", "", tok) + } else if (!is.null(key)) { + ## continuation of the previous value: Nextflow splits the unquoted + ## option string on whitespace, so 'design=~ batch + condition' + ## arrives as four tokens + opts[[key]] <- trimws(paste(opts[[key]], tok)) + } else { + message(paste0("WARNING: ignoring DE option '", tok, "', expected key=value")) + } + } + opts +} + +## Value of a DE option, or `default` if it was not given or is empty. +de_option <- function(opts, key, default = "") { + value <- opts[[tolower(key)]] + if (is.null(value) || !nzchar(value)) default else value +} + +## Prepend the RUV factors W_1..W_k to the right hand side of a design formula, +## keeping the variable of interest last and preserving a suppressed intercept. +ruv_add_w_terms <- function(f, w_names) { + if (length(w_names) == 0) { + return(f) + } + tt <- stats::terms(f) + rhs <- paste(c(if (attr(tt, "intercept") == 0) "0", w_names, attr(tt, "term.labels")), collapse = " + ") + stats::as.formula(paste("~", rhs), env = environment(f)) +} + +## Variables a DE design may reference. The MONSDA annotation table carries +## exactly three columns: GROUPS -> condition, TYPES -> type, BATCHES -> batch. +DE_DESIGN_VARS <- c("condition", "type", "batch") + +## The design MONSDA derives when the config does not set one: a nuisance term +## is taken along only if it varies inside at least one of the two groups under +## test, which is what the DE scripts did before `design=` existed. +auto_de_design <- function(meta, A, B) { + varies <- function(v) { + length(unique(meta[[v]][meta$condition == A])) > 1 || + length(unique(meta[[v]][meta$condition == B])) > 1 + } + rhs <- c(if (varies("type")) "type", if (varies("batch")) "batch", "condition") + stats::as.formula(paste("~", paste(rhs, collapse = " + "))) +} + +## Turn the `design=` option into a formula and reject up front what the +## contrast machinery downstream cannot express. +parse_de_design <- function(spec, available = DE_DESIGN_VARS) { + txt <- trimws(spec) + if (!startsWith(txt, "~")) { + txt <- paste("~", txt) + } + f <- tryCatch(stats::as.formula(txt), + error = function(e) stop(paste0("DE option design='", spec, "' is not a valid R formula: ", conditionMessage(e))) + ) + if (length(f) != 2) { + stop(paste0("DE option design='", spec, "' must be one-sided, the response is always the count matrix")) + } + tt <- stats::terms(f) + if (attr(tt, "intercept") == 0) { + stop(paste0("DE option design='", spec, "' suppresses the intercept. The pairwise contrast is read off the treatment-contrast coefficient (DESeq2 condition__vs_, edgeR condition), which does not exist without an intercept, so remove the '0 +' / '- 1'.")) + } + unknown <- setdiff(all.vars(f), available) + if (length(unknown) > 0) { + stop(paste0("DE option design='", spec, "' references unknown variable(s) ", paste(unknown, collapse = ", "), ". Available are ", paste(available, collapse = ", "), ", filled from GROUPS, TYPES and BATCHES in the config.")) + } + if (!"condition" %in% attr(tt, "term.labels")) { + stop(paste0("DE option design='", spec, "' has no 'condition' main effect, which is the term the contrast tests")) + } + inter <- grep("(^|:)condition(:|$)", grep(":", attr(tt, "term.labels"), value = TRUE), value = TRUE) + if (length(inter) > 0) { + warning(paste0("DE option design='", spec, "' contains the interaction(s) ", paste(inter, collapse = ", "), ". Both engines then report the condition effect at the reference level of the interacting variable, not an effect averaged over it. Keep it only if that simple effect is what you want to test."), call. = FALSE) + } + f +} + +## Reduce a design to the terms that are actually estimable for this contrast. +## `condition` is never dropped. A nuisance term goes when it has no contrast +## left in the subset, when it is confounded with the rest of the design, or +## when it would leave fewer than `min_residual_df` residual degrees of freedom +## (set it to 0 when the dispersion is fixed rather than estimated). Dropping rather +## than aborting keeps the remaining contrasts of a run alive, but it changes +## the model, so every drop is warned about individually. +resolve_de_design <- function(f, meta, label = "", min_residual_df = 1) { + tag <- if (nzchar(label)) paste0(" for contrast ", label) else "" + labs <- attr(stats::terms(f), "term.labels") + kept <- character(0) + for (tm in setdiff(labs, "condition")) { + reason <- NULL + tm_vars <- all.vars(stats::as.formula(paste("~", tm))) + constant <- tm_vars[vapply(tm_vars, function(v) length(unique(meta[[v]])) < 2, logical(1))] + if (length(constant) > 0) { + reason <- paste0("'", paste(constant, collapse = "', '"), "' has only one level in this comparison") + } else { + mm <- stats::model.matrix(stats::as.formula(paste("~", paste(c(kept, tm, "condition"), collapse = " + "))), data = meta) + rank <- qr(mm)$rank + if (rank < ncol(mm)) { + reason <- "it is confounded with the rest of the design (rank deficient)" + } else if (nrow(mm) - rank < min_residual_df) { + reason <- paste0("it would leave fewer than ", min_residual_df, " residual degree(s) of freedom for dispersion estimation") + } + } + if (is.null(reason)) { + kept <- c(kept, tm) + } else { + warning(paste0("Design term '", tm, "' dropped", tag, ": ", reason), call. = FALSE) + } + } + stats::as.formula(paste("~", paste(labs[labs %in% c(kept, "condition")], collapse = " + "))) +} + +## Design used for one contrast: the configured one when `design=` was given, +## the auto-derived one otherwise, in both cases reduced to its estimable terms. +build_de_design <- function(spec, meta, A, B, label = "", min_residual_df = 1) { + if (nzchar(spec)) { + f <- parse_de_design(spec) + origin <- "configured" + } else { + f <- auto_de_design(meta, A, B) + origin <- "auto-derived" + } + out <- resolve_de_design(f, meta, label, min_residual_df) + message(paste0("Design (", origin, ")", if (nzchar(label)) paste0(" for contrast ", label) else "", ": ", paste(deparse(out), collapse = ""))) + out +} + + +## Spike-in derived size factors from the control gene counts alone, by the +## median of ratios against the geometric mean reference sample. +## +## A fixed amount of spike-in is added per cell / per unit of input, so the +## observed control gene level of a library is proportional to its sequencing +## depth divided by its total endogenous RNA content. Scaling endogenous counts +## by it therefore recovers abundance per cell instead of per library, which is +## the entire point of a spike-in experiment. +## +## The scale must come from the CONTROL GENES and never from the endogenous +## library size: the endogenous library size carries the very RNA content +## difference the spike-ins are there to measure, so using it cancels the +## signal of interest. +## +## Median of ratios rather than TMM on the control genes, for two reasons. +## It is the identical estimator to DESeq2's estimateSizeFactors(controlGenes), +## so both engines are offset by the same quantity and their results can be +## compared directly. And it is the more precise of the two here: over 40 +## simulated spike-in datasets with a known global shift of log2(1.6), both are +## unbiased (-0.02) but the median of ratios has 35% lower RMSE than TMM +## (0.117 vs 0.179); the plain control gene total is unbiased too but far +## noisier (0.508). TMM's trimmed mean corrects composition bias, which control +## genes do not have by construction, and it pays for that robustness with +## variance when only a few dozen spike-ins are available. +spikein_size_factors <- function(spike_counts) { + spike_counts <- as.matrix(spike_counts) + if (nrow(spike_counts) < 2) { + stop("spike-in normalization needs at least two control genes present in the count table") + } + usable <- rowSums(spike_counts <= 0) == 0 + if (sum(usable) < 2) { + stop(paste0( + "only ", sum(usable), " of ", nrow(spike_counts), " control genes are non-zero in every sample, ", + "the spike-in scale cannot be estimated; check the spike-in coverage of the shallowest library" + )) + } + logc <- log(spike_counts[usable, , drop = FALSE]) + sf <- exp(apply(logc - rowMeans(logc), 2, stats::median)) + if (any(!is.finite(sf)) || any(sf <= 0)) { + stop("spike-in size factors are not positive and finite, check that every sample has control gene counts") + } + stats::setNames(sf / exp(mean(log(sf))), colnames(spike_counts)) +} + +## edgeR applies the offset lib.size * norm.factors, so externally derived size +## factors are installed by expressing them relative to the library sizes the +## DGEList currently carries. The norm.factors are rescaled to geometric mean +## 1 (edgeR's convention), which leaves the offsets proportional to +## `size_factors` whatever the library sizes happen to be, before or after +## filtering and with keep.lib.sizes either way. +set_spikein_offsets <- function(dge, size_factors) { + if (length(size_factors) != ncol(dge)) { + stop("set_spikein_offsets: exactly one size factor per sample is required") + } + nf <- as.numeric(size_factors) / dge$samples$lib.size + dge$samples$norm.factors <- nf / exp(mean(log(nf))) + dge +} + +## Permute the samples independently within every column (gene) of a +## samples x genes matrix. Ordering the flattened matrix by (column, random +## key) exploits the column-major layout and permutes every column in one +## pass, which keeps a few hundred permutations affordable even for large +## control gene sets. +permute_within_columns <- function(m) { + keys <- stats::runif(length(m)) + matrix(m[order(col(m), keys)], nrow(m), ncol(m)) +} + +## Squared singular values of a matrix via its Gram matrix. For the variance +## spectrum only d^2 is needed and the number of samples is tiny compared to +## the number of genes, so the n x n eigendecomposition is far cheaper than a +## full SVD of the n x genes matrix. +squared_singular_values <- function(m) { + pmax(eigen(tcrossprod(m), symmetric = TRUE, only.values = TRUE)$values, 0) +} + +## Per-gene pooled within-group variance and the between/within ratio of a +## samples x genes log-expression matrix, used to show what removing k factors +## does to replicate agreement (within should drop) and to the separation of +## the groups under test (between/within should not collapse: that would mean +## the factors are absorbing the biology instead of the nuisance). +group_variance_components <- function(expr, group) { + group <- droplevels(factor(group)) + n <- nrow(expr) + ngroups <- nlevels(group) + if (ngroups < 2 || n - ngroups < 1) { + return(c(within = NA_real_, between_within = NA_real_)) + } + indicator <- stats::model.matrix(~ 0 + group) + means <- (t(indicator) / colSums(indicator)) %*% expr + fitted <- indicator %*% means + within <- colSums((expr - fitted)^2) / (n - ngroups) + grand <- colMeans(expr) + between <- colSums(colSums(indicator) * (means - rep(grand, each = ngroups))^2) / (ngroups - 1) + c( + within = mean(within), + between_within = if (mean(within) > 0) mean(between) / mean(within) else NA_real_ + ) +} + +## Association of one factor with the condition under test: fraction of its +## variance explained by the group labels and the corresponding F test. +factor_condition_association <- function(w, group) { + group <- droplevels(factor(group)) + n <- length(w) + ngroups <- nlevels(group) + if (ngroups < 2 || n - ngroups < 1) { + return(c(r2 = NA_real_, pvalue = NA_real_)) + } + ss_tot <- sum((w - mean(w))^2) + fitted <- as.vector(stats::model.matrix(~ 0 + group) %*% tapply(w, group, mean)) + ss_res <- sum((w - fitted)^2) + if (ss_tot <= 0) { + return(c(r2 = NA_real_, pvalue = NA_real_)) + } + fstat <- ((ss_tot - ss_res) / (ngroups - 1)) / (ss_res / (n - ngroups)) + c( + r2 = 1 - ss_res / ss_tot, + pvalue = if (is.finite(fstat)) stats::pf(fstat, ngroups - 1, n - ngroups, lower.tail = FALSE) else NA_real_ + ) +} + + +## Diagnostics for every candidate k from 0 (no correction) to k_max, on the +## endogenous genes, i.e. on the genes the differential expression is actually +## tested on. Because the columns of W are orthonormal, removing factor j is a +## single projection, so the k-th row is the (k-1)-th row with one more factor +## projected out and nothing has to be refactorized per k. +ruv_k_metrics <- function(expr, factors, varprop, perm_p, k_max, condition, log_sf, k_used, design_rank, contrast = NULL) { + nsamples <- nrow(expr) + groups <- if (!is.null(condition)) droplevels(factor(condition)) else NULL + rows <- list() + corrected <- expr + for (j in 0:k_max) { + if (j > 0) { + u <- factors[, j, drop = FALSE] + corrected <- corrected - u %*% (t(u) %*% corrected) + } + rle <- sweep(corrected, 2, apply(corrected, 2, stats::median), FUN = "-") + variance <- if (!is.null(condition)) { + group_variance_components(corrected, condition) + } else { + c(within = NA_real_, between_within = NA_real_) + } + association <- if (j > 0 && !is.null(condition)) { + factor_condition_association(factors[, j], condition) + } else { + c(r2 = NA_real_, pvalue = NA_real_) + } + ## The global log2 shift between the two groups: the quantity a spike-in + ## normalization exists to recover and the one a factor collinear with + ## the spike-in size factors takes away again. It has to come from the + ## joint fit of W and the condition, not from the W-projected data: as + ## soon as a factor correlates with the condition the two differ, and it + ## is the joint fit that the DE model reports. + shift <- NA_real_ + if (!is.null(groups) && !is.null(contrast)) { + model <- cbind( + intercept = 1, + if (j > 0) factors[, seq_len(j), drop = FALSE], + contrast = as.numeric(groups == contrast[1]) + ) + if (qr(model)$rank == ncol(model)) { + shift <- stats::median(stats::lm.fit(model, expr)$coefficients["contrast", ]) + } + } + rows[[length(rows) + 1]] <- data.frame( + k = j, + var_explained_controls = if (j > 0) varprop[j] else NA_real_, + cum_var_explained_controls = if (j > 0) sum(varprop[seq_len(j)]) else NA_real_, + permutation_pvalue = if (j > 0) perm_p[j] else NA_real_, + cor_W_log2_sizefactor = if (j > 0 && !is.null(log_sf)) stats::cor(factors[, j], log_sf) else NA_real_, + condition_R2 = association[["r2"]], + condition_pvalue = association[["pvalue"]], + rle_median_abs = stats::median(abs(apply(rle, 1, stats::median))), + rle_iqr_median = stats::median(apply(rle, 1, stats::IQR)), + within_group_var = variance[["within"]], + between_within_ratio = variance[["between_within"]], + global_log2_shift = shift, + residual_df = nsamples - design_rank - j, + selected = j == k_used, + stringsAsFactors = FALSE + ) + } + do.call(rbind, rows) +} + + +## Estimate how many factors of unwanted variation (k) RUVg should remove, +## from the negative control (spike-in) genes themselves. +## +## RUVg takes the gene-centered log counts of the control genes, decomposes +## them by SVD and uses the first k left singular vectors as W. How many of +## those components carry structure shared across the control genes rather +## than noise is a dimensionality question, and the standard answer for +## exactly this situation is a permutation test on the eigenvalue spectrum +## (parallel analysis, Buja & Eyuboglu 1992; the same procedure sva::num.sv +## uses by default to pick the number of surrogate variables). Permuting the +## samples independently within every control gene destroys any factor shared +## across genes while leaving each gene's marginal distribution untouched, so +## a component whose share of the total variance exceeds what permuted data +## produce cannot be explained by noise alone. The p-values are made monotone +## in the component index and k is the number of leading components at or +## below `alpha`, which is what makes the choice reproducible instead of the +## usual "look at a PCA plot and pick one". +## +## Unlike sva::num.sv the condition is NOT projected out before the test: RUVg +## assumes the control genes are unaffected by the biology and decomposes them +## as they are, so the spectrum tested here has to be the one RUVg will use. A +## factor that does line up with the condition is reported (condition_R2) and +## warned about instead of being quietly removed. +## +## k is additionally capped so the resulting model stays estimable: +## k <= number of non-degenerate components, k <= #control genes - 1, and +## k <= n_samples - rank(design) - 1 so that at least one residual degree of +## freedom is left for dispersion estimation. +## +## `k` may be "auto" (default) or an integer given via the ruvk= DE option; an +## integer beyond what is estimable is an error rather than a silent downgrade. +ruv_estimate_k <- function(counts, ctrlgenes, design, condition = NULL, size_factors = NULL, + k = "auto", max_k = 10, alpha = 0.1, nperm = 200, seed = 42, + epsilon = 1, label = "", contrast = NULL) { + counts <- as.matrix(counts) + ctrlgenes <- intersect(ctrlgenes, rownames(counts)) + nsamples <- ncol(counts) + if (length(ctrlgenes) < 2) { + stop("ruv_estimate_k needs at least two control genes present in the count table") + } + if (nsamples < 3) { + message("RUV k selection: fewer than three samples, no factor of unwanted variation is estimable (k = 0)") + } + design <- as.matrix(design) + design_rank <- qr(design)$rank + + ## the very matrix RUVg factorizes: samples x genes, log counts, gene-centered + expr <- scale(t(log(counts + epsilon)), center = TRUE, scale = FALSE) + controls <- expr[, ctrlgenes, drop = FALSE] + decomp <- svd(controls) + varprop <- decomp$d^2 / sum(decomp$d^2) + ncomp <- sum(decomp$d > 1e-8) + k_max <- max(0, min(max_k, ncomp, length(ctrlgenes) - 1, nsamples - design_rank - 1)) + + ## permutation null of the variance spectrum + set.seed(seed) + null <- matrix(NA_real_, nrow = nperm, ncol = length(varprop)) + for (b in seq_len(nperm)) { + permuted <- squared_singular_values(permute_within_columns(controls)) + kept <- seq_len(min(length(permuted), ncol(null))) + null[b, kept] <- permuted[kept] / sum(permuted) + } + perm_p <- cummax(vapply( + seq_along(varprop), + function(j) mean(null[, j] >= varprop[j], na.rm = TRUE), + numeric(1) + )) + null_quantile <- apply(null, 2, stats::quantile, probs = 1 - alpha, na.rm = TRUE) + k_auto <- min(sum(perm_p <= alpha), k_max) + + k_requested <- NA_integer_ + if (!identical(tolower(as.character(k)), "auto")) { + k_requested <- suppressWarnings(as.integer(k)) + if (is.na(k_requested) || k_requested < 0) { + stop(paste0("ruvk must be 'auto' or a non-negative integer, got '", k, "'")) + } + if (k_requested > k_max) { + stop(paste0( + "ruvk=", k_requested, " was requested but at most k=", k_max, " is estimable here (", + nsamples, " samples, design rank ", design_rank, ", ", length(ctrlgenes), + " control genes, ", ncomp, " non-degenerate components). Lower ruvk or use ruvk=auto." + )) + } + } + k_used <- if (is.na(k_requested)) k_auto else k_requested + + ## The diagnostics are reported for the endogenous genes on the scale the + ## DE test sees them: counts divided by the spike-in size factors, log2 for + ## readability, with the same factors projected out that the design would + ## adjust for. W is unaffected by the choice of log base, so this stays the + ## decomposition RUVg performs. + endogenous <- setdiff(rownames(counts), ctrlgenes) + expressed <- endogenous[rowSums(counts[endogenous, , drop = FALSE]) > 0] + normalized <- counts[expressed, , drop = FALSE] + if (!is.null(size_factors)) { + normalized <- sweep(normalized, 2, size_factors, FUN = "/") + } + log_sf <- if (!is.null(size_factors)) log2(size_factors) else NULL + ## the direction of the reported global shift follows the contrast + ## (A vs B), never the alphabetical level order + if (!is.null(contrast) && (length(contrast) != 2 || !all(contrast %in% levels(droplevels(factor(condition)))))) { + stop("ruv_estimate_k: contrast must be the two condition groups of the comparison, A first") + } + metrics <- ruv_k_metrics( + scale(t(log2(normalized + epsilon)), center = TRUE, scale = FALSE), + decomp$u, varprop, perm_p, k_max, condition, log_sf, k_used, design_rank, contrast + ) + + selection <- list( + k = k_used, k_auto = k_auto, k_max = k_max, k_requested = k_requested, + method = if (is.na(k_requested)) "permutation (parallel analysis, Buja & Eyuboglu)" else "user supplied (ruvk=)", + alpha = alpha, nperm = nperm, seed = seed, nsamples = nsamples, + n_control = length(ctrlgenes), n_endogenous = length(expressed), design_rank = design_rank, + varprop = varprop, null_quantile = null_quantile, perm_p = perm_p, + metrics = metrics, label = label, + contrast = if (is.null(contrast)) NA_character_ else paste(contrast[1], "vs", contrast[2]) + ) + ruv_announce_k(selection) + selection +} + + +## Log the outcome of the k selection and flag the two properties a selected +## factor can have that change how the _ruv branch must be read. RUVg factors +## and the spike-in size factors are estimated from the SAME control genes, so +## when the spike-in is added per cell the control genes encode RNA content per +## cell and W becomes collinear with the size factors: the _ruv branch then +## reports fold changes relative to total RNA, i.e. the spike-anchored reading +## lives in the _norm branch and only there. A factor that differs between the +## groups under test is absorbing biology rather than nuisance and makes the +## _ruv branch conservative. Both are reported, never silently acted upon. +ruv_announce_k <- function(selection, collinearity_warn = 0.8, condition_r2_warn = 0.5) { + tag <- if (nzchar(selection$label)) paste0(" [", selection$label, "]") else "" + message(sprintf( + "RUV k selection%s: k = %d (%s), permutation estimate k_auto = %d, estimable maximum k_max = %d, %d control genes, %d samples, design rank %d", + tag, selection$k, selection$method, selection$k_auto, selection$k_max, + selection$n_control, selection$nsamples, selection$design_rank + )) + if (!is.na(selection$k_requested) && selection$k_requested != selection$k_auto) { + message(sprintf( + " NOTE: ruvk=%d overrides the permutation estimate of %d factors%s", + selection$k_requested, selection$k_auto, + if (selection$k_requested > selection$k_auto) " (more factors than the control genes support, this removes variance the data do not show to be unwanted)" else "" + )) + } + if (selection$k == 0) { + message(" No factor of unwanted variation is used: the _ruv design is left unchanged, so the _ruv branch reproduces the standard branch exactly. Only the _norm branch (spike-in size factors) still differs.") + } + for (j in seq_len(selection$k)) { + row <- selection$metrics[selection$metrics$k == j, ] + message(sprintf( + " W_%d: %.1f%% of control gene variance, permutation p = %.3f, cor(W_%d, log2 sizeFactor) = %s, condition R2 = %s (p = %s)", + j, 100 * row$var_explained_controls, row$permutation_pvalue, j, + format(round(row$cor_W_log2_sizefactor, 3)), format(round(row$condition_R2, 3)), + format(signif(row$condition_pvalue, 3)) + )) + if (!is.na(row$cor_W_log2_sizefactor) && abs(row$cor_W_log2_sizefactor) >= collinearity_warn) { + shift0 <- selection$metrics$global_log2_shift[selection$metrics$k == 0] + message(sprintf( + " NOTE: W_%d is %.0f%% collinear with the spike-in size factors, both are derived from the same control genes. The _ruv branch therefore reports fold changes RELATIVE to total RNA, not spike-anchored ones%s. Read the absolute change off the _norm tables, which are scaled by the spike-ins.", + j, 100 * abs(row$cor_W_log2_sizefactor), + if (is.finite(shift0) && is.finite(row$global_log2_shift)) { + sprintf(": the global log2 shift between the groups goes from %.2f without this factor to %.2f with it", shift0, row$global_log2_shift) + } else { + "" + } + )) + } + if (!is.na(row$condition_R2) && row$condition_R2 >= condition_r2_warn) { + message(sprintf( + " WARNING: W_%d differs between the groups being tested (R2 = %.2f), it is absorbing biology, not only nuisance. Check the k-report before trusting the _ruv results.", + j, row$condition_R2 + )) + } + } + invisible(selection) +} + +## Write the per-k diagnostics next to the results, so the choice of k can be +## judged (and reproduced) without rerunning anything. Parameters of the +## selection go into leading '#' comment lines, the table itself is plain TSV. +ruv_write_k_report <- function(selection, path) { + header <- c( + paste0("# RUVg factor selection", if (nzchar(selection$label)) paste0(" for ", selection$label) else ""), + paste0("# k_selected\t", selection$k), + paste0("# k_method\t", selection$method), + paste0("# k_auto_permutation\t", selection$k_auto), + paste0("# k_requested\t", if (is.na(selection$k_requested)) "auto" else selection$k_requested), + paste0("# k_max_estimable\t", selection$k_max), + paste0("# samples\t", selection$nsamples), + paste0("# design_rank\t", selection$design_rank), + paste0("# control_genes\t", selection$n_control), + paste0("# endogenous_genes\t", selection$n_endogenous), + paste0("# permutations\t", selection$nperm), + paste0("# alpha\t", selection$alpha), + paste0("# seed\t", selection$seed), + paste0("# global_log2_shift_direction\t", selection$contrast) + ) + writeLines(header, path) + suppressWarnings(utils::write.table( + selection$metrics, path, sep = "\t", row.names = FALSE, quote = FALSE, append = TRUE + )) + invisible(path) +} + + +## Visual counterpart of the k-report: the control gene variance spectrum +## against its permutation null (what the choice of k is based on), what each +## candidate factor is correlated with (what it would cost), and how the data +## actually change with k (RLE spread and within-group variance). +ruv_plot_k <- function(selection, outfile) { + metrics <- selection$metrics + ncomp <- min(length(selection$varprop), max(selection$nsamples - 1, 1)) + grDevices::png(outfile, width = 3600, height = 1000, res = 220) + on.exit(grDevices::dev.off()) + graphics::par(mfrow = c(1, 4), mar = c(4.5, 4.5, 3, 4)) + + observed <- 100 * selection$varprop[seq_len(ncomp)] + null <- 100 * selection$null_quantile[seq_len(ncomp)] + graphics::plot(seq_len(ncomp), observed, + type = "b", pch = 19, col = "steelblue", ylim = c(0, max(observed, null, na.rm = TRUE) * 1.1), + xlab = "factor", ylab = "% of control gene variance", + main = paste0("spectrum vs permutation null\nk = ", selection$k, " (k_auto = ", selection$k_auto, ", k_max = ", selection$k_max, ")") + ) + graphics::lines(seq_len(ncomp), null, type = "b", pch = 1, lty = 2, col = "grey40") + if (selection$k > 0) { + graphics::abline(v = selection$k + 0.5, col = "firebrick", lty = 3) + } + graphics::legend("topright", bty = "n", cex = 0.8, pch = c(19, 1), lty = c(1, 2), + col = c("steelblue", "grey40"), legend = c("observed", paste0((1 - selection$alpha) * 100, "% of permuted"))) + + factors <- metrics[metrics$k > 0, , drop = FALSE] + if (nrow(factors) > 0) { + bars <- rbind(abs(factors$cor_W_log2_sizefactor), factors$condition_R2) + bars[is.na(bars)] <- 0 + graphics::barplot(bars, beside = TRUE, names.arg = paste0("W_", factors$k), ylim = c(0, 1), + col = c("darkorange", "darkolivegreen"), ylab = "association", xlab = "factor", + main = "what each factor is tied to") + graphics::abline(h = c(0.5, 0.8), lty = c(3, 2), col = c("darkolivegreen", "darkorange")) + graphics::legend("topright", bty = "n", cex = 0.8, fill = c("darkorange", "darkolivegreen"), + legend = c("|cor| with log2 sizeFactor", "R2 with condition")) + } else { + graphics::plot.new() + graphics::title(main = "no estimable factor") + } + + graphics::plot(metrics$k, metrics$rle_iqr_median, + type = "b", pch = 19, col = "steelblue", ylim = c(0, max(metrics$rle_iqr_median, na.rm = TRUE) * 1.1), + xlab = "k", ylab = "median RLE IQR", main = "effect on the endogenous genes" + ) + if (any(is.finite(metrics$within_group_var))) { + graphics::par(new = TRUE) + graphics::plot(metrics$k, metrics$within_group_var, type = "b", pch = 17, lty = 2, + col = "firebrick", axes = FALSE, xlab = "", ylab = "", + ylim = c(0, max(metrics$within_group_var, na.rm = TRUE) * 1.1)) + graphics::axis(4, col = "firebrick", col.axis = "firebrick") + graphics::mtext("within-group variance", side = 4, line = 2.2, cex = 0.7, col = "firebrick") + } + if (selection$k > 0) { + graphics::abline(v = selection$k, col = "firebrick", lty = 3) + } + + if (any(is.finite(metrics$global_log2_shift))) { + graphics::plot(metrics$k, metrics$global_log2_shift, + type = "b", pch = 19, col = "purple4", xlab = "k", ylab = "median log2 shift between groups", + main = "spike-in scale kept per k" + ) + graphics::abline(h = 0, lty = 2, col = "grey40") + if (selection$k > 0) { + graphics::abline(v = selection$k, col = "firebrick", lty = 3) + } + } else { + graphics::plot.new() + graphics::title(main = "global shift needs two groups") + } + invisible(outfile) +} + + +## apeglm fits an ADAPTIVE prior whose scale is estimated from the spread of +## the MLE effect sizes. On a contrast with almost no signal left the prior +## collapses towards a point mass at zero, and most genes come back with +## log2FoldChange == 0 and lfcSE ~ 1e-3. Those columns are then unusable for +## effect sizes, ranking or correlation while still looking like ordinary +## output, so the degenerate case is called out explicitly. Diagnostic only, +## the results themselves are left untouched. +check_shrinkage <- function(res_raw, res_shrink, label = "", min_lfcSE = 0.01, max_pinned = 0.05) { + tag <- if (nzchar(label)) paste0(" [", label, "]") else "" + median_se <- stats::median(res_shrink$lfcSE, na.rm = TRUE) + pinned <- mean(abs(res_shrink$log2FoldChange) < 1e-6, na.rm = TRUE) + raw_lfc <- stats::median(abs(res_raw$log2FoldChange), na.rm = TRUE) + shrunk_lfc <- stats::median(abs(res_shrink$log2FoldChange), na.rm = TRUE) + ratio <- if (is.finite(shrunk_lfc) && shrunk_lfc > 0) raw_lfc / shrunk_lfc else Inf + message(sprintf( + "Shrinkage check%s: median lfcSE = %.5f, fraction |LFC| < 1e-6 = %.3f, median |LFC| raw/shrunk = %.1fx", + tag, median_se, pinned, ratio + )) + if ((!is.na(median_se) && median_se < min_lfcSE) || (!is.na(pinned) && pinned > max_pinned)) { + message(paste0( + " WARNING: apeglm shrinkage looks DEGENERATE for this contrast. The shrunken log2FoldChange / lfcSE columns ", + "must not be used for effect sizes or correlations, use the *_noshrink table instead. This usually means the ", + "contrast has little signal left; check whether a covariate (e.g. a RUV factor W_i) is absorbing it." + )) + } + invisible(c(median_lfcSE = median_se, frac_pinned = pinned, raw_over_shrunk = ratio)) +} diff --git a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subflow.nf b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subflow.nf index 74b21753..12c5a929 100644 --- a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subflow.nf +++ b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -114,10 +134,10 @@ process salmon_quant{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'ISF'){ + if (params.gDTUPAIRED == 'paired'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'ISF'){ stranded = '-l ISF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'ISR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'ISR'){ stranded = '-l ISR' }else{ stranded = '-l IU' @@ -134,9 +154,9 @@ process salmon_quant{ ${params.gCOUNTBIN} ${params.gCOUNTPARAMS} quant -p ${task.cpus} -i $idx $stranded -o $fn -1 $r1 -2 $r2 &>> $lf && gzip $of && ln -fs $oz $ol """ }else{ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'SF'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'SF'){ stranded = '-l SF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'SR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'SR'){ stranded = '-l SR' }else{ stranded = '-l U' @@ -344,7 +364,7 @@ workflow DTU{ checkidx = file(params.gDTUIDX) if (checkidx.exists()){ idxfile = Channel.fromPath(params.gDTUUIDX) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -353,7 +373,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(params.gDTUREF) salmon_idx(genomefile) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subflow.nf b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subflow.nf index 129f93e8..3fb7e63f 100644 --- a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subflow.nf +++ b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -114,10 +134,10 @@ process salmon_quant{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'ISF'){ + if (params.gDTUPAIRED == 'paired'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'ISF'){ stranded = '-l ISF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'ISR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'ISR'){ stranded = '-l ISR' }else{ stranded = '-l IU' @@ -134,9 +154,9 @@ process salmon_quant{ ${params.gCOUNTBIN} ${params.gCOUNTPARAMS} quant -p ${task.cpus} -i $idx $stranded -o $fn -1 $r1 -2 $r2 &>> $lf && gzip $of && ln -fs $oz $ol """ }else{ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'SF'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'SF'){ stranded = '-l SF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'SR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'SR'){ stranded = '-l SR' }else{ stranded = '-l U' @@ -345,7 +365,7 @@ workflow DTU{ checkidx = file(params.gDTUIDX) if (checkidx.exists()){ idxfile = Channel.fromPath(params.gDTUUIDX) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -354,7 +374,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(params.gDTUREF) salmon_idx(genomefile) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subflow.nf b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subflow.nf index ccf4b98a..2748edbd 100644 --- a/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subflow.nf +++ b/tests/golden/config_DTU/nf/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -114,10 +134,10 @@ process salmon_quant{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'ISF'){ + if (params.gDTUPAIRED == 'paired'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'ISF'){ stranded = '-l ISF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'ISR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'ISR'){ stranded = '-l ISR' }else{ stranded = '-l IU' @@ -134,9 +154,9 @@ process salmon_quant{ ${params.gCOUNTBIN} ${params.gCOUNTPARAMS} quant -p ${task.cpus} -i $idx $stranded -o $fn -1 $r1 -2 $r2 &>> $lf && gzip $of && ln -fs $oz $ol """ }else{ - if (params.gSTRANDED == 'fr' || params.gSTRANDED == 'SF'){ + if (params.gDTUSTRANDED == 'fr' || params.gDTUSTRANDED == 'SF'){ stranded = '-l SF' - }else if (params.gSTRANDED == 'rf' || params.gSTRANDED == 'SR'){ + }else if (params.gDTUSTRANDED == 'rf' || params.gDTUSTRANDED == 'SR'){ stranded = '-l SR' }else{ stranded = '-l U' @@ -343,7 +363,7 @@ workflow DTU{ checkidx = file(params.gDTUIDX) if (checkidx.exists()){ idxfile = Channel.fromPath(params.gDTUUIDX) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -352,7 +372,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(params.gDTUREF) salmon_idx(genomefile) - if (params.gPAIRED == 'paired'){ + if (params.gDTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subsnake.smk b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subsnake.smk index 868fbe7a..e9fa93fb 100644 --- a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subsnake.smk +++ b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_dexseq_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -352,7 +383,7 @@ if COUNTTOOL == 'salmon': decoy = f"-d {os.path.abspath(DECOY)}" if DECOY else '', linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "set +euo pipefail; {params.mapp} index {params.ipara} {params.decoy} -p {threads} -t {input.fa} -i {output.uidx} &>> {log} && ln -fs {params.linkidx} {output.idx}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule simulate_trim: input: r1 = lambda wildcards: "FASTQ/{rawfile}_R1.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R1_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file), r2 = lambda wildcards: "FASTQ/{rawfile}_R2.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R2_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file) @@ -369,7 +400,7 @@ if COUNTTOOL == 'salmon': threads: 1 params: filetolink = lambda w, input: "{r}".format(r=os.path.abspath(input.r1)) shell: "ln -s {params.filetolink} {output.r1}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule mapping: input: r1 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R1_trimmed.fastq.gz", scombo=scombo), r2 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R2_trimmed.fastq.gz", scombo=scombo), @@ -383,7 +414,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l ISF' if (stranded == 'fr' or stranded == 'ISF') else '-l ISR' if (stranded == 'rf' or stranded == 'ISR') else '-l IU', + stranded = lambda x: '-l ISF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'ISF') else '-l ISR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'ISR') else '-l IU', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -1 {input.r1} -2 {input.r2} &>> {log} && gzip {output.ctsdir}/quant.sf && ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: @@ -399,7 +430,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l SF' if (stranded == 'fr' or stranded == 'SF') else '-l SR' if (stranded == 'rf' or stranded == 'SR') else '-l U', + stranded = lambda x: '-l SF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'SF') else '-l SR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'SR') else '-l U', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -r {input.r1} &>> {log} && gzip {output.ctsdir}/quant.sf ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: diff --git a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subsnake.smk b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subsnake.smk index 1cc1c0b5..d49c7bf4 100644 --- a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subsnake.smk +++ b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_drimseq_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -362,7 +393,7 @@ if COUNTTOOL == 'salmon': decoy = f"-d {os.path.abspath(DECOY)}" if DECOY else '', linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "set +euo pipefail; {params.mapp} index {params.ipara} {params.decoy} -p {threads} -t {input.fa} -i {output.uidx} &>> {log} && ln -fs {params.linkidx} {output.idx}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule simulate_trim: input: r1 = lambda wildcards: "FASTQ/{rawfile}_R1.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R1_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file), r2 = lambda wildcards: "FASTQ/{rawfile}_R2.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R2_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file) @@ -379,7 +410,7 @@ if COUNTTOOL == 'salmon': threads: 1 params: filetolink = lambda w, input: "{r}".format(r=os.path.abspath(input.r1)) shell: "ln -s {params.filetolink} {output.r1}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule mapping: input: r1 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R1_trimmed.fastq.gz", scombo=scombo), r2 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R2_trimmed.fastq.gz", scombo=scombo), @@ -393,7 +424,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l ISF' if (stranded == 'fr' or stranded == 'ISF') else '-l ISR' if (stranded == 'rf' or stranded == 'ISR') else '-l IU', + stranded = lambda x: '-l ISF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'ISF') else '-l ISR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'ISR') else '-l IU', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -1 {input.r1} -2 {input.r2} &>> {log} && gzip {output.ctsdir}/quant.sf && ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: @@ -409,7 +440,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l SF' if (stranded == 'fr' or stranded == 'SF') else '-l SR' if (stranded == 'rf' or stranded == 'SR') else '-l U', + stranded = lambda x: '-l SF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'SF') else '-l SR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'SR') else '-l U', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -r {input.r1} &>> {log} && gzip {output.ctsdir}/quant.sf ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: diff --git a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subsnake.smk b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subsnake.smk index 4c4f8440..c8dba7cc 100644 --- a/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subsnake.smk +++ b/tests/golden/config_DTU/smk/allconditions_fastqc-trimgalore-picard-star_DTU_spit_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -359,7 +390,7 @@ if COUNTTOOL == 'salmon': decoy = f"-d {os.path.abspath(DECOY)}" if DECOY else '', linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "set +euo pipefail; {params.mapp} index {params.ipara} {params.decoy} -p {threads} -t {input.fa} -i {output.uidx} &>> {log} && ln -fs {params.linkidx} {output.idx}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule simulate_trim: input: r1 = lambda wildcards: "FASTQ/{rawfile}_R1.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R1_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file), r2 = lambda wildcards: "FASTQ/{rawfile}_R2.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R2_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file) @@ -376,7 +407,7 @@ if COUNTTOOL == 'salmon': threads: 1 params: filetolink = lambda w, input: "{r}".format(r=os.path.abspath(input.r1)) shell: "ln -s {params.filetolink} {output.r1}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule mapping: input: r1 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R1_trimmed.fastq.gz", scombo=scombo), r2 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R2_trimmed.fastq.gz", scombo=scombo), @@ -390,7 +421,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l ISF' if (stranded == 'fr' or stranded == 'ISF') else '-l ISR' if (stranded == 'rf' or stranded == 'ISR') else '-l IU', + stranded = lambda x: '-l ISF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'ISF') else '-l ISR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'ISR') else '-l IU', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -1 {input.r1} -2 {input.r2} &>> {log} && gzip {output.ctsdir}/quant.sf && ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: @@ -406,7 +437,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l SF' if (stranded == 'fr' or stranded == 'SF') else '-l SR' if (stranded == 'rf' or stranded == 'SR') else '-l U', + stranded = lambda x: '-l SF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'SF') else '-l SR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'SR') else '-l U', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -r {input.r1} &>> {log} && gzip {output.ctsdir}/quant.sf ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" else: diff --git a/tests/golden/config_FUSIONS/nf/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subflow.nf b/tests/golden/config_FUSIONS/nf/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subflow.nf index 98edf83d..0ba5e702 100644 --- a/tests/golden/config_FUSIONS/nf/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subflow.nf +++ b/tests/golden/config_FUSIONS/nf/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' diff --git a/tests/golden/config_FUSIONS/smk/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subsnake.smk b/tests/golden/config_FUSIONS/smk/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subsnake.smk index 6aba6a5b..4cb6b080 100644 --- a/tests/golden/config_FUSIONS/smk/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subsnake.smk +++ b/tests/golden/config_FUSIONS/smk/allconditions_fastqc-trimgalore-picard-star_FUSIONS_starfusion_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) diff --git a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-bwa_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-bwa_subflow.nf index 3b140ef3..60429dbd 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-bwa_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-bwa_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -360,7 +380,7 @@ process bwa_mapping{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-rustar_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-rustar_subflow.nf index ecbcfc01..9201ac82 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-rustar_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-rustar_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -368,7 +388,7 @@ process rustar_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -381,7 +401,7 @@ process rustar_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -391,9 +411,9 @@ process rustar_mapping{ """ } else{ - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-star_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-star_subflow.nf index b54480be..bf499548 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-star_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-bbduk-picard-star_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -368,7 +388,7 @@ process star_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -381,7 +401,7 @@ process star_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -392,9 +412,9 @@ process star_mapping{ } else{ rds = sort_reads(reads[1..2]) - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-bwa_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-bwa_subflow.nf index c802ba45..03870387 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-bwa_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-bwa_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -360,7 +380,7 @@ process bwa_mapping{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-rustar_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-rustar_subflow.nf index 8aea7c38..29e31114 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-rustar_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-rustar_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -368,7 +388,7 @@ process rustar_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -381,7 +401,7 @@ process rustar_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -391,9 +411,9 @@ process rustar_mapping{ """ } else{ - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-star_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-star_subflow.nf index a7dd8500..d11da0dd 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-star_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-cutadapt-picard-star_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -368,7 +388,7 @@ process star_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -381,7 +401,7 @@ process star_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -392,9 +412,9 @@ process star_mapping{ } else{ rds = sort_reads(reads[1..2]) - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-bwa_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-bwa_subflow.nf index 432ce0a0..02968428 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-bwa_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-bwa_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -355,7 +375,7 @@ process bwa_mapping{ path "*.log", emit: logs script: idx = reads[0] - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-rustar_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-rustar_subflow.nf index f18e5841..c5401bfd 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-rustar_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-rustar_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -363,7 +383,7 @@ process rustar_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -376,7 +396,7 @@ process rustar_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -386,9 +406,9 @@ process rustar_mapping{ """ } else{ - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-star_subflow.nf b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-star_subflow.nf index b0a2c073..f973ec87 100644 --- a/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-star_subflow.nf +++ b/tests/golden/config_Test/nf/Test_fastqc-trimgalore-picard-star_subflow.nf @@ -11,6 +11,26 @@ params.gPAIRED = get_always('PAIRED') ?: null params.gRUNDEDUP = get_always('RUNDEDUP') ?: null params.gPREDEDUP = get_always('PREDEDUP') ?: null params.gSTRANDED = get_always('STRANDED') ?: null +params.gMAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: params.gPAIRED +params.gMAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: params.gSTRANDED +params.gPEAKPAIRED = get_always('PEAKPAIRED') ?: params.gMAPPINGPAIRED +params.gPEAKSTRANDED = get_always('PEAKSTRANDED') ?: params.gMAPPINGSTRANDED +params.gTRACKSPAIRED = get_always('TRACKSPAIRED') ?: params.gMAPPINGPAIRED +params.gTRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCOUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: params.gMAPPINGPAIRED +params.gCOUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: params.gMAPPINGSTRANDED +params.gCIRCSPAIRED = get_always('CIRCSPAIRED') ?: params.gMAPPINGPAIRED +params.gCIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gFUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: params.gMAPPINGPAIRED +params.gFUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEPAIRED = get_always('DEPAIRED') ?: params.gMAPPINGPAIRED +params.gDESTRANDED = get_always('DESTRANDED') ?: params.gMAPPINGSTRANDED +params.gDEUPAIRED = get_always('DEUPAIRED') ?: params.gMAPPINGPAIRED +params.gDEUSTRANDED = get_always('DEUSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDASPAIRED = get_always('DASPAIRED') ?: params.gMAPPINGPAIRED +params.gDASSTRANDED = get_always('DASSTRANDED') ?: params.gMAPPINGSTRANDED +params.gDTUPAIRED = get_always('DTUPAIRED') ?: params.gPAIRED +params.gDTUSTRANDED = get_always('DTUSTRANDED') ?: params.gSTRANDED params.gIP = get_always('IP') ?: null params.gCONDITION = get_always('CONDITION') ?: null params.gCOMBO = get_always('COMBO') ?: '' @@ -363,7 +383,7 @@ process star_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (params.gPAIRED == 'paired'){ + if (params.gMAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -376,7 +396,7 @@ process star_mapping{ """ } else{ - if (params.gPAIRED != 'singlecell'){ + if (params.gMAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -387,9 +407,9 @@ process star_mapping{ } else{ rds = sort_reads(reads[1..2]) - if (params.gSTRANDED == 'fr'){ + if (params.gMAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (params.gSTRANDED == 'rf'){ + }else if (params.gMAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-bwa_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-bwa_subsnake.smk index f30e41e7..6102cea1 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-bwa_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-bwa_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -503,7 +534,7 @@ rule generate_index: shell: "if [[ -f \"{output.uidx}\\/*\" ]]; then ln -fs {params.linkidx} {output.idx} && touch {output.uidx} && echo \"Found bwa index, continue with mapping\" ; else {params.indexer} index -p {output.uidx} {params.ipara} {input.ref} 2> {log} && ln -fs {params.linkidx} {output.idx} && touch {output.uidx};fi" bwaalg = MAPPERBIN.split(' ')[1] if bwaalg == 'mem': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -522,7 +553,7 @@ if bwaalg == 'mem': shell: "{params.mapp} {params.mpara} -t {threads} {input.index} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -536,7 +567,7 @@ if bwaalg == 'mem': #idx = lambda wildcards, input: str.join(os.sep,[str(input.index), PREFIX]) if PREFIX != '' else input.index shell: "{params.mapp} {params.mpara} -t {threads} {input.uidx} {input.query} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to continue the workflow - if paired == 'paired': # handled like sampe + if MAPPING_PAIRED == 'paired': # handled like sampe rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -572,7 +603,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} > {output.sai1} 2>> {log} && {params.mapp} {params.mpara} -t {threads} {input.ref} {input.r2} > {output.sai2} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" else: #handled like samse rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -587,7 +618,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to shell: "{params.mapp1} aln {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} && {params.mapp} {params.mpara} {input.ref} {output.sai} {input.query} 2> {log} | tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" ### FOR LATER IF WE EVER NEED aln MODE # rule mapping: -# input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", +# input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), # #index = lambda wildcards: expand(rules.generate_index.output.idx, ref=REFERENCE, dir=source_from_sample(wildcards.file, config), gen=genome(wildcards.file, config), name=namefromfile(wildcards.file, config), mape=MAPPERENV, extension=check_tool_params(wildcards.file, None, config, 'MAPPING', 2)), # ref = lambda wildcards: expand(rules.generate_index.input.fa, ref=REFERENCE, dir = source_from_sample(wildcards.file, config), gen =genome(wildcards.file, config), name=namefromfile(wildcards.file, config)) # output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), @@ -601,7 +632,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" elif bwaalg == 'samse': rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -633,7 +664,7 @@ elif bwaalg == 'sampe': mapp1 = MAPPERBIN.split(' ')[0] shell: "{params.mapp} {params.mpara} {input.ref} {input.sai1} {input.sai2} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" elif bwaalg == 'bwasw': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -650,7 +681,7 @@ elif bwaalg == 'bwasw': shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} - ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-rustar_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-rustar_subsnake.smk index de09320b..7ca296c9 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-rustar_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-rustar_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -505,7 +536,7 @@ rule generate_index: pref = PREFIX, lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir TMP/rustar_generate_index --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf TMP/rustar_generate_index" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -529,7 +560,7 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", uidx = rules.generate_index.output.uidx[0], @@ -568,7 +599,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-star_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-star_subsnake.smk index b80e8001..963c7f6e 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-star_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-bbduk-picard-star_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -506,7 +537,7 @@ rule generate_index: lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])), stmp = expand("TMP/{mape}_{unikey}/generate_index", mape=MAPPERENV, unikey=unik)[0] shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && rm -rf {params.stmp} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir {params.stmp} --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf {params.stmp}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -530,9 +561,9 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: - input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: r1 = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], dummy = rules.generate_index.output.idxfile, ref = REFERENCE @@ -569,7 +600,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-bwa_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-bwa_subsnake.smk index c77aa534..5888f06c 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-bwa_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-bwa_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -503,7 +534,7 @@ rule generate_index: shell: "if [[ -f \"{output.uidx}\\/*\" ]]; then ln -fs {params.linkidx} {output.idx} && touch {output.uidx} && echo \"Found bwa index, continue with mapping\" ; else {params.indexer} index -p {output.uidx} {params.ipara} {input.ref} 2> {log} && ln -fs {params.linkidx} {output.idx} && touch {output.uidx};fi" bwaalg = MAPPERBIN.split(' ')[1] if bwaalg == 'mem': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -522,7 +553,7 @@ if bwaalg == 'mem': shell: "{params.mapp} {params.mpara} -t {threads} {input.index} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -536,7 +567,7 @@ if bwaalg == 'mem': #idx = lambda wildcards, input: str.join(os.sep,[str(input.index), PREFIX]) if PREFIX != '' else input.index shell: "{params.mapp} {params.mpara} -t {threads} {input.uidx} {input.query} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to continue the workflow - if paired == 'paired': # handled like sampe + if MAPPING_PAIRED == 'paired': # handled like sampe rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -572,7 +603,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} > {output.sai1} 2>> {log} && {params.mapp} {params.mpara} -t {threads} {input.ref} {input.r2} > {output.sai2} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" else: #handled like samse rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -587,7 +618,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to shell: "{params.mapp1} aln {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} && {params.mapp} {params.mpara} {input.ref} {output.sai} {input.query} 2> {log} | tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" ### FOR LATER IF WE EVER NEED aln MODE # rule mapping: -# input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", +# input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), # #index = lambda wildcards: expand(rules.generate_index.output.idx, ref=REFERENCE, dir=source_from_sample(wildcards.file, config), gen=genome(wildcards.file, config), name=namefromfile(wildcards.file, config), mape=MAPPERENV, extension=check_tool_params(wildcards.file, None, config, 'MAPPING', 2)), # ref = lambda wildcards: expand(rules.generate_index.input.fa, ref=REFERENCE, dir = source_from_sample(wildcards.file, config), gen =genome(wildcards.file, config), name=namefromfile(wildcards.file, config)) # output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), @@ -601,7 +632,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" elif bwaalg == 'samse': rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -633,7 +664,7 @@ elif bwaalg == 'sampe': mapp1 = MAPPERBIN.split(' ')[0] shell: "{params.mapp} {params.mpara} {input.ref} {input.sai1} {input.sai2} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" elif bwaalg == 'bwasw': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -650,7 +681,7 @@ elif bwaalg == 'bwasw': shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} - ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-rustar_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-rustar_subsnake.smk index 15ec53d0..85e150e9 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-rustar_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-rustar_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -505,7 +536,7 @@ rule generate_index: pref = PREFIX, lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir TMP/rustar_generate_index --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf TMP/rustar_generate_index" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -529,7 +560,7 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", uidx = rules.generate_index.output.uidx[0], @@ -568,7 +599,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-star_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-star_subsnake.smk index af644e01..225b8a2e 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-star_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-cutadapt-picard-star_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -506,7 +537,7 @@ rule generate_index: lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])), stmp = expand("TMP/{mape}_{unikey}/generate_index", mape=MAPPERENV, unikey=unik)[0] shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && rm -rf {params.stmp} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir {params.stmp} --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf {params.stmp}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -530,9 +561,9 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: - input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: r1 = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], dummy = rules.generate_index.output.idxfile, ref = REFERENCE @@ -569,7 +600,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-bwa_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-bwa_subsnake.smk index 514ac158..5f5787df 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-bwa_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-bwa_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -510,7 +541,7 @@ rule generate_index: shell: "if [[ -f \"{output.uidx}\\/*\" ]]; then ln -fs {params.linkidx} {output.idx} && touch {output.uidx} && echo \"Found bwa index, continue with mapping\" ; else {params.indexer} index -p {output.uidx} {params.ipara} {input.ref} 2> {log} && ln -fs {params.linkidx} {output.idx} && touch {output.uidx};fi" bwaalg = MAPPERBIN.split(' ')[1] if bwaalg == 'mem': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -529,7 +560,7 @@ if bwaalg == 'mem': shell: "{params.mapp} {params.mpara} -t {threads} {input.index} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -543,7 +574,7 @@ if bwaalg == 'mem': #idx = lambda wildcards, input: str.join(os.sep,[str(input.index), PREFIX]) if PREFIX != '' else input.index shell: "{params.mapp} {params.mpara} -t {threads} {input.uidx} {input.query} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to continue the workflow - if paired == 'paired': # handled like sampe + if MAPPING_PAIRED == 'paired': # handled like sampe rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -579,7 +610,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} > {output.sai1} 2>> {log} && {params.mapp} {params.mpara} -t {threads} {input.ref} {input.r2} > {output.sai2} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" else: #handled like samse rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -594,7 +625,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to shell: "{params.mapp1} aln {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} && {params.mapp} {params.mpara} {input.ref} {output.sai} {input.query} 2> {log} | tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" ### FOR LATER IF WE EVER NEED aln MODE # rule mapping: -# input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", +# input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), # #index = lambda wildcards: expand(rules.generate_index.output.idx, ref=REFERENCE, dir=source_from_sample(wildcards.file, config), gen=genome(wildcards.file, config), name=namefromfile(wildcards.file, config), mape=MAPPERENV, extension=check_tool_params(wildcards.file, None, config, 'MAPPING', 2)), # ref = lambda wildcards: expand(rules.generate_index.input.fa, ref=REFERENCE, dir = source_from_sample(wildcards.file, config), gen =genome(wildcards.file, config), name=namefromfile(wildcards.file, config)) # output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), @@ -608,7 +639,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to # shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} 2>> {log} && touch {output.unmapped} && touch {output.mapped}" elif bwaalg == 'samse': rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -640,7 +671,7 @@ elif bwaalg == 'sampe': mapp1 = MAPPERBIN.split(' ')[0] shell: "{params.mapp} {params.mpara} {input.ref} {input.sai1} {input.sai2} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" elif bwaalg == 'bwasw': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -657,7 +688,7 @@ elif bwaalg == 'bwasw': shell: "{params.mapp} {params.mpara} -t {threads} {input.ref} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} - ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-rustar_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-rustar_subsnake.smk index 7ba6f00f..4b30acc4 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-rustar_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-rustar_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -512,7 +543,7 @@ rule generate_index: pref = PREFIX, lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir TMP/rustar_generate_index --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf TMP/rustar_generate_index" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -536,7 +567,7 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", uidx = rules.generate_index.output.uidx[0], @@ -575,7 +606,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-star_subsnake.smk b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-star_subsnake.smk index 800e9ebd..510a7109 100644 --- a/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-star_subsnake.smk +++ b/tests/golden/config_Test/smk/Test_fastqc-trimgalore-picard-star_subsnake.smk @@ -16,7 +16,7 @@ for x in cmd_subfolder: if x not in sys.path: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel='INFO' try: @@ -113,6 +113,18 @@ elif paired == 'singlecell': stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -122,6 +134,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -172,6 +186,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -192,6 +209,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -229,6 +249,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -270,6 +297,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -285,6 +314,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -513,7 +544,7 @@ rule generate_index: lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])), stmp = expand("TMP/{mape}_{unikey}/generate_index", mape=MAPPERENV, unikey=unik)[0] shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && rm -rf {params.stmp} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir {params.stmp} --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf {params.stmp}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -537,9 +568,9 @@ if paired == 'paired': tocopy = lambda wildcards, output: os.path.dirname(output.mapped) shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: - input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: r1 = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], dummy = rules.generate_index.output.idxfile, ref = REFERENCE @@ -576,7 +607,7 @@ else: container: "oras://ghcr.io/jfallmann/monsda:"+MAPPERENV+""+"-VERSION" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/tests/tool_audit/tool_manifest.yaml b/tests/tool_audit/tool_manifest.yaml index 92faaa68..b0f4f549 100644 --- a/tests/tool_audit/tool_manifest.yaml +++ b/tests/tool_audit/tool_manifest.yaml @@ -224,6 +224,26 @@ tools: source_refs: [workflows/hisat2.smk] note: "--un-conc-gz is the base --un-conc flag + '-gz' suffix, documented in the same help text." + bowtie2: + container: bowtie2 + binary: bowtie2 + invocations: + - name: help + args: ["--help"] + expect_exit: [0, 1] + expected_flags: ["--un-conc-gz", "--un-gz", "-x", "-p", "--threads", "-S"] + source_refs: [workflows/bowtie2.smk] + + bowtie2-build: + container: bowtie2 + binary: bowtie2-build + invocations: + - name: help + args: ["--help"] + expect_exit: [0, 1] + expected_flags: ["--threads"] + source_refs: [workflows/bowtie2.smk] + macs2: container: macs2 binary: macs2 diff --git a/workflows/bowtie2.nf b/workflows/bowtie2.nf new file mode 100644 index 00000000..c088704a --- /dev/null +++ b/workflows/bowtie2.nf @@ -0,0 +1,140 @@ +MAPENV = get_always('MAPPINGENV') +MAPBIN = get_always('MAPPINGBIN') +MAPIDX = get_always('MAPPINGIDX') +MAPUIDX = get_always('MAPPINGUIDX') +MAPUIDXNAME = get_always('MAPPINGUIDXNAME') +MAPREF = get_always('MAPPINGREF') +MAPREFDIR = "${workflow.workDir}/../"+get_always('MAPPINGREFDIR') +MAPANNO = get_always('MAPPINGANNO') +MAPPREFIX = get_always('MAPPINGPREFIX') ?: MAPBIN.split(' ')[0] +MAPUIDX = MAPUIDX.replace('.idx','') + +IDXPARAMS = get_always('bowtie2_params_INDEX') ?: '' +MAPPARAMS = get_always('bowtie2_params_MAP') ?: '' + +// NOTE: unlike hisat2, bowtie2 has no --rna-strandness/stranded-protocol +// flag, so MAPPINGSTRANDED is intentionally unused here. + +//MAPPING PROCESSES + +process collect_tomap{ + input: + path check + + output: + path "collect.txt", emit: done + + script: + """ + echo "$check Collection successful!" > collect.txt + """ +} + +process bowtie2_idx{ + conda "$MAPENV"+".yaml" + container "oras://jfallmann/monsda:"+"$MAPENV" + cpus THREADS + cache 'lenient' + label 'big_mem' + //validExitStatus 0,1 + + publishDir "${workflow.workDir}/../" , mode: 'copyNoFollow', overwrite: true, + saveAs: {filename -> + if (filename == "bowtie2.idx") "$MAPIDX" + else if (filename.indexOf(".log") >0) "LOGS/${COMBO}/${CONDITION}/MAPPING/bowtie2/index.log" + else "$MAPUIDX" + } + + input: + path genome + + output: + path "bowtie2.idx", emit: idx + path "bowtie2_*", emit: btidx + + script: + indexbin=MAPBIN.split(' ')[0]+'-build' + gen = genome.getName() + """ + zcat $gen > tmp.fa && mkdir -p $MAPUIDXNAME && $indexbin $IDXPARAMS --threads ${task.cpus} tmp.fa $MAPUIDXNAME/$MAPPREFIX &> index.log && ln -fs $MAPUIDXNAME bowtie2.idx + """ + +} + +process bowtie2_mapping{ + conda "$MAPENV"+".yaml" + container "oras://jfallmann/monsda:"+"$MAPENV" + cpus THREADS + cache 'lenient' + label 'big_mem' + //validExitStatus 0,1 + + publishDir "${workflow.workDir}/../" , mode: 'link', + saveAs: {filename -> + if (filename.indexOf("_unmapped.fastq.gz") > 0) "UNMAPPED/${COMBO}/${CONDITION}/"+"${file(filename).getName()}" + else if (filename.indexOf(".log") >0) "LOGS/${COMBO}/${CONDITION}/MAPPING/bowtie2/${file(filename).getName()}" + else null + } + + input: + path reads + + output: + path "*.sam.gz", emit: maps + path "*fastq.gz", includeInputs:false, emit: unmapped + path "*.log", emit: logs + + script: + if (' ' in MAPBIN){ + mapbin = MAPBIN.split(' ')[1] + } else { + mapbin = MAPBIN + } + + idx = reads[0] + if (MAPPINGPAIRED == 'paired'){ + rds = sort_reads(reads[1..2]) + r1 = rds[0] + r2 = rds[1] + fn = file(r1).getSimpleName().replaceAll(/_R1(_dedup)?_trimmed$/,"") + pf = fn+"_mapped.sam" + ufo = fn+"_R1_unmapped.fastq.gz" + uft = fn+"_R2_unmapped.fastq.gz" + lf = "bowtie2_"+fn+".log" + """ + $MAPBIN $MAPPARAMS -p ${task.cpus} -x ${idx}/${MAPPREFIX} -1 $r1 -2 $r2 --un-conc-gz ${fn}.unmapped -S $pf &> $lf && gzip $pf && touch $ufo $uft && rename 's/.unmapped.([1|2]).gz/_R\$1_unmapped.fastq.gz/' ${fn}.unmapped.*.gz &>> $lf + """ + }else{ + read = reads[1] + fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"") + pf = fn+"_mapped.sam" + uf = fn+"_unmapped.fastq.gz" + lf = "bowtie2_"+fn+".log" + """ + $MAPBIN $MAPPARAMS -p ${task.cpus} -x ${idx}/${MAPPREFIX} -U $read --un-gz $uf -S $pf &> $lf && gzip $pf && touch $uf &>> $lf + """ + } +} + +workflow MAPPING{ + take: collection + + main: + + checkidx = file(MAPUIDX) + collection.filter(~/.fastq.gz/) + + if (checkidx.exists()){ + idxfile = Channel.fromPath(MAPUIDX) + bowtie2_mapping(idxfile.combine(collection)) + } + else{ + genomefile = Channel.fromPath(MAPREF) + bowtie2_idx(genomefile) + bowtie2_mapping(bowtie2_idx.out.btidx.combine(collection)) + } + + emit: + mapped = bowtie2_mapping.out.maps + logs = bowtie2_mapping.out.logs +} diff --git a/workflows/bowtie2.smk b/workflows/bowtie2.smk new file mode 100644 index 00000000..fcab0a10 --- /dev/null +++ b/workflows/bowtie2.smk @@ -0,0 +1,65 @@ +MAPPERBIN, MAPPERENV = env_bin_from_config(config,'MAPPING') +keydict = sub_dict(tool_params(SAMPLES[0], None, config, 'MAPPING', MAPPERENV)['OPTIONS'], ['INDEX']) +keydict["REF"] = REFERENCE +keydict["DECOY"] = DECOY +keydict["ENV"] = MAPPERENV +unik = get_dict_hash(keydict) + +rule generate_index: + input: fa = REFERENCE + output: idx = directory(INDEX), + uidx = directory(expand("{refd}/INDICES/{mape}_{unikey}", refd=REFDIR, mape=MAPPERENV, unikey=unik)), + idxfile = expand("{refd}/INDICES/{mape}_{unikey}/{pref}", refd=REFDIR, mape=MAPPERENV, unikey=unik, pref = PREFIX), + tmp = temp(expand("TMP/{mape}/ref.fa", mape=MAPPERENV)) + log: expand("LOGS/{sets}/MAPPING/{mape}/idx.log", sets=SETS, mape=MAPPERENV) + conda: ""+MAPPERENV+".yaml" + container: "oras://jfallmann/monsda:"+MAPPERENV+"" + threads: MAXTHREAD + params: indexer = MAPPERBIN.split(' ')[0], + ipara = lambda wildcards, input: tool_params(SAMPLES[0], None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('INDEX', ""), + lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])), + pref = PREFIX + shell: "if [[ -f \"{output.idxfile}\" ]]; then ln -fs {params.lnkidx} {output.idx} && mkdir -p {output.uidx} && touch {output.tmp} {output.idxfile} && echo \"Found bowtie2 index, continue with mapping\" ; else zcat {input.fa} > {output.tmp} && mkdir -p {output.uidx} ;{params.indexer}-build {params.ipara} --threads {threads} {output.tmp} {output.uidx}/{params.pref} 2> {log} && ln -fs {params.lnkidx} {output.idx} && touch {output.uidx} && touch {output.idxfile};fi" + +# NOTE: unlike hisat2, bowtie2 has no --rna-strandness/stranded-protocol flag, +# so MAPPING_STRANDED is intentionally unused here. + +if MAPPING_PAIRED == 'paired': + rule mapping: + input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", + r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", + uidx = rules.generate_index.output.uidx[0], + dummy = rules.generate_index.output.idxfile, + ref = REFERENCE + output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), + unmapped_r1 = "UNMAPPED/{combo}/{file}_R1_unmapped.fastq.gz", + unmapped_r2 = "UNMAPPED/{combo}/{file}_R2_unmapped.fastq.gz" + log: "LOGS/{combo}/{file}/MAPPING/bowtie2/mapping.log" + conda: ""+MAPPERENV+".yaml" + container: "oras://jfallmann/monsda:"+MAPPERENV+"" + threads: MAXTHREAD + params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), + mapp=MAPPERBIN, + um = lambda wildcards, output: output.unmapped_r1.replace('_R1_unmapped.fastq.gz', '.unmapped.gz'), + umn = lambda wildcards, output: output.unmapped_r1.replace('_R1_unmapped.fastq.gz', ''), + muz = lambda wildcards, output: output.mapped.replace('.gz', ''), + pref = PREFIX + shell: "set +euo pipefail; {params.mapp} {params.mpara} -p {threads} -x {input.uidx}/{params.pref} -1 {input.r1} -2 {input.r2} --un-conc-gz {params.umn}.unmapped -S {params.muz} &> {log} && gzip {params.muz} && touch {params.umn}.unmapped.1.gz {params.umn}.unmapped.2.gz; rename 's/.unmapped.([1|2]).gz/_R$1_unmapped.fastq.gz/' {params.umn}.unmapped.*.gz; touch {output.unmapped_r1} {output.unmapped_r2} &>> {log}" + +else: + rule mapping: + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), + uidx = rules.generate_index.output.uidx[0], + dummy = rules.generate_index.output.idxfile, + ref = REFERENCE + output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), + unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" + log: "LOGS/{combo}/{file}/MAPPING/bowtie2/mapping.log" + conda: ""+MAPPERENV+".yaml" + container: "oras://jfallmann/monsda:"+MAPPERENV+"" + threads: MAXTHREAD + params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), + mapp=MAPPERBIN, + muz = lambda wildcards, output: output.mapped.replace('.gz', ''), + pref = PREFIX + shell: "set +euo pipefail; {params.mapp} {params.mpara} -p {threads} -x {input.uidx}/{params.pref} -U {input.query} --un-gz {output.unmapped} -S {params.muz} &> {log} && gzip {params.muz} && touch {output.unmapped}" diff --git a/workflows/bwa.nf b/workflows/bwa.nf index fc8aa40f..a58cd998 100644 --- a/workflows/bwa.nf +++ b/workflows/bwa.nf @@ -86,7 +86,7 @@ process bwa_mapping{ script: idx = reads[0] - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/bwa.smk b/workflows/bwa.smk index 8bfebc92..c99f0daa 100644 --- a/workflows/bwa.smk +++ b/workflows/bwa.smk @@ -22,7 +22,7 @@ rule generate_index: bwaalg = MAPPERBIN.split(' ')[1] if bwaalg == 'mem': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -42,7 +42,7 @@ if bwaalg == 'mem': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -57,7 +57,7 @@ if bwaalg == 'mem': shell: "{params.mapp} {params.mpara} -t {threads} {input.uidx} {input.query} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to continue the workflow - if paired == 'paired': # handled like sampe + if MAPPING_PAIRED == 'paired': # handled like sampe rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -94,7 +94,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to else: #handled like samse rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -109,7 +109,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to shell: "{params.mapp1} aln {params.mpara} -t {threads} {input.ref} {input.query} > {output.sai} && {params.mapp} {params.mpara} {input.ref} {output.sai} {input.query} 2> {log} | tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" ### FOR LATER IF WE EVER NEED aln MODE # rule mapping: -# input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", +# input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), # #index = lambda wildcards: expand(rules.generate_index.output.idx, ref=REFERENCE, dir=source_from_sample(wildcards.file, config), gen=genome(wildcards.file, config), name=namefromfile(wildcards.file, config), mape=MAPPERENV, extension=check_tool_params(wildcards.file, None, config, 'MAPPING', 2)), # ref = lambda wildcards: expand(rules.generate_index.input.fa, ref=REFERENCE, dir = source_from_sample(wildcards.file, config), gen =genome(wildcards.file, config), name=namefromfile(wildcards.file, config)) # output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), @@ -124,7 +124,7 @@ elif bwaalg == 'aln': # not supported as stand alone as we need mappign files to elif bwaalg == 'samse': rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: sai = report("MAPPED/{combo}/{file}_mapped.sai", category="MAPPING"), mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), @@ -158,7 +158,7 @@ elif bwaalg == 'sampe': shell: "{params.mapp} {params.mpara} {input.ref} {input.sai1} {input.sai2} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" elif bwaalg == 'bwasw': - if paired == 'paired': + if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -176,7 +176,7 @@ elif bwaalg == 'bwasw': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/workflows/bwa2.nf b/workflows/bwa2.nf index 04e3e403..a5c0a084 100644 --- a/workflows/bwa2.nf +++ b/workflows/bwa2.nf @@ -86,7 +86,7 @@ process bwa2_mapping{ script: idx = reads[0] - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/bwa2.smk b/workflows/bwa2.smk index 79bbb9e6..be8f6a52 100644 --- a/workflows/bwa2.smk +++ b/workflows/bwa2.smk @@ -19,7 +19,7 @@ rule generate_index: tolink = lambda wildcards, output: str(os.path.abspath(str.join(os.sep, str(output.idx).split(os.sep)[:-1]))) shell: "if [[ -f \"{output.uidx}\\/*\" ]]; then ln -fs {params.linkidx} {output.idx} && touch {output.uidx} && echo \"Found bwa index, continue with mapping\" ; else {params.indexer} index -p {output.uidx} {params.ipara} {input.ref} 2> {log} && ln -fs {params.linkidx} {output.idx} && touch {output.uidx};fi" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -39,7 +39,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), diff --git a/workflows/bwameth.nf b/workflows/bwameth.nf index bfaaf07e..e5f118f4 100644 --- a/workflows/bwameth.nf +++ b/workflows/bwameth.nf @@ -88,7 +88,7 @@ process bwameth_mapping{ script: idx = reads[0] - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/bwameth.smk b/workflows/bwameth.smk index ab6e3034..dc1aa3ab 100644 --- a/workflows/bwameth.smk +++ b/workflows/bwameth.smk @@ -18,7 +18,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(str.join(os.sep, str(output.uidx[0]).split(os.sep)[:-1]))) if PREFIX != '' else str(os.path.abspath(str(output.uidx[0]))), shell: "if [[ -f \"{output.uidx}\" ]]; then ln -fs {params.linkidx} {output.idx} && touch {output.uidx} && echo \"Found bwa index, continue with mapping\";else zcat {input.ref} > {output.uidx} && {params.indexer} {output.uidx} {params.ipara} 2> {log} && ln -fs {params.linkidx} {output.idx};fi" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -35,7 +35,7 @@ if paired == 'paired': shell: "{params.mapp} {params.mpara} --threads {threads} --reference {input.uidx} {input.r1} {input.r2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/workflows/countreads.nf b/workflows/countreads.nf index 4b319cac..120c3bf5 100644 --- a/workflows/countreads.nf +++ b/workflows/countreads.nf @@ -212,15 +212,15 @@ process featurecount{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (COUNTINGPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (COUNTINGSTRANDED == 'fr' || COUNTINGSTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (COUNTINGSTRANDED == 'rf' || COUNTINGSTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/countreads.smk b/workflows/countreads.smk index 8e149b1f..e6c913f7 100644 --- a/workflows/countreads.smk +++ b/workflows/countreads.smk @@ -121,8 +121,8 @@ rule featurecount: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "COUNTING", COUNTENV)['OPTIONS'].get('COUNT', "")+' -t '+wildcards.feat+' -g '+config['COUNTING']['FEATURES'][wildcards.feat], - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if COUNTING_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if COUNTING_STRANDED == 'fr' else '-s 2' if COUNTING_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(gzip -cdfq {params.anno}) -o {output.t} {input.s} 2> {log} && head -n2 {output.t} |gzip > {output.cts} && export LC_ALL=C; tail -n+3 {output.t}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.cts} && mv {output.t}.summary {output.cts}.summary && cp -f {output.cts}.summary $(dirname {log})/" @@ -137,8 +137,8 @@ rule featurecount_unique: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "COUNTING", COUNTENV)['OPTIONS'].get('COUNT', "")+' -t '+wildcards.feat+' -g '+config['COUNTING']['FEATURES'][wildcards.feat], - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if COUNTING_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if COUNTING_STRANDED == 'fr' else '-s 2' if COUNTING_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(gzip -cdfq {params.anno}) -o {output.t} {input.u} 2> {log} && head -n2 {output.t} |gzip > {output.cts} && export LC_ALL=C; tail -n+3 {output.t}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.cts} && mv {output.t}.summary {output.cts}.summary && cp -f {output.cts}.summary $(dirname {log})/" @@ -153,8 +153,8 @@ rule featurecount_dedup: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "COUNTING", COUNTENV)['OPTIONS'].get('COUNT', "")+' -t '+wildcards.feat+' -g '+config['COUNTING']['FEATURES'][wildcards.feat], - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if COUNTING_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if COUNTING_STRANDED == 'fr' else '-s 2' if COUNTING_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(gzip -cdfq {params.anno}) -o {output.t} {input.s} 2> {log} && head -n2 {output.t} |gzip > {output.cts} && export LC_ALL=C; tail -n+3 {output.t}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.cts} && mv {output.t}.summary {output.cts}.summary && cp -f {output.cts}.summary $(dirname {log})/" @@ -169,8 +169,8 @@ rule featurecount_unique_dedup: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "COUNTING", COUNTENV)['OPTIONS'].get('COUNT', "")+' -t '+wildcards.feat+' -g '+config['COUNTING']['FEATURES'][wildcards.feat], - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if COUNTING_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if COUNTING_STRANDED == 'fr' else '-s 2' if COUNTING_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(gzip -cdfq {params.anno}) -o {output.t} {input.u} 2> {log} && head -n2 {output.t} |gzip > {output.cts} && export LC_ALL=C; tail -n+3 {output.t}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.cts} && mv {output.t}.summary {output.cts}.summary && cp -f {output.cts}.summary $(dirname {log})/" diff --git a/workflows/deseq2_DE.nf b/workflows/deseq2_DE.nf index 3f1e9c13..d3a5b133 100644 --- a/workflows/deseq2_DE.nf +++ b/workflows/deseq2_DE.nf @@ -79,15 +79,15 @@ process featurecount_deseq{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DEPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DESTRANDED == 'fr' || DESTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DESTRANDED == 'rf' || DESTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/deseq2_DE.smk b/workflows/deseq2_DE.smk index 2d48f6da..28474477 100644 --- a/workflows/deseq2_DE.smk +++ b/workflows/deseq2_DE.smk @@ -53,8 +53,8 @@ if COUNTTOOL == 'featureCounts': threads: MAXTHREAD params: countb = COUNTBIN, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DE", DEENV.split('_')[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if DE_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if DE_STRANDED == 'fr' else '-s 2' if DE_STRANDED == 'rf' else '', sortmem = get_sortmem, prefix = lambda wildcards: "DE/"+combo+"/Featurecounts/"+wildcards.file shell: "mkdir -p $(dirname {params.prefix}); {params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a {input.anno} -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/dexseq_DEU.nf b/workflows/dexseq_DEU.nf index 6d01b7a6..302e5f0e 100644 --- a/workflows/dexseq_DEU.nf +++ b/workflows/dexseq_DEU.nf @@ -46,9 +46,9 @@ process prepare_deu_annotation{ da = fn+"_dexseqflat.gtf.gz" ol = "featurecount_dexseq_annotation.log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DEUSTRANDED == 'fr' || DEUSTRANDED == 'ISF'){ stranded = '-s' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DEUSTRANDED == 'rf' || DEUSTRANDED == 'ISR'){ stranded = '-s' }else{ stranded = '' @@ -89,15 +89,15 @@ process featurecount_dexseq{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DEUPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DEUSTRANDED == 'fr' || DEUSTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DEUSTRANDED == 'rf' || DEUSTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/dexseq_DEU.smk b/workflows/dexseq_DEU.smk index 00fc830b..b7eaf361 100644 --- a/workflows/dexseq_DEU.smk +++ b/workflows/dexseq_DEU.smk @@ -35,7 +35,7 @@ rule prepare_deu_annotation: container: "oras://jfallmann/monsda:"+COUNTENV+"" threads: MAXTHREAD params: bins = BINS, - countstrand = lambda x: '-s' if stranded == 'fr' or stranded == 'rf' else '' + countstrand = lambda x: '-s' if DEU_STRANDED == 'fr' or DEU_STRANDED == 'rf' else '' shell: "{params.bins}/Analysis/DEU/prepare_deu_annotation.py -f {output.countgtf} {params.countstrand} <(gzip -cdfq {input.anno}) {output.deugtf} 2>> {log}" rule featurecount_unique: @@ -52,9 +52,9 @@ rule featurecount_unique: threads: MAXTHREAD params: countb = COUNTBIN, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DEU", DEUENV.split("_")[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', + paired = lambda x: '-p' if DEU_PAIRED == 'paired' else '', bins = BINS, - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + stranded = lambda x: '-s 1' if DEU_STRANDED == 'fr' else '-s 2' if DEU_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(zcat {input.countgtf}) -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k2,2 -k3,3n -k4,4n -k1,1 -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/dexseq_DTU.nf b/workflows/dexseq_DTU.nf index 7e268524..a3c05322 100644 --- a/workflows/dexseq_DTU.nf +++ b/workflows/dexseq_DTU.nf @@ -160,7 +160,7 @@ workflow DTU{ if (checkidx.exists()){ idxfile = Channel.fromPath(DTUUIDX) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -169,7 +169,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(DTUREF) salmon_idx(genomefile) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/workflows/diego_DAS.nf b/workflows/diego_DAS.nf index b6b06359..420778af 100644 --- a/workflows/diego_DAS.nf +++ b/workflows/diego_DAS.nf @@ -52,15 +52,15 @@ process featurecount_diego{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DASPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DASSTRANDED == 'fr' || DASSTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DASSTRANDED == 'rf' || DASSTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/diego_DAS.smk b/workflows/diego_DAS.smk index 5771d417..fdaf9e84 100644 --- a/workflows/diego_DAS.smk +++ b/workflows/diego_DAS.smk @@ -30,8 +30,8 @@ rule featurecount_unique: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DAS", DASENV.split('_')[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if DAS_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if DAS_STRANDED == 'fr' else '-s 2' if DAS_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(zcat {params.anno}) -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/drimseq_DTU.nf b/workflows/drimseq_DTU.nf index bad83c5e..51cb06c6 100644 --- a/workflows/drimseq_DTU.nf +++ b/workflows/drimseq_DTU.nf @@ -161,7 +161,7 @@ workflow DTU{ if (checkidx.exists()){ idxfile = Channel.fromPath(DTUUIDX) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -170,7 +170,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(DTUREF) salmon_idx(genomefile) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/workflows/dtu_base.nf b/workflows/dtu_base.nf index 8ce085b7..b47da8b5 100644 --- a/workflows/dtu_base.nf +++ b/workflows/dtu_base.nf @@ -23,10 +23,10 @@ process salmon_quant{ script: idx = reads[0] - if (PAIRED == 'paired'){ - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DTUPAIRED == 'paired'){ + if (DTUSTRANDED == 'fr' || DTUSTRANDED == 'ISF'){ stranded = '-l ISF' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DTUSTRANDED == 'rf' || DTUSTRANDED == 'ISR'){ stranded = '-l ISR' }else{ stranded = '-l IU' @@ -43,9 +43,9 @@ process salmon_quant{ $COUNTBIN $COUNTPARAMS quant -p ${task.cpus} -i $idx $stranded -o $fn -1 $r1 -2 $r2 &>> $lf && gzip $of && ln -fs $oz $ol """ }else{ - if (STRANDED == 'fr' || STRANDED == 'SF'){ + if (DTUSTRANDED == 'fr' || DTUSTRANDED == 'SF'){ stranded = '-l SF' - }else if (STRANDED == 'rf' || STRANDED == 'SR'){ + }else if (DTUSTRANDED == 'rf' || DTUSTRANDED == 'SR'){ stranded = '-l SR' }else{ stranded = '-l U' diff --git a/workflows/dtu_base.smk b/workflows/dtu_base.smk index 61142ff2..986dd64d 100644 --- a/workflows/dtu_base.smk +++ b/workflows/dtu_base.smk @@ -13,7 +13,7 @@ if COUNTTOOL == 'salmon': linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "set +euo pipefail; {params.mapp} index {params.ipara} {params.decoy} -p {threads} -t {input.fa} -i {output.uidx} &>> {log} && ln -fs {params.linkidx} {output.idx}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule simulate_trim: input: r1 = lambda wildcards: "FASTQ/{rawfile}_R1.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R1_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file), r2 = lambda wildcards: "FASTQ/{rawfile}_R2.fastq.gz".format(rawfile=[x for x in SAMPLES if x.split(os.sep)[-1] in wildcards.file][0]) if not usededup else "DEDUP_FASTQ/{scombo}/{file}_R2_dedup.fastq.gz".format(scombo=scombo, file=wildcards.file) @@ -32,7 +32,7 @@ if COUNTTOOL == 'salmon': params: filetolink = lambda w, input: "{r}".format(r=os.path.abspath(input.r1)) shell: "ln -s {params.filetolink} {output.r1}" - if paired == 'paired': + if DTU_PAIRED == 'paired': rule mapping: input: r1 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R1_trimmed.fastq.gz", scombo=scombo), r2 = expand("TRIMMED_FASTQ/{scombo}/{{file}}_R2_trimmed.fastq.gz", scombo=scombo), @@ -46,7 +46,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l ISF' if (stranded == 'fr' or stranded == 'ISF') else '-l ISR' if (stranded == 'rf' or stranded == 'ISR') else '-l IU', + stranded = lambda x: '-l ISF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'ISF') else '-l ISR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'ISR') else '-l IU', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -1 {input.r1} -2 {input.r2} &>> {log} && gzip {output.ctsdir}/quant.sf && ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" @@ -63,7 +63,7 @@ if COUNTTOOL == 'salmon': threads: MAXTHREAD params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""), mapp=COUNTBIN, - stranded = lambda x: '-l SF' if (stranded == 'fr' or stranded == 'SF') else '-l SR' if (stranded == 'rf' or stranded == 'SR') else '-l U', + stranded = lambda x: '-l SF' if (DTU_STRANDED == 'fr' or DTU_STRANDED == 'SF') else '-l SR' if (DTU_STRANDED == 'rf' or DTU_STRANDED == 'SR') else '-l U', linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)) shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.index} {params.stranded} {params.cpara} -o {output.ctsdir} -r {input.r1} &>> {log} && gzip {output.ctsdir}/quant.sf ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}" diff --git a/workflows/edger_DAS.nf b/workflows/edger_DAS.nf index 6ccc2099..fce07da6 100644 --- a/workflows/edger_DAS.nf +++ b/workflows/edger_DAS.nf @@ -50,15 +50,15 @@ process featurecount_edger{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DASPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DASSTRANDED == 'fr' || DASSTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DASSTRANDED == 'rf' || DASSTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/edger_DAS.smk b/workflows/edger_DAS.smk index 2a23d6e7..f2d6e559 100644 --- a/workflows/edger_DAS.smk +++ b/workflows/edger_DAS.smk @@ -36,8 +36,8 @@ rule featurecount_unique: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DAS", DASENV.split('_')[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if DAS_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if DAS_STRANDED == 'fr' else '-s 2' if DAS_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(zcat {params.anno}) -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/edger_DE.nf b/workflows/edger_DE.nf index 5966e26b..d4edf384 100644 --- a/workflows/edger_DE.nf +++ b/workflows/edger_DE.nf @@ -79,15 +79,15 @@ process featurecount_edger{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DEPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DESTRANDED == 'fr' || DESTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DESTRANDED == 'rf' || DESTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/edger_DE.smk b/workflows/edger_DE.smk index 9ae8d63b..f818240e 100644 --- a/workflows/edger_DE.smk +++ b/workflows/edger_DE.smk @@ -53,8 +53,8 @@ if COUNTTOOL == 'featureCounts': threads: MAXTHREAD params: countb = COUNTBIN, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DE", DEENV.split('_')[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if DE_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if DE_STRANDED == 'fr' else '-s 2' if DE_STRANDED == 'rf' else '', sortmem = get_sortmem, prefix = lambda wildcards: "DE/"+combo+"/Featurecounts/"+wildcards.file shell: "mkdir -p $(dirname {params.prefix}); {params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a {input.anno} -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/edger_DEU.nf b/workflows/edger_DEU.nf index 9ea9c4b8..e80a000f 100644 --- a/workflows/edger_DEU.nf +++ b/workflows/edger_DEU.nf @@ -50,15 +50,15 @@ process featurecount_edger{ os = fn+".counts.summary" ol = fn+".log" def sortmem = Math.ceil(task.memory.giga as double) as int - if (PAIRED == 'paired'){ + if (DEUPAIRED == 'paired'){ pair = "-p" } else{ pair= "" } - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (DEUSTRANDED == 'fr' || DEUSTRANDED == 'ISF'){ stranded = '-s 1' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (DEUSTRANDED == 'rf' || DEUSTRANDED == 'ISR'){ stranded = '-s 2' }else{ stranded = '' diff --git a/workflows/edger_DEU.smk b/workflows/edger_DEU.smk index e2983ca9..47862d5c 100644 --- a/workflows/edger_DEU.smk +++ b/workflows/edger_DEU.smk @@ -36,8 +36,8 @@ rule featurecount_unique: params: countb = COUNTBIN, anno = ANNOTATION, cpara = lambda wildcards: tool_params(wildcards.file, None, config, "DEU", DEUENV.split('_')[0])['OPTIONS'].get('COUNT', ""), - paired = lambda x: '-p' if paired == 'paired' else '', - stranded = lambda x: '-s 1' if stranded == 'fr' else '-s 2' if stranded == 'rf' else '', + paired = lambda x: '-p' if DEU_PAIRED == 'paired' else '', + stranded = lambda x: '-s 1' if DEU_STRANDED == 'fr' else '-s 2' if DEU_STRANDED == 'rf' else '', sortmem = get_sortmem shell: "{params.countb} -T {threads} {params.cpara} {params.paired} {params.stranded} -a <(gzip -cdfq {params.anno}) -o {output.tmp} {input.reads} 2> {log} && head -n2 {output.tmp} |gzip > {output.tmph} && export LC_ALL=C; tail -n+3 {output.tmp}|sort --parallel={threads} -S {params.sortmem}G -T TMP -k1,1 -k2,2n -k3,3n -u |gzip >> {output.tmpc} && zcat {output.tmph} {output.tmpc} |gzip > {output.cts} && mv {output.tmp}.summary {output.cts}.summary" diff --git a/workflows/header.nf b/workflows/header.nf index fedff093..1f938216 100644 --- a/workflows/header.nf +++ b/workflows/header.nf @@ -64,6 +64,32 @@ PAIRED = get_always('PAIRED') ?: null RUNDEDUP = get_always('RUNDEDUP') ?: null PREDEDUP = get_always('PREDEDUP') ?: null STRANDED = get_always('STRANDED') ?: null +// Per-step pairedness/strandedness, default to the global values above and +// overridden by the workflow-specific MONSDA config key SEQUENCING, e.g. a +// quantseq MAPPING step can run 'single' while TRIMMING/DEDUP still see the +// global 'paired' mode needed for UMI extraction from read2. +MAPPINGPAIRED = get_always('MAPPINGPAIRED') ?: PAIRED +MAPPINGSTRANDED = get_always('MAPPINGSTRANDED') ?: STRANDED +PEAKPAIRED = get_always('PEAKPAIRED') ?: MAPPINGPAIRED +PEAKSTRANDED = get_always('PEAKSTRANDED') ?: MAPPINGSTRANDED +TRACKSPAIRED = get_always('TRACKSPAIRED') ?: MAPPINGPAIRED +TRACKSSTRANDED = get_always('TRACKSSTRANDED') ?: MAPPINGSTRANDED +COUNTINGPAIRED = get_always('COUNTINGPAIRED') ?: MAPPINGPAIRED +COUNTINGSTRANDED = get_always('COUNTINGSTRANDED') ?: MAPPINGSTRANDED +CIRCSPAIRED = get_always('CIRCSPAIRED') ?: MAPPINGPAIRED +CIRCSSTRANDED = get_always('CIRCSSTRANDED') ?: MAPPINGSTRANDED +FUSIONSPAIRED = get_always('FUSIONSPAIRED') ?: MAPPINGPAIRED +FUSIONSSTRANDED = get_always('FUSIONSSTRANDED') ?: MAPPINGSTRANDED +DEPAIRED = get_always('DEPAIRED') ?: MAPPINGPAIRED +DESTRANDED = get_always('DESTRANDED') ?: MAPPINGSTRANDED +DEUPAIRED = get_always('DEUPAIRED') ?: MAPPINGPAIRED +DEUSTRANDED = get_always('DEUSTRANDED') ?: MAPPINGSTRANDED +DASPAIRED = get_always('DASPAIRED') ?: MAPPINGPAIRED +DASSTRANDED = get_always('DASSTRANDED') ?: MAPPINGSTRANDED +// DTU's default salmon path quantifies directly from TRIMMED_FASTQ, independent +// of the MAPPING step, so it falls back to the global layout, not MAPPING's +DTUPAIRED = get_always('DTUPAIRED') ?: PAIRED +DTUSTRANDED = get_always('DTUSTRANDED') ?: STRANDED IP = get_always('IP') ?: null CONDITION = get_always('CONDITION') ?: null COMBO = get_always('COMBO') ?: '' diff --git a/workflows/header.smk b/workflows/header.smk index 3fec00b6..0d0a4265 100644 --- a/workflows/header.smk +++ b/workflows/header.smk @@ -18,7 +18,7 @@ for x in cmd_subfolder: sys.path.insert(0, x) from MONSDA.Logger import makelogdir, setup_logger -from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters +from MONSDA.Params import checkpaired, checkstranded, check_IP, sampleslong, basecall_samples, download_samples, env_bin_from_config, tool_params, samplecond, conditiononly, comparable_as_string, get_cutoff_as_string, get_reps, get_diego_samples, get_diego_groups, get_pairing, set_pairing, get_samples_postprocess, fixRunParameters, resolve_sequencing from MONSDA.Utils import get_dict_hash, sub_dict, makeoutdir, keysets_from_dict, dict_inst loglevel="INFO" @@ -136,6 +136,20 @@ stranded = checkstranded([SAMPLES[0]], config) if stranded != '': log.info('RUNNING SNAKEMAKE WITH STRANDEDNESS '+str(stranded)) + +# Per-step pairedness/strandedness, default to the global values above and +# overridden below wherever a workflow step defines its own SEQUENCING key. +MAPPING_PAIRED, MAPPING_STRANDED = paired, stranded +PEAKS_PAIRED, PEAKS_STRANDED = paired, stranded +TRACKS_PAIRED, TRACKS_STRANDED = paired, stranded +COUNTING_PAIRED, COUNTING_STRANDED = paired, stranded +CIRCS_PAIRED, CIRCS_STRANDED = paired, stranded +FUSIONS_PAIRED, FUSIONS_STRANDED = paired, stranded +DE_PAIRED, DE_STRANDED = paired, stranded +DEU_PAIRED, DEU_STRANDED = paired, stranded +DAS_PAIRED, DAS_STRANDED = paired, stranded +DTU_PAIRED, DTU_STRANDED = paired, stranded + # MAPPING Variables if 'MAPPING' in config: MAPPERBIN, MAPPERENV = env_bin_from_config(config, 'MAPPING') @@ -145,6 +159,8 @@ if 'MAPPING' in config: REF = MAPCONF[MAPPERENV].get("REFERENCE", MAPCONF.get("REFERENCE", config['MAPPING'].get("REFERENCE"))) MANNO = MAPCONF[MAPPERENV].get("ANNOTATION", MAPCONF.get("ANNOTATION", config['MAPPING'].get("ANNOTATION"))) MDECOY = MAPCONF[MAPPERENV].get("DECOY", MAPCONF.get("DECOY", config['MAPPING'].get("DECOY"))) + MSEQ = MAPCONF[MAPPERENV].get("SEQUENCING", MAPCONF.get("SEQUENCING", config['MAPPING'].get("SEQUENCING"))) + MAPPING_PAIRED, MAPPING_STRANDED = resolve_sequencing(MSEQ, paired, stranded) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -197,6 +213,9 @@ if 'PEAKS' in config: PEAKBIN, PEAKENV = env_bin_from_config(config, 'PEAKS') REF = PEAKCONF.get('REFERENCE', PEAKCONF[PEAKENV].get('REFERENCE', config['PEAKS'].get('REFERENCE'))) ANNOPEAK = PEAKCONF.get('ANNOTATION', PEAKCONF[PEAKENV].get('ANNOTATION', config['PEAKS'].get('ANNOTATION'))) + PSEQ = PEAKCONF.get('SEQUENCING', PEAKCONF[PEAKENV].get('SEQUENCING', config['PEAKS'].get('SEQUENCING'))) + # downstream steps consume the MAPPING output, so default to its pairedness + PEAKS_PAIRED, PEAKS_STRANDED = resolve_sequencing(PSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -219,6 +238,9 @@ for x in ['TRACKS', 'COUNTING']: REF = XCONF[XENV].get('REFERENCE', XCONF.get('REFERENCE', config[x].get('REFERENCE'))) XANNO = XCONF[XENV].get('ANNOTATION', XCONF.get('ANNOTATION', config[x].get('ANNOTATION'))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + # TRACKS/COUNTING consume the MAPPING output, so default to its pairedness + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -258,6 +280,13 @@ for x in ['DE', 'DEU', 'DAS', 'DTU']: REF = XCONF[XENV].get("REFERENCE", XCONF.get("REFERENCE", config[x].get("REFERENCE"))) XANNO = XCONF[XENV].get("ANNOTATION", XCONF.get("ANNOTATION", config[x].get("ANNOTATION"))) XDECOY = XCONF[XENV].get("DECOY", XCONF.get("DECOY", config[x].get("DECOY"))) + XSEQ = XCONF[XENV].get("SEQUENCING", XCONF.get("SEQUENCING", config[x].get("SEQUENCING"))) + if x == 'DTU': + # DTU's default salmon path quantifies directly from TRIMMED_FASTQ, + # independent of the MAPPING step, so fall back to the global layout + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, paired, stranded) + else: + globals()[x+'_PAIRED'], globals()[x+'_STRANDED'] = resolve_sequencing(XSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if XANNO and XANNO != '': ANNOTATION = XANNO else: @@ -300,6 +329,8 @@ if 'CIRCS' in config: log.debug(logid+'CIRCCONFIG: '+str(SETUP)+'\t'+str(CIRCCONF)) REF = CIRCCONF.get('REFERENCE', CIRCCONF[XENV].get('REFERENCE', config['CIRCS'].get('REFERENCE'))) XANNO = CIRCCONF.get('ANNOTATION', CIRCCONF[XENV].get('ANNOTATION', config['CIRCS'].get('ANNOTATION'))) + CSEQ = CIRCCONF.get('SEQUENCING', CIRCCONF[XENV].get('SEQUENCING', config['CIRCS'].get('SEQUENCING'))) + CIRCS_PAIRED, CIRCS_STRANDED = resolve_sequencing(CSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) @@ -316,6 +347,8 @@ if 'FUSIONS' in config: log.debug(logid+'FUSCONFIG: '+str(SETUP)+'\t'+str(FUSCONF)) REF = FUSCONF.get('REFERENCE', FUSCONF[XENV].get('REFERENCE', config['FUSIONS'].get('REFERENCE'))) XANNO = FUSCONF.get('ANNOTATION', FUSCONF[XENV].get('ANNOTATION', config['FUSIONS'].get('ANNOTATION'))) + FSEQ = FUSCONF.get('SEQUENCING', FUSCONF[XENV].get('SEQUENCING', config['FUSIONS'].get('SEQUENCING'))) + FUSIONS_PAIRED, FUSIONS_STRANDED = resolve_sequencing(FSEQ, MAPPING_PAIRED, MAPPING_STRANDED) if REF: REFERENCE = REF REFDIR = str(os.path.dirname(REFERENCE)) diff --git a/workflows/hisat2.nf b/workflows/hisat2.nf index 1d0ecb01..246262bd 100644 --- a/workflows/hisat2.nf +++ b/workflows/hisat2.nf @@ -92,13 +92,13 @@ process hisat2_mapping{ } idx = reads[0] - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) - if (STRANDED == 'fr'){ + if (MAPPINGSTRANDED == 'fr'){ stranded = '--rna-strandness FR --fr' - }else if (STRANDED == 'rf'){ + }else if (MAPPINGSTRANDED == 'rf'){ stranded = '--rna-strandness RF --rf' - }else if (STRANDED == 'ff'){ + }else if (MAPPINGSTRANDED == 'ff'){ stranded = '--ff' }else{ stranded = '' @@ -115,9 +115,9 @@ process hisat2_mapping{ $MAPBIN $MAPPARAMS $stranded -p ${task.cpus} -x ${idx}/${MAPPREFIX} -1 $r1 -2 $r2 --un-conc-gz ${fn}.unmapped -S $pf &> $lf && gzip $pf && touch $ufo $uft && rename 's/.unmapped.([1|2]).gz/_R\$1_unmapped.fastq.gz/' ${fn}.unmapped.*.gz &>> $lf """ }else{ - if (STRANDED == 'fr'){ + if (MAPPINGSTRANDED == 'fr'){ stranded = '--rna-strandness F' - }else if (STRANDED == 'rf'){ + }else if (MAPPINGSTRANDED == 'rf'){ stranded = '--rna-strandness R' }else{ stranded = '' diff --git a/workflows/hisat2.smk b/workflows/hisat2.smk index 6719e04f..c6979e4b 100644 --- a/workflows/hisat2.smk +++ b/workflows/hisat2.smk @@ -23,7 +23,7 @@ rule generate_index: hs2alg = MAPPERBIN.split(' ')[1] if ' ' in MAPPERBIN else MAPPERBIN -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -40,7 +40,7 @@ if paired == 'paired': threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), mapp=MAPPERBIN, - stranded = lambda x: '--rna-strandness FR --fr' if stranded == 'fr' else '--rna-strandness RF --rf' if stranded == 'rf' else '--ff' if stranded == 'ff' else '', + stranded = lambda x: '--rna-strandness FR --fr' if MAPPING_STRANDED == 'fr' else '--rna-strandness RF --rf' if MAPPING_STRANDED == 'rf' else '--ff' if MAPPING_STRANDED == 'ff' else '', um = lambda wildcards, output: output.unmapped_r1.replace('_R1_unmapped.fastq.gz', '.unmapped.gz'), umn = lambda wildcards, output: output.unmapped_r1.replace('_R1_unmapped.fastq.gz', ''), muz = lambda wildcards, output: output.mapped.replace('.gz', ''), @@ -49,7 +49,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], dummy = rules.generate_index.output.idxfile, ref = REFERENCE @@ -62,7 +62,7 @@ else: threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), mapp=MAPPERBIN, - stranded = lambda x: '--rna-strandness F' if stranded == 'fr' else '--rna-strandness R' if stranded == 'rf' else '', + stranded = lambda x: '--rna-strandness F' if MAPPING_STRANDED == 'fr' else '--rna-strandness R' if MAPPING_STRANDED == 'rf' else '', muz = lambda wildcards, output: output.mapped.replace('.gz', ''), pref = PREFIX shell: "set +euo pipefail; {params.mapp} {params.mpara} {params.stranded} -p {threads} -x {input.uidx}/{params.pref} -U {input.query} --new-summary --summary-file {output.summary} --un-gz {output.unmapped} -S {params.muz} &> {log} && gzip {params.muz} && touch {output.unmapped}" diff --git a/workflows/minimap.nf b/workflows/minimap.nf index e57465c8..ea148573 100644 --- a/workflows/minimap.nf +++ b/workflows/minimap.nf @@ -85,7 +85,7 @@ process minimap_mapping{ script: idxfile = reads[0] idx = idxfile.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/minimap.smk b/workflows/minimap.smk index 237a225c..7ca86ba5 100644 --- a/workflows/minimap.smk +++ b/workflows/minimap.smk @@ -18,7 +18,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "{params.indexer} -t {threads} -d {output.uidx} {params.ipara} {input.fa} 2> {log} && ln -s {params.linkidx} {output.idx}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: q1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", q2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -36,7 +36,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0] output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/workflows/rammap.nf b/workflows/rammap.nf index 1e4cf518..40a09a01 100644 --- a/workflows/rammap.nf +++ b/workflows/rammap.nf @@ -68,7 +68,7 @@ process rammap_mapping{ script: idxfile = reads[0] idx = idxfile.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/rammap.smk b/workflows/rammap.smk index cce02da8..a8c6ca15 100644 --- a/workflows/rammap.smk +++ b/workflows/rammap.smk @@ -19,7 +19,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "gzip -cdfq {input.fa} > {output.tmpfa} && {params.indexer} -t {threads} -d {output.uidx} {params.ipara} {output.tmpfa} 2> {log} && ln -s {params.linkidx} {output.idx}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: q1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", q2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -37,7 +37,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0] output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" diff --git a/workflows/rustar.nf b/workflows/rustar.nf index b5696276..8f2b6687 100644 --- a/workflows/rustar.nf +++ b/workflows/rustar.nf @@ -92,7 +92,7 @@ process rustar_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -105,7 +105,7 @@ process rustar_mapping{ """ } else{ - if (PAIRED != 'singlecell'){ + if (MAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -115,9 +115,9 @@ process rustar_mapping{ """ } else{ - if (STRANDED == 'fr'){ + if (MAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (STRANDED == 'rf'){ + }else if (MAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/workflows/rustar.smk b/workflows/rustar.smk index ac16fc78..fe3cb32c 100644 --- a/workflows/rustar.smk +++ b/workflows/rustar.smk @@ -23,7 +23,7 @@ rule generate_index: lnkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir TMP/rustar_generate_index --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf TMP/rustar_generate_index" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -48,7 +48,7 @@ if paired == 'paired': shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", uidx = rules.generate_index.output.uidx[0], @@ -87,7 +87,7 @@ else: container: "oras://jfallmann/monsda:"+MAPPERENV+"" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX, diff --git a/workflows/salmonalign.nf b/workflows/salmonalign.nf index 5e5f8e33..a1dc3b7b 100644 --- a/workflows/salmonalign.nf +++ b/workflows/salmonalign.nf @@ -70,11 +70,11 @@ process salmonalign_mapping{ script: idxfile = reads[0] idx = idxfile.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) - if (STRANDED == 'fr' || STRANDED == 'ISF'){ + if (MAPPINGSTRANDED == 'fr' || MAPPINGSTRANDED == 'ISF'){ stranded = '-l ISF' - }else if (STRANDED == 'rf' || STRANDED == 'ISR'){ + }else if (MAPPINGSTRANDED == 'rf' || MAPPINGSTRANDED == 'ISR'){ stranded = '-l ISR' }else{ stranded = '-l IU' @@ -90,9 +90,9 @@ process salmonalign_mapping{ $MAPBIN quant -p ${task.cpus} -i $idx $stranded $MAPPARAMS --writeMappings=/dev/stdout -o $fn -1 $r1 -2 $r2 2> $lf|tee >(samtools view -h -F 4 |gzip > $pf) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 $uf1 -2 $uf2 ) 2>> $lf &>/dev/null && touch $uf1 $uf2 2>> $lf &> /dev/null """ }else{ - if (STRANDED == 'fr' || STRANDED == 'SF'){ + if (MAPPINGSTRANDED == 'fr' || MAPPINGSTRANDED == 'SF'){ stranded = '-l SF' - }else if (STRANDED == 'rf' || STRANDED == 'SR'){ + }else if (MAPPINGSTRANDED == 'rf' || MAPPINGSTRANDED == 'SR'){ stranded = '-l SR' }else{ stranded = '-l U' diff --git a/workflows/salmonalign.smk b/workflows/salmonalign.smk index fdc2cb7c..28be5f2e 100644 --- a/workflows/salmonalign.smk +++ b/workflows/salmonalign.smk @@ -20,7 +20,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "set +euo pipefail; {params.indexer} index {params.ipara} {params.decoy} -p {threads} -t {input.fa} -i {output.uidx} &>> {log} && ln -fs {params.linkidx} {output.idx}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: q1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", q2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -34,13 +34,13 @@ if paired == 'paired': threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), mapp = MAPPERBIN, - stranded = lambda x: '-l ISF' if (stranded == 'fr' or stranded == 'ISF') else '-l ISR' if (stranded == 'rf' or stranded == 'ISR') else '-l IU', + stranded = lambda x: '-l ISF' if (MAPPING_STRANDED == 'fr' or MAPPING_STRANDED == 'ISF') else '-l ISR' if (MAPPING_STRANDED == 'rf' or MAPPING_STRANDED == 'ISR') else '-l IU', outdir = lambda wildcards: "MAPPED/"+wildcards.combo+"/"+wildcards.file+"_salmonalign" shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.uidx} {params.stranded} {params.mpara} --writeMappings=/dev/stdout -o {params.outdir} -1 {input.q1} -2 {input.q2} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} - ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0] output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), unmapped = "UNMAPPED/{combo}/{file}_unmapped.fastq.gz" @@ -50,6 +50,6 @@ else: threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), mapp = MAPPERBIN, - stranded = lambda x: '-l SF' if (stranded == 'fr' or stranded == 'SF') else '-l SR' if (stranded == 'rf' or stranded == 'SR') else '-l U', + stranded = lambda x: '-l SF' if (MAPPING_STRANDED == 'fr' or MAPPING_STRANDED == 'SF') else '-l SR' if (MAPPING_STRANDED == 'rf' or MAPPING_STRANDED == 'SR') else '-l U', outdir = lambda wildcards: "MAPPED/"+wildcards.combo+"/"+wildcards.file+"_salmonalign" shell: "set +euo pipefail; {params.mapp} quant -p {threads} -i {input.uidx} {params.stranded} {params.mpara} --writeMappings=/dev/stdout -o {params.outdir} -r {input.query} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools fastq -n - | pigz > {output.unmapped}) 2>> {log} &>/dev/null && touch {output.unmapped}" diff --git a/workflows/segemehl.nf b/workflows/segemehl.nf index 94bc508b..58d3b3a3 100644 --- a/workflows/segemehl.nf +++ b/workflows/segemehl.nf @@ -87,7 +87,7 @@ process segemehl_mapping{ gen = genome.getName() idx = idxfile.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[2..3]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/segemehl.smk b/workflows/segemehl.smk index 892b414a..9dfc8019 100644 --- a/workflows/segemehl.smk +++ b/workflows/segemehl.smk @@ -18,7 +18,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "{params.indexer} --threads {threads} {params.ipara} -d {input.fa} -x {output.uidx} &> {log} && ln -fs {params.linkidx} {output.idx}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -37,7 +37,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], fa = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), diff --git a/workflows/segemehl3.nf b/workflows/segemehl3.nf index 53dbb20d..bd9dcc3d 100644 --- a/workflows/segemehl3.nf +++ b/workflows/segemehl3.nf @@ -91,7 +91,7 @@ process segemehl3_mapping{ gen = genome.getName() idx = idxfile.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[2..3]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/segemehl3.smk b/workflows/segemehl3.smk index 11a8e7e7..952b8f58 100644 --- a/workflows/segemehl3.smk +++ b/workflows/segemehl3.smk @@ -18,7 +18,7 @@ rule generate_index: linkidx = lambda wildcards, output: str(os.path.abspath(output.uidx[0])) shell: "{params.indexer} --threads {threads} {params.ipara} -d {input.fa} -x {output.uidx} &> {log} && ln -fs {params.linkidx} {output.idx}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -41,7 +41,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], ref = REFERENCE output: mapped = temp(report("MAPPED/{combo}/{file}_mapped.sam.gz", category="MAPPING")), diff --git a/workflows/segemehl3_bisulfite.nf b/workflows/segemehl3_bisulfite.nf index 098f4842..a798688e 100644 --- a/workflows/segemehl3_bisulfite.nf +++ b/workflows/segemehl3_bisulfite.nf @@ -97,7 +97,7 @@ process segemehl3_mapping{ idx = idxfile.getName() idx2 = idxfile2.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[3..4]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/segemehl3_bisulfite.smk b/workflows/segemehl3_bisulfite.smk index 6adb9d7b..eff79085 100644 --- a/workflows/segemehl3_bisulfite.smk +++ b/workflows/segemehl3_bisulfite.smk @@ -21,7 +21,7 @@ rule generate_index: linkidx2 = lambda wildcards, output: str(os.path.abspath(output.uidx2[0])) shell: "{params.indexer} --threads {threads} {params.ipara} -d {input.fa} -x {output.uidx1} -y {output.uidx2} &> {log} && ln -fs {params.linkidx1} {output.idx1} && ln -fs {params.linkidx2} {output.idx2}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -40,7 +40,7 @@ if paired == 'paired': shell: "set +o pipefail;{params.mapp} {params.mpara} -d {input.fa} -i {input.uidx1} -j {input.uidx2} -q {input.r1} -p {input.r2} --threads {threads} 2> {log}| tee >(samtools view -h -F 4 |gzip > {output.mapped}) >(samtools view -h -f 4 |samtools collate -u -O -|samtools fastq -n -c 6 -1 {output.unmapped1} -2 {output.unmapped2} - ) 2>> {log} &>/dev/null && touch {output.unmapped1} {output.unmapped2}" else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx1 = rules.generate_index.output.uidx1[0], uidx2= rules.generate_index.output.uidx2[0], fa = REFERENCE diff --git a/workflows/segemehl_bisulfite.nf b/workflows/segemehl_bisulfite.nf index eb583cbe..9008007f 100644 --- a/workflows/segemehl_bisulfite.nf +++ b/workflows/segemehl_bisulfite.nf @@ -97,7 +97,7 @@ process segemehl_mapping{ idx = idxfile.getName() idx2 = idxfile2.getName() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[3..4]) r1 = rds[0] r2 = rds[1] diff --git a/workflows/segemehl_bisulfite.smk b/workflows/segemehl_bisulfite.smk index b702979f..fee078db 100644 --- a/workflows/segemehl_bisulfite.smk +++ b/workflows/segemehl_bisulfite.smk @@ -21,7 +21,7 @@ rule generate_index: linkidx2 = lambda wildcards, output: str(os.path.abspath(output.uidx2[0])) shell: "{params.indexer} --threads {threads} {params.ipara} -d {input.fa} -x {output.uidx1} -y {output.uidx2} &> {log} && ln -fs {params.linkidx1} {output.idx1} && ln -fs {params.linkidx2} {output.idx2}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -42,7 +42,7 @@ if paired == 'paired': else: rule mapping: - input: query = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: query = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx1 = rules.generate_index.output.uidx1[0], uidx2 = rules.generate_index.output.uidx2[0], fa = REFERENCE diff --git a/workflows/spit_DTU.nf b/workflows/spit_DTU.nf index 2ae920e5..0499af78 100644 --- a/workflows/spit_DTU.nf +++ b/workflows/spit_DTU.nf @@ -159,7 +159,7 @@ workflow DTU{ if (checkidx.exists()){ idxfile = Channel.fromPath(DTUUIDX) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(idxfile.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(idxfile.combine(trimsamples_ch.collate(1))) @@ -168,7 +168,7 @@ workflow DTU{ else{ genomefile = Channel.fromPath(DTUREF) salmon_idx(genomefile) - if (PAIRED == 'paired'){ + if (DTUPAIRED == 'paired'){ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(2))) } else{ salmon_quant(salmon_idx.out.idx.combine(trimsamples_ch.collate(1))) diff --git a/workflows/star.nf b/workflows/star.nf index 6d42b9d8..d0f96ccc 100644 --- a/workflows/star.nf +++ b/workflows/star.nf @@ -92,7 +92,7 @@ process star_mapping{ script: idx = reads[0] idxdir = idx.toRealPath() - if (PAIRED == 'paired'){ + if (MAPPINGPAIRED == 'paired'){ rds = sort_reads(reads[1..2]) r1 = rds[0] r2 = rds[1] @@ -105,7 +105,7 @@ process star_mapping{ """ } else{ - if (PAIRED != 'singlecell'){ + if (MAPPINGPAIRED != 'singlecell'){ read = reads[1] fn = file(reads[1]).getSimpleName().replaceAll(/(_dedup)?_trimmed$/,"")+"." of = fn+'Aligned.out.sam' @@ -116,9 +116,9 @@ process star_mapping{ } else{ rds = sort_reads(reads[1..2]) - if (STRANDED == 'fr'){ + if (MAPPINGSTRANDED == 'fr'){ stranded = '--soloStrand Forward' - }else if (STRANDED == 'rf'){ + }else if (MAPPINGSTRANDED == 'rf'){ stranded = '--soloStrand Reverse' }else{ stranded = '--soloStrand Unstranded' diff --git a/workflows/star.smk b/workflows/star.smk index 0f3972d4..e3e2e55d 100644 --- a/workflows/star.smk +++ b/workflows/star.smk @@ -24,7 +24,7 @@ rule generate_index: stmp = expand("TMP/{mape}_{unikey}/generate_index", mape=MAPPERENV, unikey=unik)[0] shell: "if [[ -f \"{output.idxfile}\" ]]; then touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && echo \"Found SAindex, continue with mapping\" ; else zcat {input.fa} > {output.tmpfa} && zcat {params.anno} > {output.tmpref} && mkdir -p {output.uidx} && rm -rf {params.stmp} && {params.mapp} {params.ipara} --runThreadN {threads} --runMode genomeGenerate --outFileNamePrefix {output.uidx}/{params.pref} --outTmpDir {params.stmp} --genomeDir {output.uidx} --genomeFastaFiles {output.tmpfa} &> {log} && touch {output.idxfile} && ln -fs {params.lnkidx} {output.idx} && cat {output.uidx}/*Log.out >> {log};fi && rm -rf {params.stmp}" -if paired == 'paired': +if MAPPING_PAIRED == 'paired': rule mapping: input: r1 = "TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz", r2 = "TRIMMED_FASTQ/{combo}/{file}_R2_trimmed.fastq.gz", @@ -49,9 +49,9 @@ if paired == 'paired': shell: "{params.mapp} {params.mpara} --runThreadN {threads} --genomeDir {input.uidx} --readFilesCommand zcat --readFilesIn {input.r1} {input.r2} --outFileNamePrefix {output.tmp}. --outReadsUnmapped Fastx &> {log} && gzip -c {output.tmp}.Aligned.out.sam > {output.mapped} && rm -f {output.tmp}.Aligned.out.sam 2>> {log} && touch {output.tmp}.Unmapped.out.mate1 {output.tmp}.Unmapped.out.mate2 && cat {output.tmp}.Unmapped.out.mate1 | paste - - - - |tr \"\\t\" \"\\n\" |gzip > {output.unmapped_r1} && cat {output.tmp}.Unmapped.out.mate2| paste - - - - |tr \"\\t\" \"\\n\"| gzip > {output.unmapped_r2} && rm -f {output.tmp}.Unmapped.out* && mv -f {output.tmp}*Log.out {output.log} && mv -f {output.tmp}*Log.final.out {output.log_final} && rm -f {output.tmp}*.Log.progress.out && mv {output.tmp}*.out* {params.tocopy} 2>> {log} && touch {output.tmp} " else: - if paired != 'singlecell': + if MAPPING_PAIRED != 'singlecell': rule mapping: - input: r1 = "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz", + input: r1 = ("TRIMMED_FASTQ/{combo}/{file}_R1_trimmed.fastq.gz" if (MAPPING_PAIRED != 'paired' and paired == 'paired') else "TRIMMED_FASTQ/{combo}/{file}_trimmed.fastq.gz"), uidx = rules.generate_index.output.uidx[0], dummy = rules.generate_index.output.idxfile, ref = REFERENCE @@ -88,7 +88,7 @@ else: container: "oras://jfallmann/monsda:"+MAPPERENV+"" threads: MAXTHREAD params: mpara = lambda wildcards: tool_params(wildcards.file, None, config, 'MAPPING', MAPPERENV)['OPTIONS'].get('MAP', ""), - stranded = lambda x: '--soloStrand Forward' if stranded == 'fr' else '--soloStrand Reverse' if stranded == 'rf' else '--soloStrand Unstranded', + stranded = lambda x: '--soloStrand Forward' if MAPPING_STRANDED == 'fr' else '--soloStrand Reverse' if MAPPING_STRANDED == 'rf' else '--soloStrand Unstranded', mapp = MAPPERBIN, anno = ANNOTATION, pref = PREFIX,