Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions MONSDA/Versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,21 +359,33 @@ def write_versions(config, steps, envdir, monsda_version, configfile=None, path=

rows = collect_versions(config, steps, envdir)

body = ""
if configfile:
body += "# config: {c}\n".format(c=str(configfile))
body += (
"# versions as pinned in the conda env yamls of the configured tools, not queried from the running binaries\n"
)
body += "#tool\tversion\tmultiqc_module\tsteps\n"
for row in rows:
body += "\t".join(row) + "\n"

if os.path.isfile(path):
with open(path, "r") as existing:
old_body = "".join(
line for line in existing if not line.startswith("# MONSDA ")
)
if old_body == body:
log.info(logid + "Tool versions unchanged, keeping " + str(path))
return path

with open(path, "w") as out:
out.write(
"# MONSDA {v}\t{ts}\n".format(
v=monsda_version,
ts=datetime.datetime.now().strftime("%Y%m%d_%H_%M_%S"),
)
)
if configfile:
out.write("# config: {c}\n".format(c=str(configfile)))
out.write(
"# versions as pinned in the conda env yamls of the configured tools, not queried from the running binaries\n"
)
out.write("#tool\tversion\tmultiqc_module\tsteps\n")
for row in rows:
out.write("\t".join(row) + "\n")
out.write(body)

log.info(logid + "Tool versions written to " + str(path))
return path
7 changes: 7 additions & 0 deletions MONSDA/Workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -2356,6 +2356,13 @@ def nf_fetch_params(
retconf["REFDIR"] = REFDIR
retconf["IP"] = IP

# Resolve all annotation paths to absolute paths once, so every nf
# process can read them reliably regardless of its own work directory
# (mirrors the equivalent hardening in workflows/header.smk).
for key in list(retconf.keys()):
if key.endswith("ANNO") and isinstance(retconf[key], str) and retconf[key]:
retconf[key] = os.path.abspath(retconf[key])

return retconf


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,25 @@ process salmon_quant{
"""
}
}
process decompress_annotation_dtu{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
cpus 1
cache 'lenient'
publishDir "${workflow.workDir}/../" , mode: 'link',
saveAs: {filename ->
if (filename.indexOf("log") > 0) "LOGS/${params.gSCOMBO}/DTU/"+"${params.gCOUNTENV}"+"/decompress_annotation.log"
}
input:
path anno
output:
path "annotation.gtf", emit: anno_plain
path "log", emit: log
script:
"""
zcat -f $anno > annotation.gtf 2> log; if [ ! -s annotation.gtf ]; then echo "ERROR: decompressed annotation from $anno is empty, check source file and filesystem" >> log; exit 1; fi
"""
}
process oarfish_dtu_quant{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
Expand All @@ -163,18 +182,20 @@ process oarfish_dtu_quant{
else "DTU/${params.gSCOMBO}/"+"${params.gCOUNTENV}"+"/${params.gCONDITION}/${file(filename).getName()}"
}
input:
path bam
path fls
output:
path "*.gz", includeInputs:false, emit: counts
path "*.log", emit: logs
script:
anno = fls[0]
bam = fls[1]
fn = file(bam).getSimpleName().replaceAll(/\Q_mapped_sorted\E/,"")
lf = "${params.gCOUNTENV}_"+fn+".log"
prefix = fn+"/"+params.gCOUNTENV
oz = fn+"/quant.sf.gz"
ol = fn+"_counts.gz"
"""
mkdir -p $fn; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; zcat -f ${params.gDTUANNO} > $fn/annotation.gtf 2>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $fn/annotation.gtf --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam $fn/annotation.gtf && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
mkdir -p $fn; : > $lf; if [ ! -s $anno ]; then echo "ERROR: decompressed annotation $anno is empty, check source file and filesystem" >> $lf; exit 1; fi; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $anno --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
"""
}
process create_summary_snippet{
Expand Down Expand Up @@ -345,7 +366,9 @@ workflow DTU{
element -> return "${workflow.workDir}/../MAPPED/${params.gCOMBO}/"+element+"_mapped_sorted.bam"
}
mapsamples_ch = Channel.fromPath(MAPPEDSAMPLES.sort())
oarfish_dtu_quant(mapsamples_ch.collate(1))
decompress_annotation_dtu(annofile)
annogtf_dtu = decompress_annotation_dtu.out.anno_plain
oarfish_dtu_quant(annogtf_dtu.combine(mapsamples_ch.collate(1)))
quantcounts = oarfish_dtu_quant.out.counts
}
prepare_dtu_annotation()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,25 @@ process salmon_quant{
"""
}
}
process decompress_annotation_dtu{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
cpus 1
cache 'lenient'
publishDir "${workflow.workDir}/../" , mode: 'link',
saveAs: {filename ->
if (filename.indexOf("log") > 0) "LOGS/${params.gSCOMBO}/DTU/"+"${params.gCOUNTENV}"+"/decompress_annotation.log"
}
input:
path anno
output:
path "annotation.gtf", emit: anno_plain
path "log", emit: log
script:
"""
zcat -f $anno > annotation.gtf 2> log; if [ ! -s annotation.gtf ]; then echo "ERROR: decompressed annotation from $anno is empty, check source file and filesystem" >> log; exit 1; fi
"""
}
process oarfish_dtu_quant{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
Expand All @@ -163,18 +182,20 @@ process oarfish_dtu_quant{
else "DTU/${params.gSCOMBO}/"+"${params.gCOUNTENV}"+"/${params.gCONDITION}/${file(filename).getName()}"
}
input:
path bam
path fls
output:
path "*.gz", includeInputs:false, emit: counts
path "*.log", emit: logs
script:
anno = fls[0]
bam = fls[1]
fn = file(bam).getSimpleName().replaceAll(/\Q_mapped_sorted\E/,"")
lf = "${params.gCOUNTENV}_"+fn+".log"
prefix = fn+"/"+params.gCOUNTENV
oz = fn+"/quant.sf.gz"
ol = fn+"_counts.gz"
"""
mkdir -p $fn; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; zcat -f ${params.gDTUANNO} > $fn/annotation.gtf 2>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $fn/annotation.gtf --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam $fn/annotation.gtf && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
mkdir -p $fn; : > $lf; if [ ! -s $anno ]; then echo "ERROR: decompressed annotation $anno is empty, check source file and filesystem" >> $lf; exit 1; fi; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $anno --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
"""
}
process create_summary_snippet{
Expand Down Expand Up @@ -346,7 +367,9 @@ workflow DTU{
element -> return "${workflow.workDir}/../MAPPED/${params.gCOMBO}/"+element+"_mapped_sorted.bam"
}
mapsamples_ch = Channel.fromPath(MAPPEDSAMPLES.sort())
oarfish_dtu_quant(mapsamples_ch.collate(1))
decompress_annotation_dtu(annofile)
annogtf_dtu = decompress_annotation_dtu.out.anno_plain
oarfish_dtu_quant(annogtf_dtu.combine(mapsamples_ch.collate(1)))
quantcounts = oarfish_dtu_quant.out.counts
}
prepare_dtu_annotation()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,25 @@ process salmon_quant{
"""
}
}
process decompress_annotation_dtu{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
cpus 1
cache 'lenient'
publishDir "${workflow.workDir}/../" , mode: 'link',
saveAs: {filename ->
if (filename.indexOf("log") > 0) "LOGS/${params.gSCOMBO}/DTU/"+"${params.gCOUNTENV}"+"/decompress_annotation.log"
}
input:
path anno
output:
path "annotation.gtf", emit: anno_plain
path "log", emit: log
script:
"""
zcat -f $anno > annotation.gtf 2> log; if [ ! -s annotation.gtf ]; then echo "ERROR: decompressed annotation from $anno is empty, check source file and filesystem" >> log; exit 1; fi
"""
}
process oarfish_dtu_quant{
conda "<REPO>/envs/${params.gCOUNTENV}"+".yaml"
container "oras://ghcr.io/jfallmann/monsda:"+"${params.gCOUNTENV}"+"-VERSION"
Expand All @@ -163,18 +182,20 @@ process oarfish_dtu_quant{
else "DTU/${params.gSCOMBO}/"+"${params.gCOUNTENV}"+"/${params.gCONDITION}/${file(filename).getName()}"
}
input:
path bam
path fls
output:
path "*.gz", includeInputs:false, emit: counts
path "*.log", emit: logs
script:
anno = fls[0]
bam = fls[1]
fn = file(bam).getSimpleName().replaceAll(/\Q_mapped_sorted\E/,"")
lf = "${params.gCOUNTENV}_"+fn+".log"
prefix = fn+"/"+params.gCOUNTENV
oz = fn+"/quant.sf.gz"
ol = fn+"_counts.gz"
"""
mkdir -p $fn; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; zcat -f ${params.gDTUANNO} > $fn/annotation.gtf 2>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $fn/annotation.gtf --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam $fn/annotation.gtf && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
mkdir -p $fn; : > $lf; if [ ! -s $anno ]; then echo "ERROR: decompressed annotation $anno is empty, check source file and filesystem" >> $lf; exit 1; fi; samtools collate -@ ${task.cpus} -o namecollated.bam $bam &>> $lf; ${params.gCOUNTBIN} --threads ${task.cpus} --genome-alignments namecollated.bam --annotation $anno --genome-fasta ${params.gDTUREF} ${params.gCOUNTPARAMS} --output $prefix &>> $lf && rm -f namecollated.bam && python3 ${params.gBINS}/Analysis/oarfish_to_salmon.py -i ${prefix}.quant -o $oz &>> $lf && ln -fs $oz $ol
"""
}
process create_summary_snippet{
Expand Down Expand Up @@ -344,7 +365,9 @@ workflow DTU{
element -> return "${workflow.workDir}/../MAPPED/${params.gCOMBO}/"+element+"_mapped_sorted.bam"
}
mapsamples_ch = Channel.fromPath(MAPPEDSAMPLES.sort())
oarfish_dtu_quant(mapsamples_ch.collate(1))
decompress_annotation_dtu(annofile)
annogtf_dtu = decompress_annotation_dtu.out.anno_plain
oarfish_dtu_quant(annogtf_dtu.combine(mapsamples_ch.collate(1)))
quantcounts = oarfish_dtu_quant.out.counts
}
prepare_dtu_annotation()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ if 'FUSIONS' in config:
ANNOTATION = FANNO
else:
ANNOTATION = ANNO.get('GTF') if 'GTF' in ANNO and ANNO.get('GTF') != '' else ANNO.get('GFF') # by default GTF format will be used
if ANNOTATION and isinstance(ANNOTATION, str):
# Resolve once to an absolute path so every rule's shell command can read
# it reliably regardless of its own working directory (e.g. containers).
ANNOTATION = os.path.abspath(ANNOTATION)
combo = ''
if ANNOTATION:
log.info(logid+f'Using COMBO: {combo}, REF: {REFERENCE}, ANNO: {ANNOTATION}, DECOY: {DECOY}, INDEX: {INDEX}, INDEX2: {INDEX2}, PREFIX: {PREFIX}, IP: {IP}, SETUP: {SETUP}, SEQUENCING: {SEQUENCING}, STRANDED: {stranded}, PAIRED: {paired}, RUNDEDUP: {rundedup}, PREDEDUP: {prededup}, SAMPLES: {SAMPLES}, BINS: {BINS}, MAXTHREAD: {MAXTHREAD}')
Expand Down Expand Up @@ -404,9 +408,17 @@ else:
# is needed, output is normalized to the salmon quant.sf schema so
# downstream DTU tooling (tximport type="salmon", build_DTU_table.py,
# SPIT.py) does not need to change.
rule decompress_annotation:
input: anno = ANNOTATION
output: anno = temp("DTU/{combo}/"+COUNTENV+"/annotation.gtf")
log: "LOGS/{combo}/DTU/"+COUNTENV+"/decompress_annotation.log"
conda: "<REPO>/envs/"+COUNTENV+".yaml"
container: "oras://ghcr.io/jfallmann/monsda:"+COUNTENV+""+"-VERSION"
threads: 1
shell: "zcat -f {input.anno} > {output.anno} 2> {log}; if [ ! -s {output.anno} ]; then echo 'ERROR: decompressed annotation from {input.anno} is empty, check source file and filesystem' >> {log}; exit 1; fi"
rule mapping:
input: bam = expand("MAPPED/{scombo}/{{file}}_mapped_sorted.bam", scombo=scombo),
anno = ANNOTATION,
anno = rules.decompress_annotation.output.anno,
fa = REFERENCE
output: cnts = report("DTU/{combo}/"+COUNTENV+"/{file}_counts.sf.gz", category="COUNTING"),
ctsdir = report(directory("DTU/{combo}/"+COUNTENV+"/{file}"), category="COUNTING")
Expand All @@ -417,11 +429,10 @@ else:
params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""),
mapp = COUNTBIN,
nbam = lambda wildcards, output: os.path.join(str(output.ctsdir), "namecollated.bam"),
anno = lambda wildcards, output: os.path.join(str(output.ctsdir), "annotation.gtf"),
prefix = lambda wildcards, output: os.path.join(str(output.ctsdir), COUNTENV),
linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)),
bins = BINS
shell: "set +euo pipefail; mkdir -p {output.ctsdir}; samtools collate -@ {threads} -o {params.nbam} {input.bam} &>> {log}; zcat -f {input.anno} > {params.anno} 2>> {log}; {params.mapp} --threads {threads} --genome-alignments {params.nbam} --annotation {params.anno} --genome-fasta {input.fa} {params.cpara} --output {params.prefix} &>> {log} && rm -f {params.nbam} {params.anno} && python3 {params.bins}/Analysis/oarfish_to_salmon.py -i {params.prefix}.quant -o {output.ctsdir}/quant.sf.gz &>> {log} ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}"
shell: "set +euo pipefail; mkdir -p {output.ctsdir}; samtools collate -@ {threads} -o {params.nbam} {input.bam} &>> {log}; {params.mapp} --threads {threads} --genome-alignments {params.nbam} --annotation {input.anno} --genome-fasta {input.fa} {params.cpara} --output {params.prefix} &>> {log} && rm -f {params.nbam} && python3 {params.bins}/Analysis/oarfish_to_salmon.py -i {params.prefix}.quant -o {output.ctsdir}/quant.sf.gz &>> {log} ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}"
if runterminus:
rule terminus_group:
input: ctsdir = rules.mapping.output.ctsdir
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ if 'FUSIONS' in config:
ANNOTATION = FANNO
else:
ANNOTATION = ANNO.get('GTF') if 'GTF' in ANNO and ANNO.get('GTF') != '' else ANNO.get('GFF') # by default GTF format will be used
if ANNOTATION and isinstance(ANNOTATION, str):
# Resolve once to an absolute path so every rule's shell command can read
# it reliably regardless of its own working directory (e.g. containers).
ANNOTATION = os.path.abspath(ANNOTATION)
combo = ''
if ANNOTATION:
log.info(logid+f'Using COMBO: {combo}, REF: {REFERENCE}, ANNO: {ANNOTATION}, DECOY: {DECOY}, INDEX: {INDEX}, INDEX2: {INDEX2}, PREFIX: {PREFIX}, IP: {IP}, SETUP: {SETUP}, SEQUENCING: {SEQUENCING}, STRANDED: {stranded}, PAIRED: {paired}, RUNDEDUP: {rundedup}, PREDEDUP: {prededup}, SAMPLES: {SAMPLES}, BINS: {BINS}, MAXTHREAD: {MAXTHREAD}')
Expand Down Expand Up @@ -414,9 +418,17 @@ else:
# is needed, output is normalized to the salmon quant.sf schema so
# downstream DTU tooling (tximport type="salmon", build_DTU_table.py,
# SPIT.py) does not need to change.
rule decompress_annotation:
input: anno = ANNOTATION
output: anno = temp("DTU/{combo}/"+COUNTENV+"/annotation.gtf")
log: "LOGS/{combo}/DTU/"+COUNTENV+"/decompress_annotation.log"
conda: "<REPO>/envs/"+COUNTENV+".yaml"
container: "oras://ghcr.io/jfallmann/monsda:"+COUNTENV+""+"-VERSION"
threads: 1
shell: "zcat -f {input.anno} > {output.anno} 2> {log}; if [ ! -s {output.anno} ]; then echo 'ERROR: decompressed annotation from {input.anno} is empty, check source file and filesystem' >> {log}; exit 1; fi"
rule mapping:
input: bam = expand("MAPPED/{scombo}/{{file}}_mapped_sorted.bam", scombo=scombo),
anno = ANNOTATION,
anno = rules.decompress_annotation.output.anno,
fa = REFERENCE
output: cnts = report("DTU/{combo}/"+COUNTENV+"/{file}_counts.sf.gz", category="COUNTING"),
ctsdir = report(directory("DTU/{combo}/"+COUNTENV+"/{file}"), category="COUNTING")
Expand All @@ -427,11 +439,10 @@ else:
params: cpara = lambda wildcards: tool_params(wildcards.file, None, config, 'DTU', DTUENV)['OPTIONS'].get('QUANT', ""),
mapp = COUNTBIN,
nbam = lambda wildcards, output: os.path.join(str(output.ctsdir), "namecollated.bam"),
anno = lambda wildcards, output: os.path.join(str(output.ctsdir), "annotation.gtf"),
prefix = lambda wildcards, output: os.path.join(str(output.ctsdir), COUNTENV),
linksf = lambda wildcards, output: str(os.path.abspath(output.ctsdir)),
bins = BINS
shell: "set +euo pipefail; mkdir -p {output.ctsdir}; samtools collate -@ {threads} -o {params.nbam} {input.bam} &>> {log}; zcat -f {input.anno} > {params.anno} 2>> {log}; {params.mapp} --threads {threads} --genome-alignments {params.nbam} --annotation {params.anno} --genome-fasta {input.fa} {params.cpara} --output {params.prefix} &>> {log} && rm -f {params.nbam} {params.anno} && python3 {params.bins}/Analysis/oarfish_to_salmon.py -i {params.prefix}.quant -o {output.ctsdir}/quant.sf.gz &>> {log} ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}"
shell: "set +euo pipefail; mkdir -p {output.ctsdir}; samtools collate -@ {threads} -o {params.nbam} {input.bam} &>> {log}; {params.mapp} --threads {threads} --genome-alignments {params.nbam} --annotation {input.anno} --genome-fasta {input.fa} {params.cpara} --output {params.prefix} &>> {log} && rm -f {params.nbam} && python3 {params.bins}/Analysis/oarfish_to_salmon.py -i {params.prefix}.quant -o {output.ctsdir}/quant.sf.gz &>> {log} ; ln -fs {params.linksf}/quant.sf.gz {output.cnts} &>> {log}"
if runterminus:
rule terminus_group:
input: ctsdir = rules.mapping.output.ctsdir
Expand Down
Loading
Loading