' for n, r in sorted(runs.items()))
self.log("index", status=200)
- page = ('\n'
- 'Runs'
- '
Runs
'
- ''
- f'
{items or "
No run under this folder yet.
"}
\n')
+ with open(os.path.join(self.live.cfg.assets, "home.html")) as f:
+ page = f.read()
+ page = (page.replace("{{assets}}", "/assets").replace("{{data}}", "/runs.json")
+ .replace("{{runs}}", items or "
No run under this folder yet.
"))
self.send_body(200, TYPES[".html"], page)
+ def runs_json(self):
+ self.log("runs", status=200)
+ self.send_json(self.live.runs_index())
+
def run_page(self, run):
plugins_dir = os.path.join(self.live.cfg.assets, "plugins")
js = sorted(f for f in os.listdir(plugins_dir) if ASSET_RE.match(f) and f.endswith(".js")) \
if os.path.isdir(plugins_dir) else []
tags = "\n".join(f'' for f in js)
- page = (self.live.index_page().replace("{{assets}}", "/assets").replace("{{run}}", html.escape(run["name"]))
- .replace("{{mode}}", "live").replace("{{plugins}}", tags))
+ page = (self.live.index_page(run.get("kind")).replace("{{assets}}", "/assets")
+ .replace("{{run}}", html.escape(run["name"])).replace("{{mode}}", "live").replace("{{plugins}}", tags))
self.log("run", run["name"], status=200)
self.send_body(200, TYPES[".html"], page)
diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py
index 7f1f20ef..17ec9f87 100644
--- a/bin/owngenome_live_family.py
+++ b/bin/owngenome_live_family.py
@@ -11,6 +11,12 @@
GET /family// pedigree.svg (drawn), or one of the family's
result files listed in state.json; any other
name is 404
+ GET /runs//family.json the family page's data (assets/live/family.html):
+ the state below, the pedigree's layout (so each
+ person is a link), the relationship and shared-
+ segment rows with their measured numbers, each
+ couple's gene rows, and each member's run progress
+ and attention count (from its summary.json only)
state.json gets plugins.family: {"kind": "family", ...} on a family's page,
{"kind": "member", "families": [...]} on the page of a sample in a family,
@@ -61,8 +67,9 @@ def family_dirs(root):
def read_tsv(path):
- """Rows of a tab-separated file with a header line, as dicts; [] when it is not there."""
- if not os.path.isfile(path):
+ """Rows of a tab-separated file with a header line, as dicts; [] when it is not there or is a
+ link out of its own folder (family.json serves only what the family folder itself holds)."""
+ if not os.path.isfile(path) or not inside(path, os.path.dirname(os.path.abspath(path))):
return []
rows = []
with open(path, errors="replace") as f:
@@ -138,6 +145,74 @@ def link(f):
"files": files}
+def layout_of(order, members):
+ """The pedigree drawing's geometry (bin/pedigree_svg.py layout) as data, so the page draws each
+ person as a link: people with their centre, partner lines and the descent lines of each sibship."""
+ pos, gen, pairs, widest, ngen = pedigree_svg.layout(order, members)
+ h = pedigree_svg.SIZE / 2
+ people = [{"sample": s, "x": round(pos[s][0], 1), "y": round(pos[s][1], 1), "gen": gen[s], "sex": members[s]["sex"],
+ "sequenced": bool(members[s]["sequenced"])} for s in order]
+ lines = []
+ for a, b in pairs:
+ (xa, ya), (xb, yb) = sorted((pos[a], pos[b]))
+ lines.append([xa + h, ya, xb - h, yb])
+ sibships = {}
+ for s in order:
+ f, mo = members[s]["father"], members[s]["mother"]
+ f, mo = (f if f in pos else None), (mo if mo in pos else None)
+ if f or mo:
+ sibships.setdefault((f, mo), []).append(s)
+ for (f, mo), kids in sibships.items():
+ if f and mo:
+ ox, oy = (pos[f][0] + pos[mo][0]) / 2, (pos[f][1] + pos[mo][1]) / 2
+ else:
+ ox, oy = pos[f or mo][0], pos[f or mo][1] + h
+ bar = min(pos[k][1] for k in kids) - h - 22
+ xs = [pos[k][0] for k in kids] + [ox]
+ lines += [[ox, oy, ox, bar], [min(xs), bar, max(xs), bar]]
+ lines += [[pos[k][0], bar, pos[k][0], pos[k][1] - h] for k in kids]
+ width = 2 * pedigree_svg.MARGIN + pedigree_svg.SIZE + (widest - 1) * pedigree_svg.STEP_X
+ height = pedigree_svg.TOP + pedigree_svg.SIZE + (ngen - 1) * pedigree_svg.STEP_Y + 30
+ return {"width": width, "height": height, "size": pedigree_svg.SIZE, "people": people,
+ "lines": [[round(v, 1) for v in ln] for ln in lines]}
+
+
+def family_page(name, d):
+ """family.json without the member runs: the state, the layout, the full pair rows and couple rows."""
+ st = family_state(name, d)
+ import render_report # the couple screen's caveat, word for word
+ out = {"state": st, "layout": None, "relationships": [], "ibd_pairs": [], "couples": {},
+ "couple_note": render_report.COUPLE_NOTE}
+ if st.get("error"):
+ return out
+ order, members = members_of(d)
+ out["layout"] = layout_of(order, members)
+ keep = ("sample_a", "sample_b", "declared", "expected_relatedness", "relatedness", "ibs0", "ibs2", "sites",
+ "verdict", "reason")
+ out["relationships"] = [{k: r.get(k) for k in keep} for r in read_tsv(os.path.join(d, "relationships.tsv"))]
+ keep = ("sample_a", "sample_b", "declared", "segments", "shared_cm", "ibd2_cm", "longest_cm", "relatedness",
+ "degree")
+ out["ibd_pairs"] = [{k: r.get(k) for k in keep} for r in read_tsv(os.path.join(d, "ibd_pairs.tsv"))]
+ order_of = {"shared": 0, "x_linked": 1, "one_partner": 2}
+ for a, b, base in couple_names(order, members):
+ rows = read_tsv(os.path.join(d, base + ".tsv"))
+ keep = ("gene", "inheritance", "condition", "result", "offspring_risk", "partner_a", "findings_a",
+ "partner_b", "findings_b", "explanation")
+ rows = sorted(({k: r.get(k) for k in keep} for r in rows), key=lambda r: order_of.get(r.get("result"), 3))
+ out["couples"]["%s %s" % (a, b)] = rows
+ # Step 38 writes the verdict "ok" for a pair that matches its declared relation. The page says that
+ # in words: no word on the page reads as an all-clear (state.json keeps the file's own value).
+ out["state"] = dict(st, relationships=[dict(r, verdict=verdict_words(r.get("verdict")))
+ for r in st.get("relationships") or []])
+ for r in out["relationships"]:
+ r["verdict"] = verdict_words(r.get("verdict"))
+ return out
+
+
+def verdict_words(v):
+ return "matches the declared relation" if (v or "").strip().lower() == "ok" else v
+
+
def member_state(root, sample):
fams = []
for name, d in family_dirs(root):
@@ -155,7 +230,40 @@ def root():
return os.path.realpath(app.config.root)
def runs(app_):
- return [{"name": n, "dir": d, "kind": "family"} for n, d in family_dirs(root())]
+ out = []
+ for n, d in family_dirs(root()):
+ try:
+ order, members = members_of(d)
+ people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"],
+ "mother": members[s]["mother"], "partner": members[s]["partner"],
+ "sequenced": members[s]["sequenced"]} for s in order]
+ except (OSError, ValueError):
+ people = []
+ out.append({"name": n, "dir": d, "kind": "family", "members": people})
+ return out
+
+ def page_data(req, m):
+ name = m.group(1)
+ d = req.run_dir(name)
+ if not d or not is_family(d) or not inside(d, root()):
+ app.log("family", name, "family.json", 404)
+ return req.send_body(404, "text/plain; charset=utf-8", "not found\n")
+ out = family_page(name, d)
+ live = getattr(req, "live", None)
+ runs = {}
+ for p in out["state"].get("members") or []:
+ r = live.run(p["sample"]) if live else None
+ if not r or r.get("kind") == "family":
+ continue
+ st = live.base_state(r)
+ planned = [s for s in st["steps"] if s["state"] != "not planned"]
+ n, mtime = live.attention(r["dir"] or os.path.join(live.root, r["name"]))
+ runs[p["sample"]] = {"url": "/runs/%s/" % p["sample"], "state": st["run"]["state"],
+ "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"),
+ "attention": n, "results": mtime is not None}
+ out["runs"] = runs
+ app.log("family", name, "family.json", 200)
+ return req.send_json(out)
def hook(run_dir, state, events):
if not run_dir:
@@ -191,3 +299,4 @@ def serve(req, m):
app.run_kind(runs)
app.state_hook("family", hook)
app.route("GET", r"/family/(%s)/(%s)" % (NAME, NAME), serve)
+ app.route("GET", r"/runs/(%s)/family\.json" % NAME, page_data)
diff --git a/bin/owngenome_live_results.py b/bin/owngenome_live_results.py
new file mode 100644
index 00000000..1684cf04
--- /dev/null
+++ b/bin/owngenome_live_results.py
@@ -0,0 +1,51 @@
+"""owngenome_live_results.py: a person's results on the live site, a plugin of bin/owngenome_live.py.
+
+ GET /runs//results.json bin/live_results.py's build() of the run's summary: the
+ report's seven chapters, one card per section with its
+ facts, its rows, its caveats and its attention flag
+
+The summary is the one the server already reads for the step headlines
+(Live.summary): the run's summary.json when step 24 wrote it after every done
+step, else collect_summary.collect() of the folder, read again only when a step
+ends. A reader error gives no cards and a note, never a 500. The raw
+summary.json is never served: no run_file glob is added, and results.json holds
+what the report prints. Every request writes one access-log row.
+"""
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+
+NAME = r"[A-Za-z0-9][A-Za-z0-9._-]*"
+
+
+def register(app):
+ def results(req, m):
+ live = req.live
+ run = live.run(m.group(1))
+ if not run or run.get("kind") == "family":
+ app.log("results", m.group(1), "-", 404)
+ return req.send_body(404, "text/plain; charset=utf-8", "not found\n")
+ name = run["name"]
+ d = run["dir"] or os.path.join(live.root, name)
+ out = {"schema_version": 1, "sample": name, "chapters": [], "note": None}
+ try:
+ import live_results
+ st = live.base_state(run)
+ summary = live.summary(name, d, st)
+ if summary:
+ files = live.run_files(run) if run["dir"] else set()
+ out = live_results.build(name, summary, files)
+ else:
+ out["note"] = "No step has finished yet: the results appear here as the steps end."
+ except Exception as e: # a reader bug must not take the page down
+ print(f"owngenome_live: results of {name} not built: {type(e).__name__}", file=sys.stderr)
+ out = {"schema_version": 1, "sample": name, "chapters": [],
+ "note": f"The results could not be read ({type(e).__name__}); the report and the Progress tab "
+ "still work."}
+ app.log("results", name, "-", 200)
+ return req.send_json(out)
+
+ app.route("GET", r"/runs/(%s)/results\.json" % NAME, results)
+
diff --git a/bin/render_report.py b/bin/render_report.py
index 654b450c..0e17c2db 100755
--- a/bin/render_report.py
+++ b/bin/render_report.py
@@ -90,6 +90,35 @@ def ancient_lines(d):
"only. See a genetic counsellor before acting on a result, or on its absence (docs/39-couple.md).")
PRS_ANCESTRY_NOTE = ("Most PRS scores were developed in European-ancestry populations and predict less well for "
"other ancestries, even with an ancestry-adjusted percentile.")
+# The caveats of the sections that had none, printed by both reports and the live site.
+PHARMCAT_NOTE = ("PharmCAT calls star alleles and phenotypes from the variants this genome holds in the genes it "
+ "covers. A variant it cannot see (a deletion, a duplication, a hybrid gene outside CYP2D6) is not in "
+ "its answer. Its drug guidance is for a prescriber to read with you: never change a medicine on "
+ "this report alone.")
+CPIC_NOTE = ("CPIC recommendations follow the phenotype PharmCAT called. They are guidance for prescribers, not a "
+ "prescription. A gene that was not called has no recommendation here, and that is not the same as a "
+ "typical response.")
+PGX_CALLERS_NOTE = ("CYP2D6 sits beside its look-alike CYP2D7, and short reads often cannot tell them apart, so "
+ "pypgx, Cyrius and step 36 can disagree or give no call. A disagreement or a missing call means "
+ "the gene was not read reliably here; a clinical CYP2D6 test settles it.")
+CPSR_NOTE = ("CPSR grades variants in cancer predisposition genes with ACMG rules. A pathogenic or likely pathogenic "
+ "row is something to take to a clinical geneticist, not a diagnosis. No row does not rule out an "
+ "inherited cancer risk: large deletions, genes outside its panel and family history are not part of "
+ "this screen.")
+VNTYPER_NOTE = ("The MUC1 repeat is long and GC-rich, and short reads cover it unevenly. VNtyper looks only for "
+ "known frameshifts in it: a negative result lowers the chance of those and does not rule out "
+ "ADTKD-MUC1, and a positive one needs a clinical test (SNaPshot for dupC) to confirm it.")
+SV_CALLERS_NOTE = ("Short reads find large deletions and duplications better than insertions and inversions, and "
+ "the callers disagree on many events. The totals count every call, most of them common in the "
+ "population; only the AnnotSV class 4 and 5 rows are graded for disease.")
+COVERAGE_NOTE = ("Mean depth is an average over the genome: some genes get far fewer reads, and the gene "
+ "callability card lists those. The sex, identity and contamination checks catch a swapped or "
+ "mixed sample, not every lab error.")
+VARIANTS_NOTE = ("Counts of the calls DeepVariant made against the reference. A whole genome usually holds 4 to 5 "
+ "million, most of them common. PASS calls passed the caller's filter; a count says nothing about "
+ "health on its own.")
+LPA_NOTE = ("The KIV-2 repeat count from read depth is a research estimate, not an Lp(a) level: the level is a "
+ "blood test, and the count explains only part of the difference between people.")
def combined_cell(h):
@@ -393,6 +422,9 @@ def pgx_copy_number(S):
MISSING = {"title": "", "state": "missing", "note": None, "data": {}}
+SEX_CHECK_AGREE = "declared and measured sex agree"
+
+
def freemix_text(q):
"""FREEMIX with its verdict, as both reports print it."""
if q.get("contamination") == "not_run" or not q.get("freemix"):
@@ -886,6 +918,48 @@ def annotsv_lines(d):
)
# The section id of each summary key.
SECTION_OF = {k: sid for _, _, _, secs in CHAPTERS for sid, _, keys in secs for k in keys}
+# The caveats of each section, by constant name, in the order the card shows them: the first is the
+# one the live site shows under a card's title. caveat_notes() is unkeyed, so the live site
+# (bin/live_results.py) reads this map; tests/test_live_results.py checks every section id has one.
+SECTION_NOTES = {
+ "qc": ("COVERAGE_NOTE", "KARYOTYPE_NOTE"),
+ "callability": ("CALLABILITY_NOTE",),
+ "variants": ("VARIANTS_NOTE",),
+ "clinvar": ("CLINVAR_FREQ_NOTE", "CDNA_SCAN_NOTE", "DEEP_INTRONIC_NOTE"),
+ "clinical-filter": ("ACMG_NOTE",),
+ "cpsr": ("CPSR_NOTE",),
+ "slivar": ("COMPHET_NOTE",),
+ "exomiser": ("EXOMISER_NOTE",),
+ "structural-variants": ("SV_CALLERS_NOTE", "ANNOTSV_NOTE", "CNVPYTOR_NOTE"),
+ "mei": ("MEI_NOTE",),
+ "expansions": ("EH_NOTE", "NOT_GRADED_NOTE", "RFC1_NOTE"),
+ "vntyper": ("VNTYPER_NOTE",),
+ "mito": ("MITO_NOTE", "MITO_POLYC_NOTE", "MITO_DELETION_NOTE", "MITO_COPIES_NOTE"),
+ "lpa": ("LPA_NOTE",),
+ "chip": ("CHIP_NOTE",),
+ "mosaic": ("MOSAIC_NOTE",),
+ "mosaic-mca": ("MCA_NOTE",),
+ "carrier-list": ("CARRIER_NOTE",),
+ "couple": ("COUPLE_NOTE",),
+ "paralogs": ("PARASCOPY_NOTE", "PARALOG_NOTE", "PARASCOPY_LOCI_NOTE"),
+ "pharmcat": ("PHARMCAT_NOTE",),
+ "cpic": ("CPIC_NOTE",),
+ "cyp2d6": ("PGX_CALLERS_NOTE",),
+ "hla": ("HLA_NOTE", "HLA_DERIVED_NOTE", "KIR_NOTE"),
+ "locityper": ("LOCITYPER_NOTE",),
+ "ancestry-fine": ("ANCESTRY_FINE_NOTE",),
+ "mt-haplogroup": ("EXPLORE_NOTE",),
+ "y-chromosome": ("Y_TREE_NOTE", "YSTR_NOTE", "EXPLORE_NOTE"),
+ "archaic": ("ARCHAIC_NOTE",),
+ "ancient": ("ANCIENT_NOTE",),
+ "roh": ("ROH_NOTE", "ROH_UPD_NOTE", "ROH_F_NOTE", "ROH_BAND_NOTE"),
+ "traits-panel": ("TRAITS_NOTE",),
+ "prs": ("PRS_ANCESTRY_NOTE",),
+ "bloodtype": ("BLOODTYPE_NOTE",),
+ "telomere": ("TELOMERE_NOTE",),
+ "viral": ("VIRAL_NOTE",),
+ "consumer-export": ("CONSUMER_NOTE",),
+}
def section_keys(sid):
@@ -974,7 +1048,8 @@ def head(key, title=None):
if sqc["state"] in ("ok", "stale"):
q = sqc["data"]
w(f" Sex from the reads (somalier): {q.get('inferred_sex')}"
- + (" ** DOES NOT MATCH the declared sex **" if q.get("sex_check") == "mismatch" else ""))
+ + (" ** DOES NOT MATCH the declared sex **" if q.get("sex_check") == "mismatch" else "")
+ + (f" ({SEX_CHECK_AGREE})" if q.get("sex_check") == "ok" else ""))
if q.get("sex_check") == "not_checked":
w(f" not checked: {q.get('sex_check_reason')}")
w(f" Contamination (VerifyBamID2 FREEMIX): {freemix_text(q)}")
@@ -982,6 +1057,7 @@ def head(key, title=None):
w(f" ** The same person as: {q['same_person_as']} (a duplicate or a sample swap) **")
for line in karyotype_lines(S):
w(f" {line}")
+ w(f" {COVERAGE_NOTE}")
w("")
if head("callability", "Genes This Genome Could Not Read Reliably"):
@@ -1004,6 +1080,7 @@ def head(key, title=None):
w(f" Total variants: {d['total']}")
w(f" PASS variants: {d['pass']}")
w(f" SNPs: {d['snps']} Indels: {d['indels']}")
+ w(f" {VARIANTS_NOTE}")
w("")
if head("clinvar", "ClinVar Pathogenic Screen"):
@@ -1066,6 +1143,7 @@ def head(key, title=None):
w(f" Report: {S['pharmcat']['source']} (PharmCAT {d.get('version') or 'version unknown'})")
if d.get("html_report"):
w(" HTML report beside it (open in a browser for full details)")
+ w(f" {PHARMCAT_NOTE}")
w("")
if head("cpic", "CPIC Drug-Gene Recommendations"):
@@ -1081,6 +1159,7 @@ def head(key, title=None):
for warn in d["warnings"]:
w(f" WARNING: {warn}")
w(f" Drugs per gene: {os.path.join(os.path.dirname(S['cpic']['source']), s['sample'] + '_cpic_recommendations.txt')}")
+ w(f" {CPIC_NOTE}")
w("")
if head("pypgx", "pypgx Pharmacogenomics"):
@@ -1121,6 +1200,7 @@ def head(key, title=None):
w(f" Step 36: {verdict}")
if s["cyp2d6"].get("probable"):
w(f" {s['cyp2d6']['probable']['label'].capitalize()}: {probable_text(s['cyp2d6']['probable'])}")
+ w(f" {PGX_CALLERS_NOTE}")
w("")
if head("paralogs", "Paralog Genes (Paraphase, PacBio HiFi)"):
@@ -1200,18 +1280,25 @@ def head(key, title=None):
w(f" {ANCESTRY_FINE_NOTE}")
w("")
+ sv_note = [f" {SV_CALLERS_NOTE}"] # printed once, under the first structural caller with results
if head("manta", "Structural Variants (Manta)"):
d = S["manta"]["data"]
w(f" Total SVs: {d['total']} PASS: {d['pass']}")
+ while sv_note:
+ w(sv_note.pop())
w("")
if head("delly", "Structural Variants (Delly)"):
d = S["delly"]["data"]
w(f" Total SVs: {d['total']} PASS: {d['pass']}")
+ while sv_note:
+ w(sv_note.pop())
w("")
if head("cnvpytor", "Copy Number Variants (CNVpytor)"):
d = S["cnvpytor"]["data"]
w(f" Total CNVs: {d['total']} ({d['deletions']} deletions, {d['duplications']} duplications)")
w(f" Significant (e-val < 0.01): {d['significant']}")
+ while sv_note:
+ w(sv_note.pop())
filt = cnvpytor_filtered_text(d)
if filt:
w(f" Significant after filters: {filt}")
@@ -1347,11 +1434,13 @@ def head(key, title=None):
if head("lpa", "Lp(a): LPA KIV-2 Repeat Count and Risk Variants"):
for line in lpa_lines(S["lpa"]["data"]):
w(f" {line}")
+ w(f" {LPA_NOTE}")
w("")
if head("vntyper", "MUC1 VNTR Frameshifts (ADTKD-MUC1, VNtyper)"):
for line in vntyper_lines(S["vntyper"]["data"]):
w(f" {line}")
+ w(f" {VNTYPER_NOTE}")
w("")
if head("chip", "Clonal Haematopoiesis (CHIP, step 49)"):
@@ -1414,6 +1503,7 @@ def head(key, title=None):
w(" Classification table has no CLASSIFICATION column.")
for line in cpsr_lines(d, s["databases"]):
w(f" {line}")
+ w(f" {CPSR_NOTE}")
w("")
if head("clinical", "Clinical Variant Filter"):
@@ -2553,6 +2643,7 @@ def a(key, lines):
qc.append(stat("Identity and contamination (step 33)", badge("Not run", "gray")))
kl = karyotype_lines(S)
qc += [stat("Sex chromosomes", badge(x.split(": ", 1)[1], "red")) for x in kl[:-1]] + [small_note(x) for x in kl[-1:]]
+ qc.append(small_note(COVERAGE_NOTE))
qc_note = [f'
{E(sec["title"] + ": " + stale_note(sec))}
'
for sec in (cov, sexc, sqc, rdc) if stale_note(sec)]
a("coverage", ['
', "
Quality Control
"] + qc_note + qc + ["
"])
@@ -2576,7 +2667,8 @@ def a(key, lines):
v = S["variants"]["data"]
a("variants", card(S["variants"], "Variant Calling", [stat("Total variants", v.get("total")), stat("PASS variants", v.get("pass")),
- stat("SNPs", v.get("snps")), stat("Indels", v.get("indels"))]))
+ stat("SNPs", v.get("snps")), stat("Indels", v.get("indels")),
+ small_note(VARIANTS_NOTE)]))
c = S["clinvar"]["data"]
n = c.get("count", 0)
@@ -2614,12 +2706,14 @@ def a(key, lines):
pg = S["pharmcat"]["data"]
cp = S["cpic"]["data"]
- pgx_body = [stat("PharmCAT report", badge("Complete", "green") if S["pharmcat"]["state"] in ("ok", "stale")
+ # Never green: a report that was written is not a finding (the frame's own Complete badge is neutral too).
+ pgx_body = [stat("PharmCAT report", badge("Complete", "state") if S["pharmcat"]["state"] in ("ok", "stale")
else badge("Not run", "gray"))]
if pg.get("version"):
pgx_body.append(stat("PharmCAT version", E(pg["version"])))
if S["pypgx"]["state"] in ("ok", "stale") and "comparison" in S["pypgx"]["data"]:
pgx_body.append(stat("PharmCAT vs pypgx conflicts", S["pypgx"]["data"]["comparison"]["conflicts"]))
+ pgx_body.append(small_note(PHARMCAT_NOTE))
a("pharmcat", card(S["pharmcat"], "Pharmacogenomics", pgx_body))
if S["pypgx"]["state"] != "missing":
pd = S["pypgx"]["data"]
@@ -2641,6 +2735,7 @@ def a(key, lines):
cpic_body.append(stat("PharmCAT report", badge("could not be parsed", "red")))
cpic_body.append(stat("Tip", 'Recommendations per gene: '
f'cpic/{E(s["sample"])}_cpic_recommendations.txt'))
+ cpic_body.append(small_note(CPIC_NOTE))
a("cpic", card(S["cpic"], "CPIC Drug Recommendations", cpic_body))
calls = s["cyp2d6"]["calls"]
@@ -2650,13 +2745,14 @@ def a(key, lines):
if agree is False else badge("fewer than two usable calls", "gray")))
cons = s["cyp2d6"].get("consensus")
if cons:
- cy_body.append(stat("Step 36", badge(cons.get("result") or "?", "green" if cons.get("passed_to_pharmcat")
+ cy_body.append(stat("Step 36", badge(cons.get("result") or "?", "state" if cons.get("passed_to_pharmcat")
else "yellow")))
cy_body.append(small_note(("Passed to PharmCAT: " if cons.get("passed_to_pharmcat") else "Not passed to PharmCAT: ")
+ (cons.get("reason") or "no reason given")))
prob = s["cyp2d6"].get("probable")
if prob:
cy_body.append(stat(prob["label"].capitalize(), E(probable_text(prob))))
+ cy_body.append(small_note(PGX_CALLERS_NOTE))
a("cyrius", ["
", "
CYP2D6 Across Callers
"]
+ [f'
{E(S[k]["title"] + ": " + stale_note(S[k]))}
'
for k in ("cpic", "pypgx", "cyrius") if S[k]["state"] in ("stale", "failed")]
@@ -2696,6 +2792,7 @@ def a(key, lines):
if cnvpytor_filtered_text(cn):
sv_body += [stat("CNVpytor significant after filters", E(cnvpytor_filtered_text(cn))), small_note(CNVPYTOR_NOTE)]
sv_body.append(stat("Consensus SVs (2+ callers)", sv.get("consensus", "N/A")))
+ sv_body.append(small_note(SV_CALLERS_NOTE))
notes = [f'
{E(S[k]["title"] + ": " + stale_note(S[k]))}
'
for k in ("manta", "delly", "cnvpytor", "sv_consensus") if stale_note(S[k])]
a("manta", ['
', "
Structural Variants
"] + notes + sv_body + ["
"])
@@ -2724,10 +2821,11 @@ def a(key, lines):
a("annotsv", card(S["annotsv"], "Structural Variants, Class 4 and 5 (AnnotSV)", body, full=True))
cs = S["cpsr"]["data"]
- cpsr_body = [stat("CPSR report", badge("Complete", "green") if cs.get("html_report") else badge("No HTML", "gray"))]
+ cpsr_body = [stat("CPSR report", badge("Complete", "state") if cs.get("html_report") else badge("No HTML", "gray"))]
for k, val in (cs.get("classification") or {}).items():
cpsr_body.append(stat(k, val))
cpsr_body += [small_note(line.strip()) for line in cpsr_lines(cs, s["databases"])]
+ cpsr_body.append(small_note(CPSR_NOTE))
a("cpsr", card(S["cpsr"], "Cancer Predisposition", cpsr_body))
eh = S["expansions"]["data"]
@@ -2834,13 +2932,13 @@ def a(key, lines):
lp = S["lpa"]["data"]
a("lpa", card(S["lpa"], "Lp(a): LPA KIV-2 Repeat Count",
([stat("Genetic sign of high Lp(a)", badge("Yes", "yellow"))] if lp.get("flag") == "high" else [])
- + [f"
{E(line)}
" for line in lpa_lines(lp)]))
+ + [f"
{E(line)}
" for line in lpa_lines(lp)] + [small_note(LPA_NOTE)]))
if S.get("vntyper", MISSING)["state"] != "missing":
vn = S["vntyper"]["data"]
a("vntyper", card(S["vntyper"], "MUC1 VNTR Frameshifts (VNtyper)",
([stat("Result", badge(vn.get("status", "."), {"positive": "red", "possible": "yellow",
"negative": "green"}.get(vn.get("status"), "gray")))])
- + [f"
{E(line)}
" for line in vntyper_lines(vn)]))
+ + [f"
{E(line)}
" for line in vntyper_lines(vn)] + [small_note(VNTYPER_NOTE)]))
if S.get("viral", MISSING)["state"] != "missing":
a("viral", card(S["viral"], "Viral DNA in the Unmapped Reads", viral_body(S["viral"]["data"]), full=True))
if S.get("chip", MISSING)["state"] != "missing":
diff --git a/docs/images/demo-live-site.png b/docs/images/demo-live-site.png
index 9478659e..db28216f 100644
Binary files a/docs/images/demo-live-site.png and b/docs/images/demo-live-site.png differ
diff --git a/docs/live-site.md b/docs/live-site.md
index 26018b78..5434e241 100644
--- a/docs/live-site.md
+++ b/docs/live-site.md
@@ -1,22 +1,22 @@
# Watch a Run Live
-A full run takes a day or more. The live site shows it while it goes, in a browser on any device that can reach the machine: which step is running, which finished, which failed or was skipped, and one line on what each finished step found. The page repaints as messages arrive; nobody has to reload it.
+A full run takes a day or more. The live site shows each person's results as their steps finish, in a browser on any device that can reach the machine: the report's chapters as tabs, one card per section with its numbers, its rows and its caveat, and what is worth a look first. A Progress tab keeps the step map. A home page lists everyone by family, and each family has a page of its own. The pages repaint as messages arrive; nobody has to reload them.
`bin/owngenome_live.py` is the server. It uses Python's standard library only, so the `python3` on the host is enough, and it has its own password, access log and limits, so it needs no web server in front of it.
## Try the demo
-https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the invented sample of the docs pictures. It replays a made-up run from the first task to the database re-check after it: press Play, or drag the slider. Every number, result and family member in it is made up.
+https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the invented sample of the docs pictures. It replays a made-up run from the first task to the database re-check after it: press Play, or drag the slider, and watch the cards fill in as their steps end. Every number, result and family member in it is made up.
-{ loading=lazy }
+{ loading=lazy }
-The page and its plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome viewer does not: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo.
+The pages and their plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The cards come from the `results.json` the server would give at the end of the run; each shows once the replay reaches the end of its step. The home page, the charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome viewer does not: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo.
What differs from your own site:
- It is a static folder published with these docs, so it has no password and no access log. Nothing in it is private.
- The run-health card measures an invented machine (64 GiB of RAM, a 2 TiB disk 55% used), not the one that built the page.
-- The CPIC recommendations and the gene callability table have no file in the demo sample, so the demo shows those two sections as missing.
+- The CPIC recommendations have no file in the demo sample, so the demo shows that card as not run.
## Start it
@@ -43,14 +43,29 @@ weblog {
}
```
-## What the page shows
+## What the pages show
+
+**Home** (`/`). Everyone, grouped by family, then the runs in no family. A card shows the pedigree symbol with a ring for the run's progress, the person's place in the family, the steps done and how many cards the report flags. Runs still going come first in each family. A family member with no run on this server is listed, dashed. Without scripts the page is a plain list of runs.
+
+**A person** (`/runs//`). The name, the declared and measured sex, the families they are in, and a strip with one cell per planned step. Below, one tab per chapter of the report, each named by its question:
+
+- **Look first** opens by default once a step has finished: the cards the report itself flags, in chapter order, each with its headline. A card not listed is for information; its caveats still apply.
+- **Trust, Doctor, Child, Medicines, Ancestors, Traits, Other** follow the report's seven chapters. One card per section, with the section's numbers, its rows (the first ones, then a button for all), its own sentences and its charts. Under each card sits its caveat: the first sentence, then the rest behind "Full caveat". A card is marked "worth a look" (▲) only where the report or the step headline raises a flag, and "for information" (○) otherwise; no card says a section is fine. A card whose step has not ended says so; one whose step failed or is older than its input says that. A newer database release shows on the cards it would change. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel.
+- **Leaves out** lists what the run does not assess.
+- **Progress** is the step map, a step's detail, Found so far, the run-health card and the database re-check, as before.
+
+Each card links to its section of the report and to the files it names. The header search finds a gene or rsID across the results and opens the card that names it. `#q/` opens a tab, `#s/` opens a card (the report's own anchors), `#progress` the step map.
- **The step map.** One box per step the run planned, drawn from `assets/steps.tsv`: a step sits right of the steps whose output it reads. The colour is the step's state from `bin/run_state.py` ([Where a Run Stands](run-state.md)): waiting, running, done, failed, skipped, not finished. Skipped steps stay on the map, greyed.
- **A step's detail.** Click a box: its state and reason, start and end, the usual hours, each process with its tasks waiting, running, done, from the cache or failed, and the headline of its sections with a link into the report.
-- **Found so far.** One line per finished section, worded by the same code that writes the report, caveat included. A line marked for attention means the report has something to read there; no line ever says a section is fine. The ClinVar line, with no hit, is information, never an all-clear.
+- **Found so far.** One line per finished section, worded by the same code that writes the report, caveat included. No line ever says a section is fine. The ClinVar line, with no hit, is information, never an all-clear.
- **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it.
- **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links and the charts' "See the reads" links open it at their position.
+**A family** (`/runs//`). The pedigree, drawn from `members.tsv` or `pedigree.tsv`, where each sequenced person with a run here opens their page and shows how many cards their report flags. Then the members, the analyses step 38 planned, each couple's carrier screen in full (genes where both carry a finding, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report, who is related how (step 38's relationship check beside the shared segments), and each child's chromosome map.
+
+On a phone the tab column becomes a row that scrolls sideways and each table row becomes a short list of label and value. The pages follow the browser's dark or light setting.
+
## The genome viewer
`bin/owngenome_live_viewer.py` serves it, behind the same password as every page. `owngenome-live.sh` passes it the reference (`REF_FASTA`, as in every step) and igv.js, which the first start downloads into `GENOME_DIR/tools/` and checks against `IGV_JS_SHA256` in `versions.env`. Without the reference's `.fai` or igv.js the site starts without the viewer and says why; `OWNGENOME_LIVE_VIEWER=off` leaves it out on purpose. Running `owngenome_live.py` yourself (in Docker, say), pass `--viewer-fasta` and `--viewer-igv-js`.
@@ -59,11 +74,13 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it
The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none.
-`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for.
+`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members.
+
+The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload.
## Privacy and what is stored
-- The site serves only its own pages, the run's state, the report, the files the report links to by a relative link, and the genome viewer's files of the run (see above). Any other file is 404, and so is a link that leaves the run folder or a symlink out of `GENOME_DIR`. It never lists a folder: a link to a folder that holds one HTML file (the report's `cpsr/`, with CPSR's own report) opens that file, and any other folder link is 404.
+- The site serves only its own pages, the run's state, the results the report prints (`results.json`, never `summary.json` itself), the report, the files the report links to by a relative link, and the genome viewer's files of the run (see above). Any other file is 404, and so is a link that leaves the run folder or a symlink out of `GENOME_DIR`. It never lists a folder: a link to a folder that holds one HTML file (the report's `cpsr/`, with CPSR's own report) opens that file, and any other folder link is 404.
- Each message is cut to what a progress view needs before it is stored: the task, its process and status, times, CPU and memory figures. Nextflow's parameters, command line and task scripts are dropped. Three paths stay on the server for later views (the output folder, the CPU and memory caps, the work directory); none of them reaches a page.
- Messages go to `/runs//events.jsonl` (state dir: `GENOME_DIR/.owngenome-live`, or `OWNGENOME_LIVE_STATE_DIR`). The site writes nowhere else, so `GENOME_DIR` can be mounted read-only.
- The access log, `/access.log`, has one tab-separated row per page or file read and per login (the genome viewer's files excepted: one row per page open, see above): time, client, the first 8 characters of a hash of the session, action, run, file, status. It never holds a query string, a request body, a result or the password. It moves to `access.log.1` past `--access-log-max-mb` (10).
@@ -148,11 +165,11 @@ WantedBy=multi-user.target
## Plugins
-The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `live.js`, `live.css`, `login.html`) are meant to stay small. A new view adds two files instead of editing them:
+The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `home.html`, `family.html`, `core.js`, `live.js`, `person.js`, `home.js`, `family-page.js`, `live.css`, `login.html`) are meant to stay small. A new view adds two files instead of editing them:
- `bin/owngenome_live_.py` with `register(app)`. At start the server imports every such file, sorted, and calls it. `app.route(method, pattern, fn, auth=True)` adds a page or an endpoint (behind the login unless `auth=False`), `app.run_file(glob)` lets the browser fetch more files of a run folder, `app.report_filter(fn)` rewrites the served report, `app.state_hook(name, fn)` adds `plugins[name]` to `state.json`, `app.run_kind(fn)` lists more runs, `app.add_argument(...)` adds a flag, `app.log(...)` writes an access-log row, and `app.config` holds the parsed flags.
-- `assets/live/plugins/.js`. The run page loads every script in that folder after `live.js`; it calls `window.PGP.register({name, onState, onEvent, panel})`. `onState(state)` gets each new `state.json`, `onEvent(message)` each message from the stream, and `panel(element)` gets a card of its own in the side column.
+- `assets/live/plugins/.js`. The run page loads every script in that folder after `live.js`; it calls `window.PGP.register({name, onState, onEvent, panel})`. `onState(state)` gets each new `state.json`, `onEvent(message)` each message from the stream, and `panel(element)` gets a card of its own in the Progress tab, or in the element whose id `slot` names (the search box sits in the header this way). A family's page loads the same scripts without `live.js`, so a plugin returns early when `window.PGP` is missing.
The page has a replay mode for a demo with no server: `` reads `frames.json` (`{"interval_ms": 1000, "frames": [{"t": ..., "state": }]}`) and plays it with a play button and a slider, through the same drawing code.
-`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts.
+`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_demo.py` builds the demo and checks every page, phone width included; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts.
diff --git a/docs/output-contract.md b/docs/output-contract.md
index 0d49b17c..bfab4e87 100644
--- a/docs/output-contract.md
+++ b/docs/output-contract.md
@@ -10,7 +10,7 @@ The full list is one file, [`assets/output_contract.tsv`](https://github.com/Gei
- **`--tools` names:** every name `main.nf` accepts (`pharmcat`, `clinvar`, `vcfanno`, ...).
- **`run-all.sh`:** its two arguments (``) and every switch in its `SWITCHES` list (`TOOLS`, `THREADS`, `SKIP_TRIM`, `INTERVALS`, `REF_FASTA`, ...).
- **Result files:** every file the Nextflow pipeline publishes, which is also what `run-all.sh` writes, at its path under `GENOME_DIR//` (or `--outdir`). For each one the contract gives its step, its format and the fields a reader can count on: the columns of a table, the INFO and FORMAT fields of a VCF header, the top-level keys of a JSON file (`summary.json`'s keys are the ones its schema, `tests/schema/summary.schema.json`, requires).
-- **Single-step scripts:** the files listed with leg `bash` or `both`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet.
+- **Single-step scripts:** the files listed with leg `bash` or `both`. A row is `both` when the step's script (`scripts/-*.sh`) writes the same path under `GENOME_DIR`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet.
## What counts as a break
@@ -40,7 +40,7 @@ The engines that run by default, and every step that existed before 1.0, are `st
Two checks read the contract, both with [`scripts/ci/check-output-contract.py`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/check-output-contract.py):
-- **On every pull request, with no run** (`tests/test_output_contract.py`, in the `unit-tests` job of the Guard workflow): every param of `nextflow_schema.json`, every `--tools` name of `main.nf` and every switch of `run-all.sh` has a row, and every row still names one. Every file row of a Nextflow process still matches a file that process writes and publishes, and every file a module publishes has a row. A removed or renamed stable item fails as `BREAKING`. A new one with no row fails too, with the message to add it, so the contract is always the full list. The test also plants each kind of break on a copy of the code (a renamed output, a removed param, a removed switch, a new tool and a new output with no row) and requires each to fail.
+- **On every pull request, with no run** (`tests/test_output_contract.py`, in the `unit-tests` job of the Guard workflow): every param of `nextflow_schema.json`, every `--tools` name of `main.nf` and every switch of `run-all.sh` has a row, and every row still names one. Every file row of a Nextflow process still matches a file that process writes and publishes, in the folder of a `publishDir` that keeps it (a `saveAs` that returns `null` for the file does not count), and every file a module publishes has a row. A published row with leg `nextflow` fails when its step's script writes the same path too, since its leg is then `both`; the check reads the script with its shell variables filled in, so a path it cannot follow is not flagged, and a `both` row it does not find is not refused. Every default in `nextflow_schema.json` must equal the value in `nextflow.config`. A removed or renamed stable item fails as `BREAKING`. A new one with no row fails too, with the message to add it, so the contract is always the full list. The test also plants each kind of break on a copy of the code (a renamed output, a removed param, a removed switch, a new tool and a new output with no row, a schema default that differs from `nextflow.config`, a row in the folder a `saveAs` drops the file from, and a `nextflow` row a step script also writes) and requires each to fail.
- **On the HG002 end-to-end run** (`tests/e2e/output-contract-1-hg002.sh`): every stable file whose `e2e` column names a leg must be in the results that leg wrote, under its contracted name, with its required fields. The bash leg is the single-step scripts the base cases run; the Nextflow leg is case `60-nextflow`. Rows whose step the fixture run does not reach have `e2e` set to `-` and are held by the static check only. The two files `run-all.sh` writes itself, `run_manifest.tsv` and `logs/run_status.tsv`, are listed but no check holds them yet: neither the fixture run nor a module writes them. The same case copies the Nextflow results, drops the `clnsig` column of the ClinVar hits table and renames the ROH summary, and requires the check to fail on both. It also prints every file each leg wrote with its fields, which is what to read when the contract is written again.
## Changing the contract
diff --git a/main.nf b/main.nf
index 7160fb9d..fb57d54e 100644
--- a/main.nf
+++ b/main.nf
@@ -61,13 +61,12 @@ workflow {
// ─── Validate inputs ────────────────────────────────────────────────
// Every parameter must be one nextflow_schema.json declares: Nextflow
// takes any --name as a new parameter, so a misspelt option (--outdirr,
- // --max-cpus) would be ignored with no word. --pedigree is read with
- // containsKey and has no default; config_profile_* are the test
- // profiles' labels.
+ // --max-cpus) would be ignored with no word. config_profile_* are the
+ // test profiles' labels.
def schema_params = new groovy.json.JsonSlurper()
.parseText(file("${projectDir}/nextflow_schema.json").text)['definitions']
.values().collectMany { group -> (group['properties'] ?: [:]).keySet() as List }
- def known_params = schema_params + ['pedigree', 'config_profile_name', 'config_profile_description']
+ def known_params = schema_params + ['config_profile_name', 'config_profile_description']
def unknown_params = params.keySet().findAll { k -> !known_params.contains(k) }.sort()
if (unknown_params) {
error "unknown parameter${unknown_params.size() > 1 ? 's' : ''} " +
@@ -332,9 +331,8 @@ workflow {
// leading family column: one family runs, the one --family_name names
// (needed when the file holds more than one). Here only the family and
// sample columns are read; FAMILY_PREPARE (bin/family_report.py prepare)
- // checks the rest before the joint call starts. Read with containsKey,
- // so a run without the option is not told about an undefined parameter.
- def pedigree = params.containsKey('pedigree') ? params.pedigree : null
+ // checks the rest before the joint call starts.
+ def pedigree = params.pedigree
// The family's folder under --outdir: --family_name, else 'family'
def family_dir = params.family_name ?: 'family'
def family_ids = []
diff --git a/nextflow.config b/nextflow.config
index 176b8db8..5b95c3bf 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -121,6 +121,7 @@ params {
viral_ref = null // viral: the viral genomes of step 47 (setup.sh --viral-data: reference/viral-/viral_genomes.fa)
hpo = null // exomiser: the symptoms as HPO ids, comma separated (HP:0001250,HP:0001263); setting it runs exomiser (step 48)
exomiser_data = null // exomiser: Exomiser's data folder (setup.sh --exomiser-data: reference/exomiser-)
+ pedigree = null // opt-in: a pedigree or families file (assets/pedigree.example.tsv, assets/families.example.tsv); runs the family workflow (steps 38, 39)
family_name = null // --pedigree: the family to run of a families file (assets/families.example.tsv) that holds more than one; also its folder under --outdir (default family)
exomiser_proband = null // exomiser: the sample the --hpo terms describe; needed when the samplesheet has more than one
exomiser_assembly = 'hg38' // exomiser: the assembly of its data; hg19 only for Exomiser's cut-down test release (CI)
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 6795fd58..91e62b6f 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -105,7 +105,7 @@
"tools": {
"type": "string",
"description": "Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS).",
- "default": "pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,html_report,multiqc",
+ "default": "pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,traits,html_report,multiqc",
"fa_icon": "fas fa-toolbox"
}
}
@@ -344,6 +344,12 @@
"description": "Exomiser's data folder, which `scripts/setup.sh --exomiser-data` installs as reference/exomiser- (its _hg38 and _phenotype folders). Required with --hpo.",
"fa_icon": "fas fa-folder"
},
+ "pedigree": {
+ "type": "string",
+ "format": "file-path",
+ "description": "Opt-in: run the family workflow (steps 38 and 39) on the samples of the samplesheet this file names: a pedigree (assets/pedigree.example.tsv), or a families file with a leading family column (assets/families.example.tsv). A families file that holds more than one family needs --family_name, and only that family runs. The family that runs needs two or more of those samples, each with a gVCF, and --somalier_sites.",
+ "fa_icon": "fas fa-sitemap"
+ },
"family_name": {
"type": "string",
"pattern": "^[a-zA-Z0-9._][a-zA-Z0-9._-]*$",
diff --git a/scripts/ci/check-output-contract.py b/scripts/ci/check-output-contract.py
index e397cad5..d0dd192e 100755
--- a/scripts/ci/check-output-contract.py
+++ b/scripts/ci/check-output-contract.py
@@ -8,10 +8,14 @@
run: every param of nextflow_schema.json, every --tools name of main.nf
(known_tools) and every run-all.sh switch (its SWITCHES list) has a row,
and every row still names one; every file row of a process names a file
- that process still writes and publishes; and every file a module
- publishes has a row. A stable row that no longer matches is a break. A
- new param, tool, switch or published file with no row fails too, so the
- contract stays the full list.
+ that process still writes and publishes, in the folder of a publishDir
+ that keeps it; every file a module publishes has a row; a nextflow row
+ (not internal) whose step script writes the same path is refused (its
+ legs are both);
+ and every default of nextflow_schema.json equals nextflow.config's. A
+ stable row that no longer matches is a break. A new param, tool, switch
+ or published file with no row fails too, so the contract stays the full
+ list.
--outputs DIR --leg bash|nextflow --sample S [--observe]
The files a run wrote. DIR is the folder that holds / (GENOME_DIR
@@ -109,8 +113,49 @@ def concrete(pattern, sample="SAMPLE"):
return p.strip("/")
+SAVE_AS = re.compile(r"^\s*(\w+)\s*->\s*\1\s*(?:\.(endsWith|startsWith)\(\s*'([^']*)'\s*\)|==\s*'([^']*)')"
+ r"\s*\?\s*([^?:]+?)\s*:\s*([^?:]+?)\s*$", re.S)
+
+
+def closure_body(stmt):
+ """The body of a publishDir's saveAs closure, up to its own closing brace (an option
+ after it is not part of it), or None when the statement has no saveAs."""
+ m = re.search(r"saveAs:\s*\{", stmt)
+ if not m:
+ return None
+ depth, i = 1, m.end()
+ while i < len(stmt):
+ depth += {"{": 1, "}": -1}.get(stmt[i], 0)
+ if depth == 0:
+ return stmt[m.end():i]
+ i += 1
+ return stmt[m.end():]
+
+
+def save_as(closure, name):
+ """What a publishDir's saveAs closure does to a task file name (an output glob):
+ 'keep' (published under its own name), 'drop' (returns null) or 'rename'.
+ A closure this reader does not follow counts as 'rename', which the static
+ check matches by basename only."""
+ if closure is None:
+ return "keep"
+ m = SAVE_AS.match(closure)
+ if not m:
+ return "rename"
+ var, method, arg, eq, yes, no = m.groups()
+ if method == "endsWith":
+ cond = name.endswith(arg)
+ elif method == "startsWith":
+ cond = name.startswith(arg)
+ else:
+ cond = name == eq
+ expr = (yes if cond else no).strip()
+ return "drop" if expr == "null" else "keep" if expr == var else "rename"
+
+
def module_processes(repo):
- """{process: {'publish': [(dir, [patterns] or None, has_saveas)], 'outputs': [glob]}} of modules/local."""
+ """{process: {'publish': [(dir, [patterns] or None, saveAs closure body or None)], 'outputs': [glob]}}
+ of modules/local."""
procs = {}
for f in sorted(glob.glob(os.path.join(repo, "modules", "local", "*", "main.nf"))):
src = open(f).read()
@@ -134,7 +179,7 @@ def module_processes(repo):
pat = re.search(r"pattern:\s*[\"']([^\"']+)[\"']", stmt)
pubs.append((concrete(d.group(1)) if d else "",
braces(pat.group(1)) if pat else None,
- "saveAs" in stmt))
+ closure_body(stmt)))
out = re.search(r"^\s*output:\s*$(.*?)(?:^\s*(?:when|topic):|\Z)", head, re.M | re.S)
outputs = []
if out:
@@ -147,15 +192,19 @@ def module_processes(repo):
def published(proc):
- """(contracted-path glob, basename-only) pairs a process publishes; basename-only when a saveAs renames."""
+ """(output glob, contracted-path glob, basename-only) of each file a process publishes. A publishDir
+ whose saveAs drops the file does not count; basename-only when its saveAs renames the file."""
items = []
- for d, pats, saveas in proc["publish"]:
+ for d, pats, closure in proc["publish"]:
for g in proc["outputs"]:
c = concrete(g)
if pats is not None and not any(fnmatch.fnmatch(c.replace("*", "X"), p) or fnmatch.fnmatch(c, p)
for p in pats):
continue
- items.append((g, (d + "/" + c) if d else c, saveas))
+ action = save_as(closure, c)
+ if action == "drop":
+ continue
+ items.append((g, (d + "/" + c) if d else c, action == "rename"))
return items
@@ -165,6 +214,82 @@ def schema_params(repo):
return {p for group in defs.values() for p in group.get("properties", {})}
+def schema_defaults(repo):
+ s = json.load(open(os.path.join(repo, "nextflow_schema.json")))
+ defs = s.get("definitions") or s.get("$defs") or {}
+ return {p: v["default"] for group in defs.values() for p, v in group.get("properties", {}).items()
+ if "default" in v}
+
+
+def config_params(repo):
+ """{name: value} of nextflow.config's params block, for the plain literals (strings, numbers,
+ true, false, null); a value this reader cannot parse is left out."""
+ src = open(os.path.join(repo, "nextflow.config")).read()
+ m = re.search(r"^params\s*\{(.*?)^\}", src, re.M | re.S)
+ out = {}
+ if not m:
+ return out
+ lit = {"true": True, "false": False, "null": None}
+ for name, val in re.findall(r"^\s*(\w+)\s*=\s*('[^']*'|\"[^\"]*\"|[-\w.]+)", m.group(1), re.M):
+ if val[0] in "'\"":
+ out[name] = val[1:-1]
+ elif val in lit:
+ out[name] = lit[val]
+ else:
+ try:
+ out[name] = int(val)
+ except ValueError:
+ try:
+ out[name] = float(val)
+ except ValueError:
+ pass
+ return out
+
+
+VAR = re.compile(r"\$\{(\w+)\}|\$(\w+)")
+ASSIGN = re.compile(r"^\s*(?:local\s+|export\s+|readonly\s+)?([A-Za-z_]\w*)=(\"[^\"\n]*\"|'[^'\n]*'|[^\s;#()]+)", re.M)
+
+
+def bash_paths(src):
+ """The lines of a step script with its shell variables expanded: SAMPLE as {sample},
+ GENOME_DIR as G, every other variable as each value it is assigned (a few at most)."""
+ assign = {}
+ for v, val in ASSIGN.findall(src):
+ assign.setdefault(v, []).append(val.strip("\"'"))
+ fixed = {"SAMPLE": "{sample}", "GENOME_DIR": "G", "1": "{sample}"}
+
+ # depth counts every variable filled in on the line, not the nesting: a line with
+ # ${WORK}, ${P} and P's own chain of OUTDIR, S, GENOME_DIR and SAMPLE needs seven.
+ # The search stops at 64 expansions of a line, so a deep line stays cheap.
+ def expand(s, depth=0):
+ m = VAR.search(s)
+ if not m or depth > 24:
+ return [s]
+ name = m.group(1) or m.group(2)
+ vals = [fixed[name]] if name in fixed else assign.get(name) or ["\0"]
+ out = []
+ for val in vals[:4]:
+ out.extend(expand(s[:m.start()] + val + s[m.end():], depth + 1))
+ if len(out) >= 64:
+ break
+ return out[:64]
+
+ return {e for line in src.split("\n") if not line.lstrip().startswith("#") for e in expand(line)}
+
+
+def bash_writes(repo, step, name, cache):
+ """The step script that names this per-sample path (scripts/-*.sh), or None."""
+ if not name.startswith("{sample}/") or "*" in name or step == "-":
+ return None
+ if step not in cache:
+ files = sorted(glob.glob(os.path.join(repo, "scripts", f"{step}-*.sh")))
+ cache[step] = (files, "\n".join(bash_paths("".join(open(f).read() for f in files))) if files else "")
+ files, text = cache[step]
+ # the whole path: a longer name that starts with it (x.tsv.gz for x.tsv) is not it
+ hit = files and re.search(re.escape("G/" + name) + r"(?![\w.-])", text)
+ return os.path.relpath(files[0], repo) if hit else None
+
+
def known_tools(repo):
src = open(os.path.join(repo, "main.nf")).read()
m = re.search(r"def known_tools = \[(.*?)\]", src, re.S)
@@ -193,6 +318,14 @@ def check_static(repo, rows):
for name in sorted(have[kind] - set(listed)):
problems.append(f"{kind} {name} of {where[kind]} has no row in the contract: add one")
+ defaults, config = schema_defaults(repo), config_params(repo)
+ if not config:
+ problems.append("found no params in nextflow.config: the reader no longer matches the code")
+ for name, val in sorted(defaults.items()):
+ if name in config and config[name] != val:
+ problems.append(f"param {name}: nextflow_schema.json's default {val!r} is not nextflow.config's "
+ f"{config[name]!r}")
+
procs = module_processes(repo)
if not procs:
problems.append("found no processes under modules/local")
@@ -214,6 +347,13 @@ def check_static(repo, rows):
what = "BREAKING: the stable file" if r["stability"] == "stable" else "the file"
problems.append(f"{what} {r['name']} (contract line {r['line']}) is not what {r['process']} "
f"publishes ({p['file']}): renamed, moved or no longer written")
+ cache = {}
+ for r in rows:
+ if r["kind"] == "file" and r["legs"] == "nextflow" and r["stability"] != "internal":
+ script = bash_writes(repo, r["step"], r["name"], cache)
+ if script:
+ problems.append(f"{r['name']} (contract line {r['line']}) has legs nextflow, but {script} "
+ "writes it under the same name too: its legs are both")
for name, p in sorted(procs.items()):
rows_p = [concrete(r["name"]) for r in file_rows if r["process"] == name]
if not p["publish"]:
diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv
index c302ec67..a1c3636b 100644
--- a/scripts/ci/e2e-case-paths.tsv
+++ b/scripts/ci/e2e-case-paths.tsv
@@ -100,7 +100,7 @@ hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.
hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh
hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_mito_merge.py bin/pgx_outside_calls.py bin/pgx_refblock_fill.py bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh
ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/family_segments.py bin/ibd_report.py scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh
-live-server-1-weblog assets/live/index.html assets/live/live.css assets/live/login.html assets/viewer/index.html bin/owngenome_live.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh
+live-server-1-weblog assets/live/home.html assets/live/index.html assets/live/live.css assets/live/login.html assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh
lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh
mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/mei_report.py bin/sv_svlen_fix.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh
mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh
diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py
index 03124bcd..1d62b912 100644
--- a/tests/demo/build_live_demo.py
+++ b/tests/demo/build_live_demo.py
@@ -5,14 +5,19 @@
writes DIR, which the docs workflow publishes at /demo/ of the docs site:
- index.html the list of runs, as the live site's / shows it
- assets/ assets/live/ as it is: live.js, live.css, plugins/*.js
- runs/DEMO-001/index.html the live page (assets/live/index.html) in replay mode
+ index.html the home page (assets/live/home.html), as the live site's / shows it
+ runs.json what the home page draws, as /runs.json gives it
+ assets/ assets/live/ as it is: core.js, live.js, person.js, live.css, plugins/*.js
+ runs/DEMO-001/index.html the person page (assets/live/index.html) in replay mode
runs/DEMO-001/frames.json the page's state.json at each moment of the run
+ runs/DEMO-001/results.json the person's results (bin/live_results.py), as the results
+ plugin serves them at the end of the run; each card shows
+ once the replay reaches the end of its steps
runs/DEMO-001/... the files the plugins point at (the charts, the
search index, the re-check), the report as
report.html and the files it links to
- runs/DEMO-FAMILY/ the family page of DEMO-001 and its invented partner
+ runs/DEMO-FAMILY/ the family page (assets/live/family.html) of DEMO-001 and its
+ invented partner, with family.json as the family plugin serves it
family/DEMO-FAMILY/ the pedigree and the files the family page opens
No state is written by hand. The frames are bin/owngenome_live.py's own
@@ -63,7 +68,9 @@
REPO = os.path.dirname(os.path.dirname(HERE))
BIN = os.path.join(REPO, "bin")
sys.path.insert(0, BIN)
+import live_results # noqa: E402
import owngenome_live # noqa: E402
+import owngenome_live_family # noqa: E402
import owngenome_recheck # noqa: E402
import pedigree_svg # noqa: E402
import run_headroom # noqa: E402
@@ -217,9 +224,16 @@ def snap(t):
fam = live.state(live.run(FAMILY))
fam["generated_utc"] = checked
fam["plugins"] = relink(fam["plugins"])
+ index = live.runs_index()
+ run, d = live.run(SAMPLE), os.path.join(root, SAMPLE)
+ # The demo's run folder has no run_status.tsv (OTHER_RUN), so the server knows it from its
+ # messages alone; the files it may serve are those of the folder all the same.
+ results = live_results.build(SAMPLE, live.summary(SAMPLE, d, live.base_state(run)),
+ owngenome_live.Live.run_files(None, {"dir": d, "name": SAMPLE}))
+ results["generated_utc"] = checked
finally:
time.time, run_headroom._statvfs, run_headroom.physical_memory = real
- return frames, {"t": checked, "state": fam}, app
+ return frames, {"t": checked, "state": fam, "index": index, "results": results}, app
# --- the static folder ----------------------------------------------------------------------
@@ -229,16 +243,47 @@ def with_csp(page):
return page.replace('', '\n' + meta, 1)
-def run_page(name, plugin_js):
- with open(os.path.join(ASSETS, "index.html")) as f:
+def run_page(name, plugin_js, template="index.html"):
+ with open(os.path.join(ASSETS, template)) as f:
page = f.read()
tags = "\n".join(f'' for f in plugin_js)
page = (page.replace("{{assets}}", "../../assets").replace("{{run}}", html.escape(name))
- .replace("{{mode}}", "replay").replace("{{plugins}}", tags)
- .replace('All runs', 'All runs'))
+ .replace("{{mode}}", "replay").replace("{{plugins}}", tags).replace('href="/"', 'href="../../"'))
return with_csp(page)
+def home_page(index):
+ """assets/live/home.html as the server fills it, with a line saying the people are invented and no
+ log-out button (a static host has no session)."""
+ with open(os.path.join(ASSETS, "home.html")) as f:
+ page = f.read()
+ items = "\n".join(f'
' for r in index["runs"])
+ page = (page.replace("{{assets}}", "assets").replace("{{data}}", "runs.json").replace("{{runs}}", items)
+ .replace('',
+ '
A demo: the people, the run and the results are invented.
'))
+ return with_csp(page)
+
+
+def unroot(v):
+ """/runs.json's absolute links as links relative to the site's root."""
+ if isinstance(v, dict):
+ return {k: unroot(x) for k, x in v.items()}
+ if isinstance(v, list):
+ return [unroot(x) for x in v]
+ if isinstance(v, str) and re.match(r"^/(?:runs|family)/", v):
+ return v[1:]
+ return v
+
+
+def family_json(root, index):
+ """family.json as bin/owngenome_live_family.py serves it, its links relative to runs/DEMO-FAMILY/."""
+ out = owngenome_live_family.family_page(FAMILY, os.path.join(root, FAMILY))
+ by = {r["name"]: r for r in index["runs"]}
+ out["runs"] = {p["sample"]: {k: by[p["sample"]][k] for k in ("url", "state", "planned", "done", "attention", "results")}
+ for p in out["state"].get("members") or [] if p["sample"] in by}
+ return relink(out)
+
+
def plain_page(title, body, css):
return with_csp('\n'
''
@@ -255,23 +300,22 @@ def write_json(path, obj):
def write_site(out, root, frames, fam):
if os.path.exists(out):
shutil.rmtree(out)
- shutil.copytree(ASSETS, os.path.join(out, "assets"), ignore=shutil.ignore_patterns("index.html", "login.html"))
+ shutil.copytree(ASSETS, os.path.join(out, "assets"),
+ ignore=shutil.ignore_patterns("index.html", "login.html", "home.html", "family.html"))
plugin_dir = os.path.join(ASSETS, "plugins")
plugin_js = sorted(f for f in os.listdir(plugin_dir) if owngenome_live.ASSET_RE.match(f) and f.endswith(".js")) \
if os.path.isdir(plugin_dir) else []
- runs = [(SAMPLE, "sample"), (FAMILY, "family")]
- items = "\n".join(f'
' for n, k in runs)
with open(os.path.join(out, "index.html"), "w") as f:
- f.write(plain_page("Runs", '
A demo of the live site on DEMO-001, an invented sample: '
- 'the run, the results and the family are made up. Nothing here is a real person.
'
- f'
{items}
', "assets/live.css"))
+ f.write(home_page(fam["index"]))
+ write_json(os.path.join(out, "runs.json"), unroot(fam["index"]))
sd, d = os.path.join(out, "runs", SAMPLE), os.path.join(root, SAMPLE)
os.makedirs(sd)
with open(os.path.join(sd, "index.html"), "w") as f:
f.write(run_page(SAMPLE, plugin_js))
write_json(os.path.join(sd, "frames.json"), {"interval_ms": INTERVAL_MS, "autoplay": True, "frames": frames})
+ write_json(os.path.join(sd, "results.json"), fam["results"])
files = {f"{SAMPLE}_report.html": "report.html"}
# The files the report links to, as the live server lists them (Live.run_files reads no state).
files.update({rel: rel for rel in owngenome_live.Live.run_files(None, {"dir": d, "name": SAMPLE})})
@@ -293,8 +337,9 @@ def write_site(out, root, frames, fam):
fd = os.path.join(out, "runs", FAMILY)
os.makedirs(fd)
with open(os.path.join(fd, "index.html"), "w") as f:
- f.write(run_page(FAMILY, plugin_js))
- write_json(os.path.join(fd, "frames.json"), {"interval_ms": INTERVAL_MS, "frames": [fam]})
+ f.write(run_page(FAMILY, plugin_js, "family.html"))
+ write_json(os.path.join(fd, "frames.json"), {"interval_ms": INTERVAL_MS, "frames": [{"t": fam["t"], "state": fam["state"]}]})
+ write_json(os.path.join(fd, "family.json"), family_json(root, fam["index"]))
famdir, ffiles = os.path.join(out, "family", FAMILY), os.path.join(root, FAMILY)
os.makedirs(famdir)
order, members, _ = pedigree_svg.load(ffiles)
diff --git a/tests/demo/render_pictures.py b/tests/demo/render_pictures.py
index 042b7139..795eccc1 100755
--- a/tests/demo/render_pictures.py
+++ b/tests/demo/render_pictures.py
@@ -22,7 +22,8 @@
as the docs workflow publishes it: built by
tests/demo/build_live_demo.py on a DEMO-001 it
writes itself (--genome-dir is not read), at the
- last moment of its replay, in a 1280 x 1000 window
+ last moment of its replay, on the person page's
+ doctor tab (#q/clinical), in a 1280 x 1000 window
Every PNG is written again from its pixels alone, so it holds no text, time
or EXIF chunk. Needs playwright (with its Chromium) and pillow;
@@ -122,12 +123,13 @@ def live_picture(out):
# all 'self', so the picture is the same.
page = browser.new_page(viewport={"width": 1280, "height": 1000}, device_scale_factor=1,
color_scheme="dark", bypass_csp=True)
- page.goto(f"http://127.0.0.1:{srv.server_address[1]}/runs/{SAMPLE}/")
+ page.goto(f"http://127.0.0.1:{srv.server_address[1]}/runs/{SAMPLE}/#q/clinical")
page.wait_for_function("document.documentElement.dataset.pgpStatus === 'ready'")
# The last moment: the run ended, the charts are drawn, the re-check is in.
page.evaluate("() => { const s = document.getElementById('replay-slider'); s.value = s.max; "
"s.dispatchEvent(new Event('input')); }")
- page.wait_for_function("Number(document.documentElement.dataset.pgpCharts) > 0")
+ page.wait_for_function("Number(document.documentElement.dataset.pgpCharts) > 0 "
+ "&& Number(document.documentElement.dataset.pgpResults) > 0")
page.wait_for_timeout(1000)
path = os.path.join(out, "demo-live-site.png")
page.screenshot(path=path)
diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py
index efc760d8..33c54dee 100644
--- a/tests/test_live_demo.py
+++ b/tests/test_live_demo.py
@@ -12,16 +12,21 @@
once the report step has ended, the re-check only after the run;
4. the run-health card measures the invented machine, never the machine that
builds the demo, and no frame holds a path of that machine;
- 5. every link a frame gives is relative and reaches a file of the demo;
+ 5. every link a frame gives is relative and reaches a file of the demo, and
+ so is every link of runs.json, results.json and family.json;
6. every page the build writes carries the server's Content-Security-Policy
as a meta tag, and no page or script of the demo loads anything from
outside;
7. in a headless Chrome (skipped without Chrome, except in CI), served as a
- static host serves it (no policy header): the runs list, the replay to its
- last frame, the charts, a search, the re-check banner, the run-health
- card, the family link and its pedigree, the viewer page, and no policy
- violation.
-Checks 1, 2, 3 and 5 also run against a planted mistake and must fail there (2:
+ static host serves it (no policy header): the home page's people and
+ family, the replay to its last frame, the charts, a search, the re-check
+ banner, the run-health card, the family link and its pedigree, the viewer
+ page, and no policy violation; the person page's cards (the doctor tab
+ with the ClinVar card, its caveat and its chart), the tab row moved with
+ the arrow keys, #s/clinvar focusing its card; the family page with a
+ clickable pedigree and no step map or report link; and at a phone's width
+ (390 px) no page scrolls sideways.
+Checks 1, 2, 3, 5 and the phone width also run against a planted mistake and must fail there (2:
a build with one plugin's state hook dropped): a check that cannot fail is no check.
Run: python3 tests/test_live_demo.py
@@ -144,6 +149,27 @@ def link_problems(out, built):
return bad
+def json_link_problems(out):
+ """Links of runs.json, results.json and family.json that are absolute or reach no file of the demo."""
+ bad = []
+ for rel, base in (("runs.json", out), (os.path.join("runs", S, "results.json"), os.path.join(out, "runs", S)),
+ (os.path.join("runs", FAM, "family.json"), os.path.join(out, "runs", FAM))):
+ with open(os.path.join(out, rel)) as f:
+ data = json.load(f)
+ found = list(links(data))
+ if rel.endswith("results.json"):
+ found += [x for ch in data["chapters"] for c in ch["sections"] for x in c.get("links") or []]
+ if not found:
+ bad.append(f"{rel}: no link at all")
+ for u in found:
+ if u.startswith("https://"): # an explore link of the report (a haplogroup tree), opened by the reader
+ continue
+ p = os.path.normpath(os.path.join(base, u))
+ if u.startswith("/") or not (os.path.isfile(p) or os.path.isfile(os.path.join(p, "index.html"))):
+ bad.append(f"{rel}: {u}")
+ return bad
+
+
def located_term(out):
"""A term of the demo's search index with a mention at a locus (its hit links the viewer)."""
with open(os.path.join(out, "runs", S, f"{S}_search_index.json")) as f:
@@ -175,9 +201,12 @@ def browser_checks(out):
b = Chrome(chrome, os.path.join(os.path.dirname(out), "chrome"))
try:
b.go(f"{base}/")
- runs = b.wait("document.querySelectorAll('ul.runs a').length === 2 && "
- "Array.from(document.querySelectorAll('ul.runs a')).map(function (a) { return a.getAttribute('href'); }).join(' ')")
- check("browser: the runs list links the sample and the family", runs == f"runs/{S}/ runs/{FAM}/", runs)
+ people = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpPeople")
+ runs = b.eval("Array.from(document.querySelectorAll('.person-card a.person-link')).map(function (a) { "
+ "return a.getAttribute('href'); }).join(' ') + ' | ' + Array.from(document.querySelectorAll("
+ "'.family-head a')).map(function (a) { return a.getAttribute('href'); }).join(' ')")
+ check("browser: the home page shows both people of the family, the sequenced one with a run linked, and the family",
+ people == "2" and runs == f"runs/{S}/ | runs/{FAM}/ runs/{FAM}/", (people, runs))
b.go(f"{base}/runs/{S}/")
first = b.wait("document.documentElement.dataset.pgpFrame && document.documentElement.dataset.pgpStatus === 'ready' "
"&& document.querySelectorAll('#map g.box').length")
@@ -231,12 +260,77 @@ def browser_checks(out):
check("browser: the policy blocked nothing on the sample page", not blocked, blocked)
csp = b.eval("var m = document.querySelector('meta[http-equiv=Content-Security-Policy]'); m && m.content")
check("browser: the page runs under the meta policy", csp == build_live_demo.CSP, csp)
+ person_checks(b, base)
+ phone_checks(b, base)
finally:
b.close()
srv.shutdown()
srv.server_close()
+def person_checks(b, base):
+ """The person page at the end of the run: cards, keyboard, a section link; the family page's own view."""
+ b.eval("location.hash = '#q/clinical'; true")
+ n = b.wait("document.documentElement.dataset.pgpTab === 'clinical' && document.documentElement.dataset.pgpResults")
+ card = b.eval("var c = document.getElementById('card-clinvar'); c && [!!c.querySelector('.caveat .caveat-lead'), "
+ "!!c.querySelector('figure[data-chart]'), c.classList.contains('attn')].join(' ')")
+ check("browser: the doctor tab shows its cards; ClinVar's has its caveat, its chart and its flag",
+ n and int(n) > 3 and card == "true true true", (n, card))
+ b.eval("var t = document.querySelector('#rail [aria-selected=true]'); t.focus(); "
+ "t.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true })); true")
+ tab = b.wait("document.documentElement.dataset.pgpTab !== 'clinical' && document.documentElement.dataset.pgpTab")
+ check("browser: the arrow key moves to the next chapter", tab == "carrier", tab)
+ b.eval("location.hash = '#s/clinvar'; true")
+ focus = b.wait("document.documentElement.dataset.pgpTab === 'clinical' && document.activeElement && "
+ "document.activeElement.id")
+ check("browser: #s/clinvar opens the doctor tab and focuses the ClinVar card", focus == "card-clinvar", focus)
+ b.go(f"{base}/runs/{FAM}/")
+ fam = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && [!document.getElementById('map'), "
+ "!document.getElementById('report-link'), (document.querySelector('svg.pedigree-svg a') || {"
+ "getAttribute: function () { return null; } }).getAttribute('href')].join(' ')")
+ check("browser: the family page has a clickable pedigree, no step map and no report link",
+ fam == f"true true ../../runs/{S}/", fam)
+
+
+def too_wide(b):
+ return b.eval("document.documentElement.scrollWidth > window.innerWidth ? "
+ "document.documentElement.scrollWidth + ' > ' + window.innerWidth : ''")
+
+
+def phone_checks(b, base):
+ # mobile False: a mobile viewport zooms out to fit a page that overflows, which would hide the overflow.
+ b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False},
+ b.session)
+ try:
+ wide = {}
+ for name, path, ready in (("home", "/", "document.documentElement.dataset.pgpStatus === 'ready'"),
+ ("person", f"/runs/{S}/#q/clinical", "document.documentElement.dataset.pgpResults"),
+ ("family", f"/runs/{FAM}/", "document.documentElement.dataset.pgpStatus === 'ready'")):
+ b.go(base + path)
+ b.wait(ready)
+ if name == "person":
+ b.eval("var s = document.getElementById('replay-slider'); s.value = s.max; "
+ "s.dispatchEvent(new Event('input')); true")
+ b.wait("document.documentElement.dataset.pgpFrame === document.getElementById('replay-slider').max")
+ time.sleep(1)
+ # Every tab of the person page, not only the one the link opens.
+ tabs = b.eval("Array.prototype.map.call(document.querySelectorAll('[role=tab][data-tab]'), "
+ "function (t) { return t.getAttribute('data-tab'); }).join(' ')") or ""
+ for tab in tabs.split():
+ b.eval(f"document.querySelector('[role=tab][data-tab=\"{tab}\"]').click(); true")
+ time.sleep(0.3)
+ if too_wide(b):
+ wide[f"person {tab}"] = too_wide(b)
+ wide["person tabs"] = "" if len(tabs.split()) >= 3 else f"only {tabs!r}"
+ wide[name] = too_wide(b)
+ check("browser: at 390 px no page scrolls sideways", not any(wide.values()), wide)
+ b.eval("var d = document.createElement('div'); d.style.width = '900px'; d.style.height = '4px'; "
+ "document.querySelector('main').appendChild(d); true")
+ control("a 900 px wide block on the family page", not too_wide(b))
+ finally:
+ b.call("Emulation.clearDeviceMetricsOverride", {}, b.session)
+
+
def main():
work = tempfile.mkdtemp(prefix="test_live_demo.")
try:
@@ -289,6 +383,15 @@ def main():
planted = copy.deepcopy(built)
planted["family"]["state"]["plugins"]["family"]["pedigree"] = f"/family/{FAM}/pedigree.svg"
control("an absolute pedigree link", not link_problems(out, planted))
+ bad = json_link_problems(out)
+ check("5. every link of runs.json, results.json and family.json is relative and reaches a file", not bad, bad)
+ with open(os.path.join(out, "runs.json")) as f:
+ idx = json.load(f)
+ with open(os.path.join(out, "runs.json"), "w") as f:
+ json.dump(dict(idx, runs=[dict(r, url="/" + r["url"]) for r in idx["runs"]]), f)
+ control("an absolute link in runs.json", not json_link_problems(out))
+ with open(os.path.join(out, "runs.json"), "w") as f:
+ json.dump(idx, f)
# 6. the policy on every page, nothing from outside
pages, texts = [], {}
diff --git a/tests/test_live_results.py b/tests/test_live_results.py
new file mode 100644
index 00000000..b1a9bc5e
--- /dev/null
+++ b/tests/test_live_results.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""A person's results and a family's page on the live site: bin/live_results.py, the results plugin
+(bin/owngenome_live_results.py), /runs.json and the family plugin's family.json.
+
+Checks:
+ 1. render_report.SECTION_NOTES names a caveat for every section id of CHAPTERS,
+ and each name is a text constant of bin/render_report.py;
+ 2. results.json of DEMO-001 (tests/demo/make_demo_sample.py, collect_summary):
+ the seven chapters with every section id once, and each card's caveat is
+ its SECTION_NOTES constants, word for word, never empty;
+ 3. the wording gate: no string of results.json, of family.json
+ (tests/fixtures/families_live family_a) or of the page assets
+ (assets/live: the text of the pages, the string literals of the scripts)
+ says a result is fine (live_results.FORBIDDEN), a quoted tool term and a
+ caveat aside;
+ 4. ClinVar: with hits the card lists them in a table; with none it has no
+ table and its empty text says ClinVar lists only part of what causes
+ disease (never an all-clear);
+ 5. through the real server: /runs//results.json is behind the login
+ (401 without a session), answers the cards with one, and writes one access
+ log row per request; summary.json stays 404; /runs.json lists the run with
+ its progress and attention count, behind the login too;
+ 6. APOE: the Traits card's hidden text, behind its warning, is each APOE row
+ as the report prints it (trait, rsID, genotype, reading), never a dict;
+ 7. the adapters carry what the report says of a section, on small made-up
+ sections: the ancient PCA plot (base64 in the summary) as an image, the
+ Slivar phase lines, every Exomiser variant of a gene, a not-assessed
+ secondary-findings list, a carrier list not made and an unparsed PharmCAT
+ report as sentences (never zero counts), an unsized "normal" locus as not
+ sized, AnnotSV's exon-edge check, a genotype rule and a deep intronic call.
+Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's
+entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card
+and "All clear" in a script, 4 with the hits emptied, 5 with the results route
+registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing. A check
+that cannot fail is no check.
+
+Run: python3 tests/test_live_results.py
+"""
+import contextlib
+import base64
+import copy
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import urllib.parse
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(REPO, "bin"))
+sys.path.insert(0, os.path.join(REPO, "tests"))
+import collect_summary # noqa: E402
+import live_results # noqa: E402
+import owngenome_live # noqa: E402
+import owngenome_live_family # noqa: E402
+import render_report as rr # noqa: E402
+
+SAMPLE = "DEMO-001"
+ASSETS = os.path.join(REPO, "assets", "live")
+FAMILIES = os.path.join(REPO, "tests", "fixtures", "families_live")
+PASSWORD = "synthetic test password 7"
+FAILS = []
+
+
+def check(desc, ok, detail=""):
+ print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:800]}"))
+ if not ok:
+ FAILS.append(desc)
+
+
+def control(desc, still_passes):
+ check(f"control: {desc}", not still_passes, "the check passed on the planted mistake")
+
+
+# --- 1. the caveat of every section ---------------------------------------------------------
+
+def notes_problems(chapters, notes, module):
+ ids = [sid for _, _, _, secs in chapters for sid, _, _ in secs]
+ out = [f"{sid}: no caveat" for sid in ids if not notes.get(sid)]
+ out += [f"{sid}: {n} is not a text constant" for sid, names in notes.items() for n in names
+ if not isinstance(getattr(module, n, None), str) or not getattr(module, n).strip()]
+ out += [f"{sid}: no such section" for sid in notes if sid not in ids]
+ return out
+
+
+# --- 2. one card per section, its caveat word for word --------------------------------------
+
+def card_problems(res):
+ out = []
+ want = [sid for _, _, _, secs in rr.CHAPTERS for sid, _, _ in secs]
+ got = [c["id"] for ch in res["chapters"] for c in ch["sections"]]
+ if got != want:
+ out.append(f"sections {got} != {want}")
+ if [ch["key"] for ch in res["chapters"]] != [k for k, *_ in rr.CHAPTERS]:
+ out.append("chapters out of the report's order")
+ for ch in res["chapters"]:
+ for c in ch["sections"]:
+ exp = [getattr(rr, n) for n in rr.SECTION_NOTES.get(c["id"], ())]
+ if not c["caveat"] or c["caveat"] != exp:
+ out.append(f"{c['id']}: caveat {c['caveat']!r:.120}")
+ return out
+
+
+# --- 3. the wording gate --------------------------------------------------------------------
+
+def asset_text(path):
+ """What a page or script of assets/live puts on screen: the text of an HTML page, the string
+ literals of a script (its comments and code left out)."""
+ with open(path) as f:
+ t = f.read()
+ if path.endswith(".html"):
+ t = re.sub(r"<(script|style)\b.*?", " ", t, flags=re.S | re.I)
+ return [x.strip() for x in re.split(r"<[^>]*>", t) if x.strip()]
+ t = re.sub(r"/\*.*?\*/", " ", t, flags=re.S)
+ t = re.sub(r"(?m)^\s*//[^\n]*", " ", t)
+ return [m.group(2) for m in re.finditer(r"(['\"])((?:\\.|(?!\1).)*)\1", t)]
+
+
+def asset_problems(texts):
+ out = []
+ for name, items in texts.items():
+ for s in items:
+ if live_results.FORBIDDEN.search(re.sub(r'"[^"]*"', "", s)):
+ out.append(f"{name}: {s[:120]}")
+ return out
+
+
+# --- 4. ClinVar with and without hits -------------------------------------------------------
+
+def clinvar_card(summary):
+ res = live_results.build(SAMPLE, summary)
+ return next(c for ch in res["chapters"] for c in ch["sections"] if c["id"] == "clinvar")
+
+
+def clinvar_problems(with_hits, without):
+ out = []
+ n = len(with_hits["table"]["rows"]) if with_hits.get("table") else 0
+ if not n:
+ out.append("hits but no table")
+ if without.get("table"):
+ out.append("no hit but a table")
+ if "lists only part" not in (without.get("empty_text") or ""):
+ out.append(f"empty text {without.get('empty_text')!r}")
+ return out
+
+
+# --- 6. APOE behind its warning -------------------------------------------------------------
+
+APOE_ROW = {"rsid": "rs429358", "gene": "APOE", "trait": "APOE e4 marker", "genotype": "T/C",
+ "status": "called", "reading": "one copy of the e4 allele", "ancestry": "", "note": "", "pmids": "1"}
+
+
+def apoe_problems(hidden):
+ out = []
+ text = (hidden or {}).get("text") or ""
+ if "{" in text or "'rsid'" in text:
+ out.append(f"a dict in the text {text!r}")
+ wants = [APOE_ROW["trait"], APOE_ROW["rsid"], APOE_ROW["genotype"], APOE_ROW["reading"]]
+ for want in wants:
+ if want not in text:
+ out.append(f"{want!r} missing from {text!r}")
+ return out
+
+
+# --- 7. the adapters on made-up sections -----------------------------------------------------
+
+def adapter_sections():
+ sv = {"type": "DEL", "sv": "chr1:100-5000", "gt": "0/1", "length": 4900, "acmg_class": 5, "genes": ["GENE1"],
+ "exon_edge": "yes", "exon_edge_detail": "GENE1 exon 2", "population_af": None, "population_source": "",
+ "common": False, "support": None}
+ return {
+ "ancient": {"state": "ok", "data": {"nearest": [], "sites_used": 1, "panel_sites": 2,
+ "plot_svg": base64.b64encode(b"").decode()}},
+ "slivar": {"state": "ok", "data": {"prioritized": 2, "compound_het_variants": 2, "compound_het_genes": 1,
+ "compound_het_pairs": 1, "compound_het_phase": {"trans": 1},
+ "compound_het_trans": [{"gene": "GENE2", "variants": "v1 v2", "reads": 3}]}},
+ "exomiser": {"state": "ok", "data": {"genes": [{"rank": 1, "gene": "GENE3", "variants": [
+ {"hgvs": "c.1A>G"}, {"hgvs": "c.2C>T"}]}]}},
+ "clinical": {"state": "ok", "data": {"acmg_sf": {"unavailable": "no VEP output"}}},
+ "carrier": {"state": "ok", "data": {"unavailable": "no ClinVar screen"}},
+ "cpic": {"state": "ok", "data": {"parse_failed": True, "genes": [], "warnings": []}},
+ "expansions": {"state": "ok", "data": {"stranger": True, "loci": [
+ {"locus": "LOCUS1", "repeat_count": "20/30", "status": "normal", "sizing": "flanking_only"}]}},
+ "annotsv": {"state": "ok", "data": {"count": 1, "rows": [sv]}},
+ "traits": {"state": "ok", "data": {"rows": [], "rules": [
+ {"trait": "Rule trait", "gene": "GENE4", "rsids": "rs1,rs2", "status": "called", "result": "AB",
+ "phase": "read_backed", "reading": "rule reading"}]}},
+ "clinvar": {"state": "ok", "data": {"hits": [], "deep_intronic": [
+ {"gene": "GENE5", "hgvs": "c.100+500A>G", "chrom": "chr1", "pos": 1, "ref": "A", "alt": "G",
+ "call": "het", "detail": "GQ 40"}]}},
+ }
+
+
+def adapter_problems(adapters):
+ S, out = adapter_sections(), []
+ def run(name, sec=None):
+ try:
+ return adapters[name](S, {"sections": S}) or {}
+ except Exception as e: # a crash is a problem, not a test error
+ out.append(f"{name}: {type(e).__name__} {e}")
+ return {}
+ text = lambda got: json.dumps(got) # noqa: E731
+ if not (run("ancient").get("image") or {}).get("src", "").endswith(S["ancient"]["data"]["plot_svg"]):
+ out.append("ancient: no plot image")
+ if "in trans on 3 reads" not in text(run("slivar")):
+ out.append("slivar: no phase lines")
+ if len((run("exomiser").get("table") or {}).get("rows") or []) != 2:
+ out.append("exomiser: not every variant")
+ got = run("clinical_filter")
+ if "not assessed" not in text(got) or "ClinVar P/LP in its genes" in text(got):
+ out.append(f"clinical: not-assessed not said {got}")
+ got = run("carrier")
+ if "not made" not in text(got) or got.get("facts"):
+ out.append(f"carrier: not-made not said {got}")
+ got = run("cpic")
+ if "could not be parsed" not in text(got) or got.get("facts"):
+ out.append(f"cpic: parse failure not said {got}")
+ rows = (run("expansions").get("table") or {}).get("rows") or [[]]
+ if "not sized" not in rows[0]:
+ out.append(f"expansions: unsized locus shown as {rows[0]}")
+ if "on exon edges" not in text(run("structural")):
+ out.append("structural: no exon-edge check")
+ if "Rule trait" not in text(run("traits")):
+ out.append("traits: no genotype rule")
+ if "GENE5" not in text(run("clinvar")):
+ out.append("clinvar: no deep intronic call")
+ return out
+
+
+# --- 5. the server --------------------------------------------------------------------------
+
+def request(port, method, path, body=None, headers=None):
+ import http.client
+ c = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
+ h = {"Host": f"127.0.0.1:{port}"}
+ h.update(headers or {})
+ c.request(method, path, body=body, headers=h)
+ r = c.getresponse()
+ data = r.read()
+ hdr = {k.lower(): v for k, v in r.getheaders()}
+ c.close()
+ return r.status, hdr, data
+
+
+def serve(root, state, pwfile, open_results=False):
+ app = owngenome_live.App()
+ if open_results: # the planted mistake: the results route without the login
+ route = app.route
+ app.route = lambda m, p, fn, auth=True: route(m, p, fn, auth and "results" not in p)
+ with contextlib.redirect_stderr(io.StringIO()):
+ live, web, ingest = owngenome_live.start(["--root", root, "--state-dir", state, "--port", "0", "--ingest-port", "0",
+ "--password-file", pwfile, "--quiet"], app)
+ threading.Thread(target=web.serve_forever, daemon=True).start()
+ return live, web, ingest
+
+
+def stop(srv):
+ live, web, ingest = srv
+ live.stopping = True
+ web.shutdown()
+ web.server_close()
+ ingest.server_close()
+
+
+def server_problems(port, state):
+ out = []
+ path = f"/runs/{SAMPLE}/results.json"
+ code, _, _ = request(port, "GET", path)
+ if code != 401:
+ out.append(f"results.json without a session: {code}")
+ code, _, _ = request(port, "GET", "/runs.json")
+ if code != 401:
+ out.append(f"runs.json without a session: {code}")
+ code, h, _ = request(port, "POST", "/login", urllib.parse.urlencode({"password": PASSWORD, "next": "/"}),
+ {"Content-Type": "application/x-www-form-urlencoded"})
+ m = re.match(r"owngenome_live=([^;]+)", h.get("set-cookie", ""))
+ if not m:
+ return out + [f"login {code}"]
+ ck = {"Cookie": f"owngenome_live={m.group(1)}"}
+ log = os.path.join(state, "access.log")
+ before = open(log).read().count("\tresults\t") if os.path.isfile(log) else 0
+ code, h, body = request(port, "GET", path, headers=ck)
+ res = json.loads(body) if code == 200 else {}
+ if code != 200 or len(res.get("chapters") or []) != 7 or res.get("sample") != SAMPLE:
+ out.append(f"results.json with a session: {code} {str(res)[:200]}")
+ after = open(log).read().count("\tresults\t") if os.path.isfile(log) else 0
+ if after - before != 1:
+ out.append(f"{after - before} access log rows for one results.json")
+ code, _, _ = request(port, "GET", f"/runs/{SAMPLE}/summary.json", headers=ck)
+ if code != 404:
+ out.append(f"summary.json served: {code}")
+ code, _, body = request(port, "GET", "/runs.json", headers=ck)
+ idx = json.loads(body) if code == 200 else {}
+ run = next((r for r in idx.get("runs") or [] if r["name"] == SAMPLE), None)
+ if not run or run["url"] != f"/runs/{SAMPLE}/" or not run.get("results") or run.get("attention") != res.get("attention"):
+ out.append(f"runs.json: {code} {run}")
+ return out
+
+
+def main():
+ work = tempfile.mkdtemp(prefix="test_live_results.")
+ srvs = []
+ try:
+ # 1.
+ bad = notes_problems(rr.CHAPTERS, rr.SECTION_NOTES, rr)
+ check(f"1. every section id has its caveat ({len(rr.SECTION_NOTES)} ids)", not bad, bad)
+ planted = {k: v for k, v in rr.SECTION_NOTES.items() if k != "vntyper"}
+ control("VNTYPER's caveat entry dropped", not notes_problems(rr.CHAPTERS, planted, rr))
+
+ # 2.
+ root = os.path.join(work, "root")
+ os.makedirs(root)
+ subprocess.run([sys.executable, os.path.join(REPO, "tests", "demo", "make_demo_sample.py"), "--genome-dir", root,
+ "--records", "2000"], check=True, stdout=subprocess.DEVNULL)
+ d = os.path.join(root, SAMPLE)
+ summary = collect_summary.collect(SAMPLE, d)
+ with open(os.path.join(d, "summary.json"), "w") as f:
+ json.dump(summary, f)
+ res = live_results.build(SAMPLE, summary)
+ bad = card_problems(res)
+ check("2. results.json: the report's chapters, one card per section, each with its caveats word for word",
+ not bad, bad)
+ blank = copy.deepcopy(res)
+ blank["chapters"][3]["sections"][0]["caveat"] = []
+ control("one card's caveat blanked", not card_problems(blank))
+
+ # 3.
+ bad = live_results.wording_problems(res)
+ check("3. no string of results.json reads as an all-clear", not bad, bad)
+ fam = owngenome_live_family.family_page("family_a", os.path.join(FAMILIES, "family_a"))
+ bad = live_results.wording_problems(fam)
+ check("3. no string of family.json reads as an all-clear", not bad and fam.get("layout"), bad)
+ planted = copy.deepcopy(res)
+ planted["chapters"][0]["sections"][0]["headline"] = "Coverage: ok"
+ control("'ok' planted in a card headline", not live_results.wording_problems(planted))
+ texts = {}
+ for p, _, fs in os.walk(ASSETS):
+ for f in fs:
+ if f.endswith((".html", ".js")):
+ texts[os.path.relpath(os.path.join(p, f), ASSETS)] = asset_text(os.path.join(p, f))
+ bad = asset_problems(texts)
+ check(f"3. no page or script text of assets/live reads as an all-clear ({len(texts)} files)",
+ not bad and len(texts) >= 10, bad)
+ planted = dict(texts, **{"person.js": texts["person.js"] + ["All clear: nothing to see"]})
+ control("'All clear' planted in person.js", not asset_problems(planted))
+
+ # 4.
+ none = copy.deepcopy(summary)
+ cv = none["sections"]["clinvar"]["data"]
+ cv["hits"], cv["count"], cv["by_stars"] = [], 0, {}
+ with_hits, without = clinvar_card(summary), clinvar_card(none)
+ bad = clinvar_problems(with_hits, without)
+ check("4. ClinVar: a table with hits; without, the sentence that ClinVar lists only part", not bad, bad)
+ control("the demo's hits emptied on both sides", not clinvar_problems(without, without))
+
+ # 6.
+ S = {"traits": {"data": {"rows": [], "called": 0, "not_covered": 0, "apoe": [APOE_ROW]}}}
+ hidden = live_results.a_traits(S, SAMPLE).get("hidden")
+ bad = apoe_problems(hidden)
+ check("6. APOE: the hidden text reads each row as the report does, never a dict", not bad, bad)
+ control("the APOE row printed as its Python dict", not apoe_problems({"text": str(APOE_ROW)}))
+
+ # 7.
+ names = {"ancient": "a_ancient", "slivar": "a_slivar", "exomiser": "a_exomiser",
+ "clinical_filter": "a_clinical_filter", "carrier": "a_carrier", "cpic": "a_cpic",
+ "expansions": "a_expansions", "structural": "a_structural", "traits": "a_traits",
+ "clinvar": "a_clinvar"}
+ adapters = {k: getattr(live_results, v) for k, v in names.items()}
+ bad = adapter_problems(adapters)
+ check("7. the adapters carry the report's plot, phase, variants, rules and not-assessed sentences", not bad, bad)
+ control("the Slivar adapter returning nothing", not adapter_problems(dict(adapters, slivar=lambda S, s: {})))
+
+ # 5.
+ pwfile = os.path.join(work, "password")
+ with open(pwfile, "w") as f:
+ f.write(owngenome_live.hash_password(PASSWORD) + "\n")
+ state = os.path.join(work, "state")
+ srv = serve(root, state, pwfile)
+ srvs.append(srv)
+ bad = server_problems(srv[1].server_address[1], state)
+ check("5. results.json and runs.json behind the login, one log row, summary.json 404", not bad, bad)
+ state2 = os.path.join(work, "state-open")
+ srv = serve(root, state2, pwfile, open_results=True)
+ srvs.append(srv)
+ control("the results route registered without the login",
+ not server_problems(srv[1].server_address[1], state2))
+ finally:
+ for s in srvs:
+ stop(s)
+ shutil.rmtree(work, ignore_errors=True)
+ if FAILS:
+ print(f"\n{len(FAILS)} check(s) failed")
+ return 1
+ print("\nAll checks passed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_output_contract.py b/tests/test_output_contract.py
index 38c6de8a..8e896754 100644
--- a/tests/test_output_contract.py
+++ b/tests/test_output_contract.py
@@ -5,9 +5,14 @@
run-all.sh switch and published Nextflow file has its row, and every row
still matches the code;
2. static controls, each on a copy of the code: a renamed published file, a
- removed param, a removed switch, a new --tools name with no row and a new
- published file with no row are each refused, and the stable ones are
- named BREAKING;
+ removed param, a removed switch, a new --tools name with no row, a new
+ published file with no row and a schema default that is not
+ nextflow.config's are each refused, and the stable ones are named
+ BREAKING; so are, on a copy of the contract, a file row in the folder of
+ a publishDir whose saveAs drops it and a nextflow row whose step script
+ writes the same path (also on a line of seven shell variables);
+ --pedigree is a declared param; a saveAs closure is read up to its own
+ brace and a script path counts only as a whole path;
3. the run check on a planted results folder made from the contract's own
e2e rows passes for both legs; then a dropped column, a dropped VCF
field, a dropped JSON key and a renamed file are each refused, naming the
@@ -63,11 +68,30 @@ def code_copy(tmp):
dst = os.path.join(tmp, "repo")
shutil.copytree(os.path.join(REPO, "modules"), os.path.join(dst, "modules"))
os.makedirs(os.path.join(dst, "scripts"))
- for f in ("main.nf", "nextflow_schema.json", os.path.join("scripts", "run-all.sh")):
+ for f in ("main.nf", "nextflow_schema.json", "nextflow.config"):
shutil.copy(os.path.join(REPO, f), os.path.join(dst, f))
+ for f in os.listdir(os.path.join(REPO, "scripts")):
+ if f.endswith(".sh"):
+ shutil.copy(os.path.join(REPO, "scripts", f), os.path.join(dst, "scripts", f))
return dst
+def contract_copy(tmp, name, change):
+ """The contract with one row changed: change(cells) edits the cells of the file row called name."""
+ path = os.path.join(tmp, "contract.tsv")
+ hit = False
+ with open(CONTRACT) as f, open(path, "w") as g:
+ for line in f:
+ c = line.rstrip("\n").split("\t")
+ if c[0] == "file" and c[1] == name:
+ change(c)
+ line = "\t".join(c) + "\n"
+ hit = True
+ g.write(line)
+ assert hit, f"no file row {name}"
+ return path
+
+
def edit(path, old, new, count=1):
src = open(path).read()
assert old in src, f"{old!r} not in {path}"
@@ -96,6 +120,9 @@ def static_controls():
lambda d: edit(os.path.join(d, "modules", "local", "roh", "main.nf"),
'path("*_roh.txt"),', 'path("*_roh.txt"),\n tuple val(meta), path("*_roh_extra.tsv"), emit: extra\n'),
[r"ROH .* writes \*_roh_extra\.tsv and has no contract row"]),
+ ("a schema default that is not nextflow.config's (traits left out of --tools)",
+ lambda d: edit(os.path.join(d, "nextflow_schema.json"), "mito_variants,traits,html_report", "mito_variants,html_report"),
+ [r"param tools: nextflow_schema\.json's default .* is not nextflow\.config's"]),
]
for desc, change, want in cases:
tmp = tempfile.mkdtemp()
@@ -109,6 +136,39 @@ def static_controls():
shutil.rmtree(tmp)
+ def to_aligned(c):
+ c[1] = "{sample}/aligned/{sample}_read_count.tsv"
+
+ def to_nextflow(c):
+ c[4] = "nextflow"
+
+ contract_cases = [
+ ("the read count in aligned/, whose publishDir drops it (it is published to qc/)",
+ "{sample}/qc/{sample}_read_count.tsv", to_aligned,
+ [r"BREAKING: the stable file \{sample\}/aligned/\{sample\}_read_count\.tsv .* is not what ALIGN_MINIMAP2 publishes"]),
+ ("legs nextflow on a file step 49 writes under the same name",
+ "{sample}/chip/{sample}_chip.tsv", to_nextflow,
+ [r"\{sample\}/chip/\{sample\}_chip\.tsv .* has legs nextflow, but scripts/49-chip\.sh writes it"]),
+ ("legs nextflow on a file step 50 writes on a line of seven shell variables",
+ "{sample}/mei/{sample}_megane.log", to_nextflow,
+ [r"\{sample\}/mei/\{sample\}_megane\.log .* has legs nextflow, but scripts/50-mei\.sh writes it"]),
+ ]
+ for desc, name, change, want in contract_cases:
+ tmp = tempfile.mkdtemp()
+ try:
+ rc, out = run("--static", "--contract", contract_copy(tmp, name, change))
+ missing = [w for w in want if not re.search(w, out)]
+ check(f"static control refused: {desc}", rc == 1 and not missing, f"rc={rc}, missing {missing}: {out[-1500:]}")
+ finally:
+ shutil.rmtree(tmp)
+
+ config = open(os.path.join(REPO, "nextflow.config")).read()
+ schema = json.load(open(os.path.join(REPO, "nextflow_schema.json")))
+ in_schema = any("pedigree" in g.get("properties", {}) for g in schema["definitions"].values())
+ check("--pedigree is declared in nextflow.config and nextflow_schema.json",
+ re.search(r"^\s*pedigree\s*=\s*null\b", config, re.M) is not None and in_schema)
+
+
def _drop_param(d, name):
path = os.path.join(d, "nextflow_schema.json")
s = json.load(open(path))
@@ -210,6 +270,29 @@ def run_controls():
shutil.rmtree(tmp)
+def reader_controls():
+ """The two readers the static check is built on, on inputs the code does not have yet."""
+ import importlib.util
+ spec = importlib.util.spec_from_file_location("contract_check", CHECK)
+ m = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(m)
+ stmt = "publishDir \"x\", saveAs: { fn -> fn.endsWith('.log') ? null : fn }, enabled: { true }"
+ body = m.closure_body(stmt)
+ check("a saveAs closure ends at its own brace (an option after it is not part of it)",
+ body is not None and "enabled" not in body and m.save_as(body, "a.log") == "drop", repr(body))
+ tmp = tempfile.mkdtemp()
+ try:
+ os.makedirs(os.path.join(tmp, "scripts"))
+ open(os.path.join(tmp, "scripts", "11-roh.sh"), "w").write(
+ 'S="${GENOME_DIR}/${SAMPLE}"\nbgzip -c x > "${S}/roh/${SAMPLE}_roh.txt.gz"\n')
+ short = m.bash_writes(tmp, "11", "{sample}/roh/{sample}_roh.txt", {})
+ full = m.bash_writes(tmp, "11", "{sample}/roh/{sample}_roh.txt.gz", {})
+ check("a step script writing x.txt.gz does not count as writing x.txt",
+ short is None and full == "scripts/11-roh.sh", f"{short!r} {full!r}")
+ finally:
+ shutil.rmtree(tmp)
+
+
def malformed():
tmp = tempfile.mkdtemp()
try:
@@ -228,6 +311,7 @@ def malformed():
static_controls()
+reader_controls()
run_controls()
malformed()
print(f"{FAILS} failed")