From 7affb8ab9807a53ea945fc556018e4a2f8fdcb1f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:49:03 +0100 Subject: [PATCH 01/10] perf(live): parse each trace file once until it changes, not on every /runs.json Bead: pgp-5bja --- bin/run_state.py | 63 ++++++++++---- tests/test_run_state_trace_cache.py | 125 ++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 18 deletions(-) create mode 100644 tests/test_run_state_trace_cache.py diff --git a/bin/run_state.py b/bin/run_state.py index a070e6d2..9e6764d6 100755 --- a/bin/run_state.py +++ b/bin/run_state.py @@ -167,27 +167,54 @@ def parse_duration(v): return total if found else None +# Parsed trace files, keyed on the path and its mtime and size: /runs.json builds every run's state, and a +# finished run's trace files do not change, so each is read once until it does. Bounded, oldest out first. +TRACE_CACHE = {} +TRACE_CACHE_MAX = 64 + + +def _trace_rows(p): + """[(process, tag, task)] of one trace file, every sample's; None when it cannot be read.""" + try: + st = os.stat(p) + except OSError: + return None + key = (st.st_mtime_ns, st.st_size) + hit = TRACE_CACHE.get(p) + if hit and hit[0] == key: + return hit[1] + try: + f = open(p, errors="replace") + except OSError: + return None + rows = [] + with f: + header = f.readline().rstrip("\n").split("\t") + for line in f: + r = dict(zip(header, line.rstrip("\n").split("\t"))) + proc, tag = task_name(r.get("name")) + if not proc or proc in SHARED_PROCESSES: + continue + start = parse_trace_time(r.get("submit")) + dur = parse_duration(r.get("duration")) + rows.append((proc, tag, {"id": r.get("task_id") or r.get("hash") or r.get("name"), "name": r.get("name"), + "process": proc, "status": (r.get("status") or "").upper(), + "exit": r.get("exit"), "start": start, + "end": start + dur if start is not None and dur is not None else None})) + TRACE_CACHE.pop(p, None) + TRACE_CACHE[p] = (key, rows) + while len(TRACE_CACHE) > TRACE_CACHE_MAX: + TRACE_CACHE.pop(next(iter(TRACE_CACHE))) + return rows + + def read_trace(paths, sample): - """Rows of the trace files tagged with the sample, as task dicts.""" + """Rows of the trace files tagged with the sample, as task dicts (each a copy: callers may change it).""" tasks = [] for p in paths: - try: - f = open(p, errors="replace") - except OSError: - continue - with f: - header = f.readline().rstrip("\n").split("\t") - for line in f: - r = dict(zip(header, line.rstrip("\n").split("\t"))) - proc, tag = task_name(r.get("name")) - if not proc or proc in SHARED_PROCESSES or not is_sample(tag, sample): - continue - start = parse_trace_time(r.get("submit")) - dur = parse_duration(r.get("duration")) - tasks.append({"id": r.get("task_id") or r.get("hash") or r.get("name"), "name": r.get("name"), - "process": proc, "status": (r.get("status") or "").upper(), - "exit": r.get("exit"), "start": start, - "end": start + dur if start is not None and dur is not None else None}) + for proc, tag, task in _trace_rows(p) or (): + if is_sample(tag, sample): + tasks.append(dict(task)) return tasks diff --git a/tests/test_run_state_trace_cache.py b/tests/test_run_state_trace_cache.py new file mode 100644 index 00000000..bc997004 --- /dev/null +++ b/tests/test_run_state_trace_cache.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""bin/run_state.py read_trace: a trace file is parsed once until it changes. + +/runs.json builds the state of every run on the server, and each build read every +trace file of every run again. read_trace now keeps the parsed rows of each file, +keyed on its path, mtime and size. Checks: + 1. the second read of an unchanged trace file opens nothing and gives the same rows; + 2. a file that grew (a task row appended) is read again and gives the new row; + 3. the rows handed out are copies: a caller that changes one does not change the cache; + 4. two samples of one trace file each get only their own rows from the one parse; + 5. the cache keeps at most TRACE_CACHE_MAX files. +Check 1 is also run against a planted mistake (a cache that keeps nothing) and must +fail there: a check that cannot fail is no check. + +Run: python3 tests/test_run_state_trace_cache.py +""" +import builtins +import os +import shutil +import sys +import tempfile + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "bin")) +import run_state # noqa: E402 + +FIX = os.path.join(REPO, "tests", "fixtures", "run_state", "trace.txt") +FAILS = [] + + +def check(desc, ok, detail=""): + print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:600]}")) + if not ok: + FAILS.append(desc) + + +def opens_of(path, fn): + """(result of fn(), how many times fn opened path).""" + n, real = [0], builtins.open + + def counting(file, *a, **k): + if os.path.realpath(str(file)) == os.path.realpath(path): + n[0] += 1 + return real(file, *a, **k) + builtins.open = counting + try: + out = fn() + finally: + builtins.open = real + return out, n[0] + + +def second_read_opens(trace): + run_state.TRACE_CACHE.clear() + first = run_state.read_trace([trace], "demo1") + again, n = opens_of(trace, lambda: run_state.read_trace([trace], "demo1")) + return first, again, n + + +def main(): + work = tempfile.mkdtemp(prefix="test_trace_cache.") + try: + trace = os.path.join(work, "trace.txt") + shutil.copy(FIX, trace) + + # 1. an unchanged file is not opened again + first, again, n = second_read_opens(trace) + check("1. the second read of an unchanged trace file opens nothing and gives the same rows", + n == 0 and first == again and len(first) > 0, (n, len(first), len(again))) + + # planted mistake: a cache that keeps nothing + keep = run_state.TRACE_CACHE_MAX + run_state.TRACE_CACHE_MAX = 0 + try: + _, _, n0 = second_read_opens(trace) + finally: + run_state.TRACE_CACHE_MAX = keep + check("control: a cache that keeps nothing fails check 1", n0 > 0, n0) + + # 2. a file that grew is read again + run_state.TRACE_CACHE.clear() + before = run_state.read_trace([trace], "demo1") + with open(trace, "a") as f: + f.write("99\tzz/000099\t9999\tPGX:CLINVAR_SCREEN (demo1)\tCOMPLETED\t0\t2026-01-02 11:00:00.000\t5s\t" + "4s\t50.0%\t1 MB\t1 MB\t1 MB\t1 MB\n") + after, n = opens_of(trace, lambda: run_state.read_trace([trace], "demo1")) + check("2. a trace file that grew is read again and gives the new row", + n == 1 and len(after) == len(before) + 1 and after[-1]["id"] == "99", (n, len(before), len(after))) + + # 3. copies + rows = run_state.read_trace([trace], "demo1") + rows[0]["status"] = "CHANGED BY A CALLER" + rows2 = run_state.read_trace([trace], "demo1") + check("3. a caller changing a row does not change the next read", rows2[0]["status"] != "CHANGED BY A CALLER", + rows2[0]) + + # 4. two samples from one parse + with open(trace, "a") as f: + f.write("100\tzz/000100\t10000\tPGX:CLINVAR_SCREEN (demo2)\tCOMPLETED\t0\t2026-01-02 11:00:00.000\t5s\t" + "4s\t50.0%\t1 MB\t1 MB\t1 MB\t1 MB\n") + run_state.TRACE_CACHE.clear() + one = run_state.read_trace([trace], "demo1") + two, n = opens_of(trace, lambda: run_state.read_trace([trace], "demo2")) + check("4. a second sample of the same trace file gets its own rows from the one parse", + n == 0 and [r["id"] for r in two] == ["100"] and "100" not in [r["id"] for r in one], (n, two)) + + # 5. bounded + run_state.TRACE_CACHE.clear() + for i in range(run_state.TRACE_CACHE_MAX + 5): + p = os.path.join(work, f"t{i}.txt") + shutil.copy(FIX, p) + run_state.read_trace([p], "demo1") + check("5. the cache keeps at most TRACE_CACHE_MAX files", len(run_state.TRACE_CACHE) == run_state.TRACE_CACHE_MAX, + len(run_state.TRACE_CACHE)) + finally: + shutil.rmtree(work, ignore_errors=True) + if FAILS: + print(f"\n{len(FAILS)} check(s) failed") + return 1 + print("\nAll checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From d678c916506bdade26786034783dd01d979f0833 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:56:23 +0100 Subject: [PATCH 02/10] docs(live): describe the person page, the family page and the card badges, and list --pedigree with the schema Bead: pgp-5bja --- README.md | 4 ++-- docs/parameters.md | 4 ++-- docs/recheck.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cc4b123b..aa77ab96 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ This pipeline takes raw sequencing data (FASTQ/BAM/VCF) from any vendor and runs - One Nextflow DSL2 pipeline from FASTQ, BAM or VCF to the report, and every step also as a bash script you can run on its own. - Optional family analysis of two or more processed relatives (step 38, a pedigree file): a GLnexus joint call, a somalier check that the samples are the people the pedigree says, which allele a child got from each parent, de novo candidates with both parents, and compound heterozygotes in trans or in cis ([multi-sample](https://geiserx.github.io/OwnGenome/multi-sample/#family-analysis-step-38-opt-in)). - Optional couple carrier screen for two partners of the pedigree (step 39): the recessive genes where both carry a change (ClinVar hits, deletions, SMN1 copy loss, FMR1 and FXN repeats) with the chance for a child, the X-linked findings, and what the screen does not see ([step 39](https://geiserx.github.io/OwnGenome/39-couple/)). -- On the live site, a page per family: the pedigree drawn from the family file, each child's chromosome map, the relationship check, the shared segments and the couple screen ([family page](https://geiserx.github.io/OwnGenome/multi-sample/#on-the-live-site)). +- On the live site, a page per family: a pedigree where each sequenced person opens their own page, the couple screen, who is related how as a grid, each child's stretches painted by parent, the de novo and compound-het candidates, and the members' medicines genes and haplogroups side by side ([family page](https://geiserx.github.io/OwnGenome/multi-sample/#on-the-live-site)). - Ends in an HTML report and a MultiQC summary. Alternative callers (GATK, FreeBayes, Strelka2, Octopus, BWA-MEM2, TIDDIT, GRIDSS) are there for benchmarking. ## Quick start @@ -66,7 +66,7 @@ The full documentation is at **https://geiserx.github.io/OwnGenome/**, one page - [Vendor compatibility guide](https://geiserx.github.io/OwnGenome/vendor-guide/): what each provider delivers and how to get it - [Pipeline overview](https://geiserx.github.io/OwnGenome/pipeline-overview/): every step with its tool and image, which ones a default run includes, and the page for each step - [Nextflow](https://geiserx.github.io/OwnGenome/nextflow/): the workflow runner, parallel steps and resume -- [Watch a run live](https://geiserx.github.io/OwnGenome/live-site/): a step map in the browser while the run goes, behind a password. The [demo](https://geiserx.github.io/OwnGenome/demo/) replays an invented run +- [Watch a run live](https://geiserx.github.io/OwnGenome/live-site/): a results page per person, laid out by the report's chapters, a page per family, and the step map in a Progress tab while the run goes, behind a password. The [demo](https://geiserx.github.io/OwnGenome/demo/) replays an invented run - [Interpreting your results](https://geiserx.github.io/OwnGenome/interpreting-results/): what each report means and what to do with it - [Multi-sample comparison](https://geiserx.github.io/OwnGenome/multi-sample/): partners, siblings, parents - [Long-read guide](https://geiserx.github.io/OwnGenome/long-read-guide/): Nanopore and PacBio HiFi diff --git a/docs/parameters.md b/docs/parameters.md index 41231f7c..021425f7 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -1,6 +1,6 @@ # Nextflow Parameters -Every parameter of the Nextflow pipeline (`main.nf`), grouped as in [`nextflow_schema.json`](https://github.com/GeiserX/OwnGenome/blob/main/nextflow_schema.json), with its default and the description the schema gives. `--pedigree` is read by `main.nf` but is not in the schema yet; its row is written here. Give one as `-- ` to `nextflow run`, or after the sex to `run-all.sh`, which hands the options after the sex to `nextflow run` unchanged. `run-all.sh` already passes the databases it finds under `GENOME_DIR` ([Full run](getting-started.md#full-run)). How a run is built: [Nextflow Execution](nextflow.md). +Every parameter of the Nextflow pipeline (`main.nf`), grouped as in [`nextflow_schema.json`](https://github.com/GeiserX/OwnGenome/blob/main/nextflow_schema.json), with its default and the description the schema gives. Give one as `-- ` to `nextflow run`, or after the sex to `run-all.sh`, which hands the options after the sex to `nextflow run` unchanged. `run-all.sh` already passes the databases it finds under `GENOME_DIR` ([Full run](getting-started.md#full-run)). How a run is built: [Nextflow Execution](nextflow.md). A default of none means the parameter is unset; each description says what a run does without it. @@ -11,7 +11,6 @@ Define where the pipeline should find input data and save output data. | Parameter | Default | What it does | |---|---|---| | `--input` | none | Path to comma-separated samplesheet: sample, then fastq_1,fastq_2 (aligned and called), or bam,bam_index (called), or vcf,vcf_index (a VCF with one sample) with an optional bam,bam_index; sex (male or female) is required on every row that is called and when expansion_hunter is enabled. A row can also give cram,crai instead of the BAM, and gvcf,gvcf_index beside the VCF; every column: [Samplesheet Format](nextflow.md#samplesheet-format). | -| `--pedigree` | none | The family workflow (step 38, and step 39 for two samples the pedigree marks as partners): a tab-separated pedigree (sample, sex, father, mother, partner; `assets/pedigree.example.tsv`) or a families file with a leading family column (`assets/families.example.tsv`), naming samplesheet rows. Needs `--somalier_sites` and at least two of the named samples. [Family workflow](nextflow.md) and [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in) say what it runs. | | `--outdir` | `./results` | The output directory where results will be saved. | | `--allow_unfiltered` | `false` | Accept a VCF that has no FILTER=PASS record by treating FILTER '.' as PASS. Without it such a VCF stops the run, because every PASS-only step would report zero hits. | | `--publish_dir_mode` | `copy` | Method used to save pipeline results to output directory. One of: `symlink`, `rellink`, `link`, `copy`, `copyNoFollow`, `move`. | @@ -95,6 +94,7 @@ Paths to clinical and population genetics reference data. | `--yleaf_data` | none | Yleaf's data folder (GRCh38 marker positions and haplogroup tree), which `scripts/setup.sh --yleaf-data` installs as reference/yleaf-<version>/data: the Yleaf image ships the code only. Its ytree/ subfolder, from the same command, holds the current YFull YTree the step also places the sample on. Required with y_haplogroup. | | `--hpo` | none | The symptoms of one person as Human Phenotype Ontology ids, comma separated (HP:0001250,HP:0001263; look them up at https://hpo.jax.org). Setting it runs exomiser (step 48), which ranks the genes and variants that could explain them; it needs --exomiser_data. | | `--exomiser_data` | none | Exomiser's data folder, which `scripts/setup.sh --exomiser-data` installs as reference/exomiser-<release> (its <release>_hg38 and <release>_phenotype folders). Required with --hpo. | +| `--pedigree` | none | Opt-in: run the family workflow (steps 38 and 39) on the samples of the samplesheet this file names: a pedigree (assets/pedigree.example.tsv), or a families file with a leading family column (assets/families.example.tsv). A families file that holds more than one family needs --family_name, and only that family runs. The family that runs needs two or more of those samples, each with a gVCF, and --somalier_sites. | | `--family_name` | none | With --pedigree: the family to run when the file is a families file (a pedigree with a leading family column, assets/families.example.tsv) that holds more than one family, and the family's folder under --outdir. Without it the folder is family/. | | `--exomiser_proband` | none | The sample the --hpo terms describe. Needed when the samplesheet has more than one sample. | | `--exomiser_assembly` | `hg38` | The assembly of the Exomiser data. Keep hg38: hg19 is there for Exomiser's cut-down test release, which CI runs. One of: `hg38`, `hg19`. | diff --git a/docs/recheck.md b/docs/recheck.md index 221c6fc0..a9c8de84 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -91,7 +91,7 @@ python3 bin/owngenome_recheck.py --launch-dir /path/to/launch/dir ## In the reports and on the live site -The next reports read `recheck/state.json`: both show one "Database re-check" line in the run facts. A check older than the sample's `run_manifest.tsv` describes an earlier run, so the reports leave it out until the next check. When the sample has a previous run in `history/` ([the delta report](delta-report.md)), a section whose results changed since it carries a badge such as "New since 2026-10-01T09:15:00Z", in both reports, and the delta report heads its changes the same way. The live site's re-check plugin (`bin/owngenome_live_recheck.py`, `assets/live/plugins/recheck.js`) shows the same line as a banner and marks each named step on the step map with a dot whose tooltip says "newer ClinVar available" or "redone 2026-10-05". +The next reports read `recheck/state.json`: both show one "Database re-check" line in the run facts. A check older than the sample's `run_manifest.tsv` describes an earlier run, so the reports leave it out until the next check. When the sample has a previous run in `history/` ([the delta report](delta-report.md)), a section whose results changed since it carries a badge such as "New since 2026-10-01T09:15:00Z", in both reports, and the delta report heads its changes the same way. The live site's re-check plugin (`bin/owngenome_live_recheck.py`, `assets/live/plugins/recheck.js`) shows the same line as a banner, puts a badge (↻) such as "newer ClinVar available" in the header of every result card of a named step on the person page, and marks each named step on the Progress tab's step map with a dot whose tooltip says the same or "redone 2026-10-05". The Nextflow `HTML_REPORT` process sees no `history/` folder, so its report has no "New since" badge. From 79a285abc3dc115a03103970dc1e832db860b046 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:56:28 +0100 Subject: [PATCH 03/10] feat(report): mark AnnotSV class 4 and 5 SVs and MoChA calls on the genome chart The report and the live page share the chart, so both get the marks. Bead: pgp-5bja --- bin/report_charts.py | 66 ++++++++++++++++++++++++++++++++++--- tests/test_report_charts.py | 47 ++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/bin/report_charts.py b/bin/report_charts.py index c38a8c6f..679c0894 100755 --- a/bin/report_charts.py +++ b/bin/report_charts.py @@ -21,9 +21,12 @@ is the report's own note; a score of the other sex, or one with no percentile, says "no percentile" and is never placed. genome the chromosomes 1 to 22, X and Y of GRCh38 with the runs of - homozygosity of 5 MB or more and a tick for each ClinVar hit; + homozygosity of 5 MB or more, a tick for each ClinVar hit, an + outlined span for each AnnotSV class 4 or 5 SV and a bar under the + chromosome for each mosaic chromosomal alteration MoChA reports; drawn in the ROH section, so only when ROH ran. - Hits on chrM and on other contigs are counted in not_drawn. + Hits, SVs and calls on chrM and on other contigs are counted in + not_drawn. mito the heteroplasmic chrM sites by position and allele fraction, the D-loop poly-C length variants marked apart. @@ -31,7 +34,8 @@ not_drawn}: section is the collect_summary.SECTIONS key whose data it shows; each mark is {id, label, value, low, high, chrom, pos, end, tone, anchor, locus, detail}, with None for what does not apply; a ClinVar tick on the -genome chart also carries the ClinVar caveat, the chart's own being ROH's. anchor is the id of the +genome chart also carries the ClinVar caveat, an SV span AnnotSV's and a MoChA +bar MoChA's, the chart's own being ROH's. anchor is the id of the report element the mark links to (the ClinVar hits table, or the section); locus is "chrom:pos" or "chrom:start-end" for the genome viewer. The caveat text is passed in by the caller (notes), never written here, so the chart and the @@ -222,7 +226,40 @@ def genome_chart(S, notes): detail=f"{h.get('significance') or '.'}, {h.get('stars')} star" f"{'' if h.get('stars') == 1 else 's'}, {h.get('genotype') or '.'}" + (", low-confidence call" if low else ""))) - return spec("genome", "roh", "ideogram", "Runs of homozygosity and ClinVar hits on the chromosomes", notes, + if _ok(S, "annotsv"): + for i, r in enumerate(S["annotsv"]["data"].get("rows") or []): + m = REGION_RE.match(str(r.get("sv", ""))) + if not m or m.group(1) not in CHROM_LEN: + not_drawn["AnnotSV class 4 and 5 SVs on other contigs"] = \ + not_drawn.get("AnnotSV class 4 and 5 SVs on other contigs", 0) + 1 + continue + chrom, start, end = m.group(1), int(m.group(2)), int(m.group(3)) + genes = ", ".join((r.get("genes") or [])[:5]) or "no gene" + marks.append(mark(f"sv-{i}", f"{r.get('type') or 'SV'} {r['sv']} (AnnotSV class {r.get('acmg_class')})", + r.get("acmg_class"), chrom=chrom, pos=start, end=max(end, start), tone="attention", + anchor="structural-variants", locus=f"{chrom}:{start}-{end}", kind="sv", + caveat=notes.get("ANNOTSV_NOTE"), + detail=f"AnnotSV class {r.get('acmg_class')} {r.get('type') or 'SV'}, " + f"{r.get('length') or '.'} bp, genotype {r.get('gt') or '.'}, genes: {genes}")) + if _ok(S, "mosaic"): + for i, c in enumerate(S["mosaic"]["data"].get("calls") or []): + chrom = c.get("chrom") + try: + start, end = int(c.get("start")), int(c.get("end")) + except (TypeError, ValueError): + start = end = None + if chrom not in CHROM_LEN or start is None or end < start: + not_drawn["MoChA calls on other contigs or with no position"] = \ + not_drawn.get("MoChA calls on other contigs or with no position", 0) + 1 + continue + marks.append(mark(f"mca-{i}", f"MoChA {c.get('type') or 'call'} {chrom}:{start}-{end}", + c.get("cell_fraction"), chrom=chrom, pos=start, end=end, tone="attention", + anchor="mosaic-mca", locus=f"{chrom}:{start}-{end}", kind="mca", + caveat=notes.get("MCA_NOTE"), + detail=f"mosaic {c.get('type') or 'alteration'} over {c.get('length_mb') or '.'} Mb, " + f"cell fraction {c.get('cell_fraction') or '.'}")) + return spec("genome", "roh", "ideogram", "Runs of homozygosity, ClinVar hits, structural variants and mosaic " + "alterations on the chromosomes", notes, "ROH_NOTE", marks, axis={"assembly": "GRCh38"}, not_drawn=[{"what": k, "count": v} for k, v in not_drawn.items()]) @@ -419,6 +456,17 @@ def svg_ideogram(sp): a, b = sx(m["pos"]), sx(m["end"]) inner = (f'') + elif m.get("kind") == "sv": + # An outlined span over the chromosome, at least 5 px so a small deletion is seen. + a, b = sx(m["pos"]), sx(m["end"]) + w = max(b - a, 5) + inner = (f'') + elif m.get("kind") == "mca": + # A bar under the chromosome: a share of the cells, not the whole genome. + a, b = sx(m["pos"]), sx(m["end"]) + inner = (f'') else: x = sx(m["pos"]) inner = (f'' + _text(x0 + 348, y + 8, "low-confidence ClinVar call", 11)) + kinds = {m.get("kind") for m in sp["marks"]} + if kinds & {"sv", "mca"}: + # A second legend row, drawn only when the chart has these marks. + y += 18 + if "sv" in kinds: + legend += (f'' + _text(x0 + 20, y + 8, "AnnotSV class 4 or 5 SV", 11)) + if "mca" in kinds: + legend += (f'' + + _text(x0 + 270, y + 8, "mosaic alteration (MoChA), a share of cells", 11)) foot, h = _footer(sp, y + 26) return _open(sp, h + 4) + "".join(body) + legend + foot + "" diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 64b82fd4..c37cb1ed 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -15,6 +15,9 @@ 4. genome: a run of homozygosity whose region cannot be read stops the chart; a run under 5 MB is not drawn; ClinVar hits on chrM and on alt contigs are counted in not_drawn; ROH_MIN_MB is collect_summary's; + each AnnotSV class 4 or 5 row is an outlined span and each reported MoChA + call a bar under its chromosome, linked to their sections, and one on + another contig is counted in not_drawn; 5. mito: one dot per heteroplasmic site, the poly-C call marked apart; 6. no caveat sentence is written in report_charts.py: they come from render_report's *_NOTE constants; @@ -119,6 +122,27 @@ def whisker_problems(svg): return out +def sv_mca_problems(sp, sv_rows, mca_calls): + """What the genome chart misses of the AnnotSV rows and MoChA calls on the primary chromosomes.""" + out = [] + marks = {m["id"]: m for m in sp["marks"]} + for i, r in enumerate(sv_rows): + if r["sv"].split(":")[0] not in report_charts.CHROM_LEN: + continue + m = marks.get(f"sv-{i}") + if not m or m["anchor"] != "structural-variants" or f'data-mark="sv-{i}"' not in sp["svg"]: + out.append(f"AnnotSV row {r['sv']} not drawn") + for i, c in enumerate(mca_calls): + m = marks.get(f"mca-{i}") + if not m or m["anchor"] != "mosaic-mca" or f'data-mark="mca-{i}"' not in sp["svg"]: + out.append(f"MoChA call {c['chrom']}:{c['start']} not drawn") + if sp["svg"].count('class="sv"') != sum(1 for r in sv_rows if r["sv"].split(":")[0] in report_charts.CHROM_LEN): + out.append("the outlined spans do not match the rows") + if sp["svg"].count('class="mca"') != len(mca_calls): + out.append("the bars do not match the calls") + return out + + def note_literals(src): """Caveat sentences written into a source file instead of passed in.""" found = [] @@ -245,6 +269,29 @@ def unit_checks(): "positions are missing", 'class="site"' not in om["svg"] and om["not_drawn"][0]["count"] == 2 and "positions of the heteroplasmic sites are not in this summary" in om["svg"] and "No heteroplasmic site" not in om["svg"], om["not_drawn"]) + sv_rows = [{"sv": "chr7:117480000-117530000", "type": "DEL", "gt": "0/1", "length": "50000", "acmg_class": "4", + "genes": ["CFTR"]}, + {"sv": "chr22_KI270879v1_alt:100-900", "type": "DUP", "gt": "0/1", "length": "800", "acmg_class": "5", + "genes": []}] + mca = [{"chrom": "chr20", "start": "30000000", "end": "42000000", "length_mb": "12.0", "type": "Del", + "cell_fraction": "0.12"}] + sm = copy.deepcopy(s) + sm["sections"]["annotsv"] = {"state": "ok", "data": {"count": 2, "rows": sv_rows}} + sm["sections"]["mosaic"] = {"state": "ok", "data": {"status": "assessed", "calls": mca}} + gm = charts(sm)["genome"] + ndm = {x["what"]: x["count"] for x in gm["not_drawn"]} + check("4. genome: each AnnotSV class 4/5 row is an outlined span, each MoChA call a bar, the alt-contig SV " + "in not_drawn, both in the legend", + sv_mca_problems(gm, sv_rows, mca) == [] and ndm.get("AnnotSV class 4 and 5 SVs on other contigs") == 1 + and "AnnotSV class 4 or 5 SV" in gm["svg"] and "mosaic alteration (MoChA)" in gm["svg"], + (sv_mca_problems(gm, sv_rows, mca), ndm)) + lost = copy.deepcopy(gm) + lost["marks"] = [m for m in lost["marks"] if m.get("kind") != "mca"] + lost["svg"] = report_charts.svg(lost) + check("4. red control: a genome chart that leaves out the MoChA call is caught", + sv_mca_problems(lost, sv_rows, mca) != []) + check("4. genome: a chart with no SV or MoChA mark draws no second legend row", + "AnnotSV class 4 or 5 SV" not in g["svg"] and "MoChA" not in g["svg"]) nr = copy.deepcopy(s) nr["sections"]["roh"]["state"] = "missing" check("4. genome: no chart when ROH did not run, even with ClinVar hits", "genome" not in charts(nr)) From 2f186fd0f8e3b994555a48a632f509ecf70806dd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:58:04 +0100 Subject: [PATCH 04/10] feat(live): draw the repeat gauge, the Lp(a) interval, CHIP VAF, copy-number dots and the Y path in their cards Bead: pgp-5bja --- assets/live/live.css | 44 +++++++++ assets/live/person.js | 169 +++++++++++++++++++++++++++++++++ bin/live_results.py | 74 ++++++++++++--- tests/test_live_widgets.py | 187 +++++++++++++++++++++++++++++++++++++ 4 files changed, 460 insertions(+), 14 deletions(-) create mode 100644 tests/test_live_widgets.py diff --git a/assets/live/live.css b/assets/live/live.css index 08f6ce5e..5827c3d4 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -152,6 +152,50 @@ table.rows tbody tr:nth-child(even) td { background: color-mix(in srgb, var(--ru .explore { font-size: 14px; color: var(--dim); } img.plot { max-width: 100%; height: auto; background: #fff; border-radius: 4px; margin-bottom: var(--s3); } +/* widgets (person.js): a track with bands, intervals and ticks placed by --a (start) and --w (width) */ +.widget { margin: 0 0 var(--s4); } +.w-row { margin: 0 0 var(--s3); padding-left: 8px; border-left: 3px solid transparent; } +.w-row.w-attn { border-left-color: var(--attn); } +.w-head { margin: 0 0 4px; display: flex; flex-wrap: wrap; gap: 2px 8px; align-items: baseline; } +.w-name { font-weight: 600; font-family: var(--mono); font-size: 14px; } +.w-val { font-variant-numeric: tabular-nums; } +.w-extra { color: var(--dim); font-size: 13px; } +.w-track { position: relative; height: 22px; border-radius: 4px; background: color-mix(in srgb, var(--rule) 55%, transparent); } +.w-band, .w-ci, .w-tick { position: absolute; left: var(--a); } +.w-band { top: 0; bottom: 0; width: var(--w); } +.w-band:first-child { border-radius: 4px 0 0 4px; } +.w-normal { background: color-mix(in srgb, var(--dim) 30%, transparent); } +.w-patho { background: repeating-linear-gradient(45deg, color-mix(in srgb, var(--attn) 55%, transparent) 0 2px, transparent 2px 7px); + border-left: 1px solid var(--attn); } +.w-blind { background: repeating-linear-gradient(-45deg, color-mix(in srgb, var(--dim) 45%, transparent) 0 2px, transparent 2px 7px); + border-right: 1px dashed var(--dim); } +.w-ci { top: 50%; height: 2px; width: var(--w); margin-top: -1px; background: var(--text); } +.w-ci::before, .w-ci::after { content: ""; position: absolute; top: -5px; width: 2px; height: 12px; background: var(--text); } +.w-ci::before { left: 0; } .w-ci::after { right: 0; } +.w-tick { top: -3px; width: 4px; height: 28px; margin-left: -2px; border-radius: 2px; background: var(--text); } +.w-tick.w-low { background: var(--card); border: 2px solid var(--low); } +.w-tick.w-call { width: 12px; height: 12px; top: 5px; margin-left: -6px; border-radius: 50%; background: var(--attn); } +.w-tick:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } +.w-legend { margin: 4px 0 0; color: var(--dim); font-size: 13px; } +.w-key { font-size: 13px; margin: 0; max-width: 75ch; } +.w-calls { margin: 6px 0 0; padding-left: 1.1em; } +.w-fallback summary { cursor: pointer; color: var(--info); font-size: 13px; margin-top: var(--s2); } +.w-copy-list { list-style: none; margin: 0; padding: 0; display: grid; gap: 6px; } +.w-copy-list li { display: grid; grid-template-columns: minmax(7ch, 12ch) auto 1fr; gap: 4px 12px; align-items: center; } +.w-copy-list .w-extra { grid-column: 2 / -1; } +.w-dots { display: inline-flex; gap: 5px; align-items: center; min-height: 14px; } +.w-dot { width: 14px; height: 14px; border-radius: 50%; background: var(--info); } +.w-low .w-dot { background: transparent; border: 2px solid var(--low); } +ol.w-path { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 6px 0; align-items: stretch; } +ol.w-path li { display: flex; flex-direction: column; padding: 6px 10px; border: 1px solid var(--rule); border-radius: 6px; + background: var(--card2); position: relative; margin-right: 18px; } +ol.w-path li:not(:last-child)::after { content: "\203a"; position: absolute; right: -14px; top: 50%; transform: translateY(-50%); + color: var(--dim); font-size: 18px; } +ol.w-path li.w-tip { border-color: var(--info); } +ol.w-path li.w-low { border-style: dashed; border-color: var(--low); } +.w-branch { font-family: var(--mono); font-weight: 600; } +.w-count { color: var(--dim); font-size: 12px; } + /* charts (plugins/charts.js) inside a card */ .chart-slot:empty { display: none; } figure.chart { margin: 0 0 var(--s3); } diff --git a/assets/live/person.js b/assets/live/person.js index 6433f7d1..83f4bca3 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -124,6 +124,174 @@ } U.tablist(rail, function (t) { go(t.getAttribute('href')); }); + // --- widgets: a card's numbers drawn --------------------------------------------------------------- + // HTML placed in percent of a track (custom properties, never a style attribute), so the text stays + // at its size on a phone. Every mark is focusable and says itself in words; colour is never alone. + function num(v) { var n = parseFloat(v); return isFinite(n) ? n : null; } + function pct(v, max) { return Math.max(0, Math.min(100, 100 * v / max)); } + function nice(v) { var step = v > 200 ? 50 : 10; return Math.max(step, Math.ceil(v / step) * step); } + function place(e, from, to, max) { + e.style.setProperty('--a', pct(from, max).toFixed(2) + '%'); + if (to !== undefined) { e.style.setProperty('--w', Math.max(0.6, pct(to, max) - pct(from, max)).toFixed(2) + '%'); } + return e; + } + function mark(cls, v, max, label) { + var m = el('span', { 'class': 'w-tick ' + (cls || ''), tabindex: '0', role: 'img', 'aria-label': label, title: label }); + return place(m, v, undefined, max); + } + function trackOf(max, parts) { + var t = el('div', { 'class': 'w-track' }); + parts.forEach(function (p) { if (p) { add(t, p); } }); + return t; + } + function band(cls, from, to, max, label) { + return place(el('span', { 'class': 'w-band ' + cls, title: label || '' }), from, to, max); + } + function whisker(lo, hi, max, label) { + return place(el('span', { 'class': 'w-ci', title: label || '' }), lo, hi, max); + } + function wrow(head, track, legend, cls) { + var r = el('div', { 'class': 'w-row ' + (cls || '') }); + add(r, head, track); + if (legend) { add(r, el('p', { 'class': 'w-legend' }, legend)); } + return r; + } + function whead(name, value, extra) { + var h = el('p', { 'class': 'w-head' }); + add(h, el('span', { 'class': 'w-name' }, name), ' ', el('span', { 'class': 'w-val' }, value)); + if (extra) { add(h, ' ', el('span', { 'class': 'w-extra' }, extra)); } + return h; + } + + // Repeat expansions: each allele against Stranger's thresholds, its 95% interval as a whisker. + function strGauge(w) { + var box = el('div', { 'class': 'widget w-str', 'data-widget': 'str' }); + (w.loci || []).forEach(function (l) { + var counts = String(l.repeat_count || '').split('/'); + var sizing = String(l.allele_sizing || '').split('/'); + var cis = String(l.ci || '').split('/'); + var nmax = num(l.normal_max), pmin = num(l.pathologic_min); + var top = 0; + counts.forEach(function (c, i) { + var ci = (cis[i] || '').split('-'); + top = Math.max(top, num(c) || 0, num(ci[1]) || 0); + }); + var max = nice(Math.max(top * 1.15, (pmin || 0) * 1.3, (nmax || 0) * 1.6, 10)); + var parts = []; + if (nmax !== null) { parts.push(band('w-normal', 0, nmax, max, 'Stranger’s "normal": up to ' + nmax)); } + if (pmin !== null) { parts.push(band('w-patho', pmin, max, max, 'pathologic from ' + pmin)); } + counts.forEach(function (c, i) { + var v = num(c); + if (v === null) { return; } + var ci = (cis[i] || '').split('-'); + var lo = num(ci[0]), hi = num(ci[1]); + if (lo !== null && hi !== null && hi > lo) { parts.push(whisker(lo, hi, max, '95% interval ' + lo + ' to ' + hi)); } + var unsized = sizing[i] && ['sized', 'unknown'].indexOf(sizing[i]) < 0; + parts.push(mark(unsized ? 'w-low' : '', v, max, l.locus + ' allele ' + (i + 1) + ': ' + c + ' repeats' + + (lo !== null && hi !== null ? ', 95% interval ' + lo + ' to ' + hi : '') + (unsized ? ', not sized from short reads' : ''))); + }); + var st = l.status ? 'Stranger: "' + l.status + '"' : 'no Stranger status'; + var legend = (nmax !== null || pmin !== null) + ? [nmax !== null ? '"normal" up to ' + nmax : '', pmin !== null ? 'pathologic from ' + pmin : '', 'scale 0 to ' + max] + .filter(Boolean).join(' · ') + : 'No Stranger thresholds for this locus. Scale 0 to ' + max + '.'; + var flagged = l.status && l.status.split(',').some(function (x) { return x && x !== 'normal'; }); + add(box, wrow(whead(l.locus, (l.repeat_count || '.') + ' repeats', st), trackOf(max, parts), legend, flagged ? 'w-attn' : '')); + }); + add(box, el('p', { 'class': 'w-key dim' }, 'Tick: an allele’s repeat count; line: its 95% interval; hollow: not sized ' + + 'from short reads. Shaded: Stranger’s "normal" band; hatched: its pathologic range.')); + return box; + } + + // A count with its 95% interval on a scale from 0 (Lp(a)'s KIV-2 repeats). + function ciBar(w) { + var box = el('div', { 'class': 'widget w-cibar', 'data-widget': 'ci' }); + var max = w.max || nice(w.high * 1.25); + var label = w.label + ': ' + w.value + ', 95% interval ' + w.low + ' to ' + w.high; + add(box, wrow(whead(w.label, w.value + ' ' + (w.unit || ''), '95% interval ' + w.low + ' to ' + w.high + + (w['class'] ? ' · class "' + w['class'] + '"' : '')), + trackOf(max, [whisker(w.low, w.high, max, '95% interval'), mark('', w.value, max, label)]), + 'Scale 0 to ' + max + ' ' + (w.unit || '') + '. The line is the 95% interval of the estimate.')); + return box; + } + + // CHIP: each call's VAF against the smallest VAF this depth can see. + function vafChart(w) { + var box = el('div', { 'class': 'widget w-vaf', 'data-widget': 'vaf' }); + var max = w.max || 0.5; + var limits = (w.genes || []).map(function (g) { return g.limit; }).filter(function (x) { return typeof x === 'number'; }); + var lo = limits.length ? Math.min.apply(null, limits) : null, hi = limits.length ? Math.max.apply(null, limits) : null; + var parts = []; + if (hi !== null) { parts.push(band('w-blind', 0, hi, max, 'below the detection limit at this depth')); } + (w.calls || []).forEach(function (c) { + parts.push(mark('w-call', c.vaf, max, c.gene + ' ' + (c.change || '') + ': VAF ' + c.vaf)); + }); + var head = whead('Variant allele fraction', (w.calls || []).length ? (w.calls.length + ' call' + (w.calls.length > 1 ? 's' : '')) + : 'no call above the limit'); + var legend = (hi !== null ? 'Hatched: below the smallest VAF this depth can see (' + lo + (hi !== lo ? ' to ' + hi : '') + + ' across ' + limits.length + ' genes); a clone that small is not seen. ' : '') + 'Scale VAF 0 to ' + max + '.'; + add(box, wrow(head, trackOf(max, parts), legend)); + if ((w.calls || []).length) { + var ul = el('ul', { 'class': 'w-calls' }); + w.calls.forEach(function (c) { add(ul, el('li', {}, c.gene + ' ' + (c.change || '') + ' at VAF ' + c.vaf)); }); + add(box, ul); + } + if ((w.genes || []).length) { + var d = el('details', { 'class': 'w-fallback' }); + add(d, el('summary', {}, 'Detection limit per gene')); + add(d, U.table({ columns: ['Gene', 'Mean depth', 'Smallest VAF seen', 'Status'], + rows: w.genes.map(function (g) { return [g.gene, g.depth === null ? '.' : String(g.depth), g.limit === null ? '.' : String(g.limit), g.status || '.']; }) }, 40)); + add(box, d); + } + return box; + } + + // Copy numbers as dots, one per copy; a call that is not "called" is hollow and says so. + function copyDots(w) { + var box = el('div', { 'class': 'widget w-copies', 'data-widget': 'copies' }); + var ul = el('ul', { 'class': 'w-copy-list' }); + (w.genes || []).forEach(function (g) { + var low = g.status !== 'called'; + var li = el('li', { 'class': low ? 'w-low' : '' }); + add(li, el('span', { 'class': 'w-name' }, g.gene)); + var dots = el('span', { 'class': 'w-dots', role: 'img', tabindex: '0', + 'aria-label': g.gene + ': ' + (g.copies === null ? 'no copy number' : g.copies + ' copies') + (low ? ', ' + g.status : '') }); + for (var i = 0; i < Math.min(g.copies || 0, 8); i++) { add(dots, el('span', { 'class': 'w-dot' })); } + if ((g.copies || 0) > 8) { add(dots, '+' + (g.copies - 8)); } + add(li, dots); + add(li, el('span', { 'class': 'w-val' }, g.copies === null ? 'not called' : g.copies + (g.copies === 1 ? ' copy' : ' copies'))); + if (low) { add(li, el('span', { 'class': 'w-extra' }, 'low confidence: ' + g.status)); } + add(ul, li); + }); + add(box, ul); + return box; + } + + // The Y haplogroup's path from the root of the tree to its placement. + function yPath(w) { + var box = el('div', { 'class': 'widget w-ypath', 'data-widget': 'ypath' }); + add(box, el('p', { 'class': 'w-head' }, 'From the root to ' + (w.tip || 'the placement') + (w.tree ? ' on ' + w.tree : ''))); + var ol = el('ol', { 'class': 'w-path' }); + (w.path || []).forEach(function (p, i, all) { + var seen = (p.derived || 0) + (p.ancestral || 0) + (p.no_call || 0); + var li = el('li', { 'class': (i === all.length - 1 ? 'w-tip' : '') + (p.ancestral ? ' w-low' : ''), tabindex: '0', + 'aria-label': 'Level ' + p.level + ', ' + p.branch + ': ' + p.derived + ' of ' + seen + ' markers derived' + + (p.ancestral ? ', ' + p.ancestral + ' ancestral' : '') }); + add(li, el('span', { 'class': 'w-branch' }, p.branch)); + add(li, el('span', { 'class': 'w-count' }, (p.derived === null ? '.' : p.derived) + ' of ' + seen + ' derived' + + (p.ancestral ? ' · ' + p.ancestral + ' ancestral' : ''))); + add(ol, li); + }); + add(box, ol); + return box; + } + var WIDGETS = { str: strGauge, ci: ciBar, vaf: vafChart, copies: copyDots, ypath: yPath }; + function widget(w) { + var fn = w && WIDGETS[w.kind]; + if (!fn) { return null; } + try { return fn(w); } catch (e) { return el('p', { 'class': 'dim' }, 'This drawing could not be made; the table below has the numbers.'); } + } + // --- a card ----------------------------------------------------------------------------------- function badge(text, cls, symbol) { var b = el('span', { 'class': 'chip ' + cls }); @@ -167,6 +335,7 @@ (c.notes || []).forEach(function (n) { add(art, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(art, el('p', { 'class': 'headline' }, c.headline)); } add(art, U.facts(c.facts)); + add(art, widget(c.widget)); Object.keys((results && results.charts) || {}).forEach(function (chart) { if (results.charts[chart] === c.id) { add(art, el('div', { 'class': 'chart-slot', 'data-chart-slot': chart })); } }); diff --git a/bin/live_results.py b/bin/live_results.py index 74f77a53..eb80cdca 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -57,6 +57,15 @@ def cell(v): return str(v) +def num(v): + """A summary value as a float, or None ('.', '', 'NA', a range).""" + try: + x = float(v) + except (TypeError, ValueError): + return None + return x if x == x and x not in (float("inf"), float("-inf")) else None + + def table(columns, rows, limit=200): rows = [[cell(c) for c in r] for r in rows] if not rows: @@ -273,9 +282,12 @@ def a_expansions(S, s): return {"facts": facts, "attention": flagged, "table": table(["Locus", "Repeats", "Stranger status", "Normal up to", "Pathologic from", "95% interval", "Sizing", "Intermediate zone"], rows), - "widget": {"kind": "str", "loci": [{k: l.get(k) for k in ("locus", "repeat_count", "status", "normal_max", - "pathologic_min", "ci")} - for l in d.get("loci") or []]}} + # The gauge: each allele's count against Stranger's thresholds, its 95% interval as a whisker. + # status is render_report's str_status: never Stranger's "normal" for an allele not sized. + "widget": {"kind": "str", "loci": [dict({k: l.get(k) for k in ("locus", "repeat_count", "normal_max", + "pathologic_min", "ci", "allele_sizing")}, + status=rr.str_status(l)) + for l in d.get("loci") or []]}} def a_vntyper(S, s): @@ -301,17 +313,32 @@ def a_lpa(S, s): d = S["lpa"]["data"] facts = [["KIV-2 repeats", f"{d.get('kiv2_copies')} (95% interval {d.get('ci95_low')} to {d.get('ci95_high')})" if d.get("status") == "estimated" else d.get("status")], ["Class", d.get("class")]] - return {"facts": facts, "lines": rr.lpa_lines(d), "attention": d.get("flag") == "high"} + out = {"facts": facts, "lines": rr.lpa_lines(d), "attention": d.get("flag") == "high"} + v, lo, hi = num(d.get("kiv2_copies")), num(d.get("ci95_low")), num(d.get("ci95_high")) + if d.get("status") == "estimated" and v is not None and lo is not None and hi is not None: + # The count and its 95% interval on a scale from 0: the page draws the bar, the facts say it in words. + out["widget"] = {"kind": "ci", "label": "KIV-2 repeats, both chromosomes", "value": v, "low": lo, "high": hi, + "unit": "repeats", "max": max(60, 10 * int(hi * 1.25 / 10 + 1)), "class": d.get("class")} + return out def a_chip(S, s): d = S["chip"]["data"] rows = [[c.get("gene"), c.get("protein_change"), c.get("vaf"), c.get("alt_reads"), c.get("depth"), c.get("status")] for c in d.get("calls") or []] - return {"facts": [["CHIP calls", len(d.get("calls") or [])], - ["Typical detection limit at this depth (VAF)", d.get("median_limit")]], - "table": table(["Gene", "Change", "VAF", "Alt reads", "Depth", "Status"], rows), - "lines": rr.chip_lines(d, s.get("sample")), "attention": bool(d.get("calls"))} + genes = [{"gene": g.get("gene"), "limit": num(g.get("detection_limit_vaf")), "depth": num(g.get("mean_depth")), + "status": g.get("status")} for g in d.get("genes") or []] + calls = [{"gene": c.get("gene"), "change": c.get("protein_change"), "vaf": num(c.get("vaf"))} + for c in d.get("calls") or []] + out = {"facts": [["CHIP calls", len(d.get("calls") or [])], + ["Typical detection limit at this depth (VAF)", d.get("median_limit")]], + "table": table(["Gene", "Change", "VAF", "Alt reads", "Depth", "Status"], rows), + "lines": rr.chip_lines(d, s.get("sample")), "attention": bool(d.get("calls"))} + if genes or calls: + # Each call's VAF against the smallest VAF this depth can see in each gene (chip_report's own limit): + # under the limit nothing can be seen, so an empty track is not an all-clear. + out["widget"] = {"kind": "vaf", "max": 0.5, "genes": genes, "calls": [c for c in calls if c["vaf"] is not None]} + return out def a_mosaic(S, s): @@ -356,16 +383,28 @@ def a_couple(S, s): "table": table(["Result", "Gene", "Condition", "Chance for a child", "This person", "Partner"], rows)} +# The genes the copy-number dots draw (the carrier screen's copy-number genes): SMN1 and SMN2, the two +# alpha-globin genes and the RCCX module of CYP21A2. +COPY_GENES = ("SMN1", "SMN2", "HBA1", "HBA2", "CYP21A2") + + def a_paralogs(S, s): lines = [] if (S.get("parascopy") or {}).get("state") in ("ok", "stale"): d = S["parascopy"]["data"] lines += rr.parascopy_lines(d) + rr.parascopy_locus_lines(d) + rr.cn_interval_lines(d.get("cn_intervals")) - rows = [] + rows, dots = [], [] if (S.get("paralogs") or {}).get("state") in ("ok", "stale"): - rows = [[g.get("gene"), g.get("copy_number"), g.get("status"), g.get("detail")] - for g in S["paralogs"]["data"].get("genes") or []] - return {"lines": lines, "table": table(["Gene", "Copies", "Status", "Detail"], rows)} + for g in S["paralogs"]["data"].get("genes") or []: + rows.append([g.get("gene"), g.get("copy_number"), g.get("status"), g.get("detail")]) + if str(g.get("gene") or "").split(" ")[0] in COPY_GENES: + n = num(g.get("copy_number")) + dots.append({"gene": g.get("gene"), "copies": int(n) if n is not None and n == int(n) and 0 <= n <= 20 + else None, "status": g.get("status") or "."}) + out = {"lines": lines, "table": table(["Gene", "Copies", "Status", "Detail"], rows)} + if dots: + out["widget"] = {"kind": "copies", "genes": dots} + return out def a_pharmcat(S, s): @@ -446,8 +485,15 @@ def a_y(S, s): st = (S.get("y_str") or {}).get("data") or {} if st.get("markers"): facts.append(["Y-STR markers read", sum(1 for m in st["markers"] if m.get("count"))]) - return {"facts": facts, "explore": explore, - "table": table(["Level", "Branch", "Derived", "Ancestral", "No call"], rows)} + out = {"facts": facts, "explore": explore, + "table": table(["Level", "Branch", "Derived", "Ancestral", "No call"], rows)} + if t and t.get("path"): + # The path from the root of the tree to the placement, one branch per level, with its marker counts. + out["widget"] = {"kind": "ypath", "tree": t.get("tree"), "tip": t.get("haplogroup"), + "path": [{"level": p.get("Level"), "branch": p.get("Branch"), "derived": num(p.get("Derived")), + "ancestral": num(p.get("Ancestral")), "no_call": num(p.get("No_call"))} + for p in t["path"]]} + return out def a_archaic(S, s): diff --git a/tests/test_live_widgets.py b/tests/test_live_widgets.py new file mode 100644 index 00000000..987ad1a0 --- /dev/null +++ b/tests/test_live_widgets.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""The widgets of a person's page: what bin/live_results.py hands the page to draw (results.json). + +Each widget is checked on small made-up sections, against the numbers of the section it draws: + 1. repeat expansions (str): one entry per locus with its count, Stranger's thresholds and its + 95% interval; an allele not sized from short reads never carries Stranger's "normal"; + 2. Lp(a) (ci): the KIV-2 count and its 95% interval, on a scale from 0 that holds the whole + interval; no bar for a count that was not estimated; + 3. CHIP (vaf): each gene's detection limit and each call's VAF as numbers; a call with no VAF + is left off the track, not put at 0; + 4. copy numbers (copies): SMN1, SMN2, HBA1, HBA2 and the RCCX module of CYP21A2, never another + gene of the paralog table; a copy number that is not a whole number is no dots; + 5. the Y haplogroup path (ypath): one entry per level of step 37's path, root first, with its + marker counts as numbers; + 6. every widget's strings pass the wording gate (live_results.FORBIDDEN); + 7. assets/live/person.js draws each kind the adapters give (a drawer per kind). +Each check runs again on a planted mistake and must fail there: the adapter's widget dropped +(1 to 5), "ok" planted in a widget (6), a drawer removed from person.js (7). A check that +cannot fail is no check. + +Run: python3 tests/test_live_widgets.py +""" +import copy +import os +import re +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "bin")) +import live_results # noqa: E402 + +FAILS = [] + + +def check(desc, ok, detail=""): + print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:800]}")) + if not ok: + FAILS.append(desc) + + +def control(desc, still_passes): + check(f"control: {desc}", not still_passes, "the check passed on the planted mistake") + + +def sections(): + return { + "expansions": {"state": "ok", "data": {"stranger": True, "records": 2, "loci": [ + {"locus": "HTT", "repeat_count": "17/42", "status": "normal,full_mutation", "normal_max": "26", + "pathologic_min": "40", "ci": "17-17/40-45", "sizing": "sized", "allele_sizing": "sized/sized"}, + {"locus": "FMR1", "repeat_count": "30", "status": "normal", "normal_max": "44", "pathologic_min": "200", + "ci": "28-90", "sizing": "flanking_only", "allele_sizing": "flanking_only"}]}}, + "lpa": {"state": "ok", "data": {"status": "estimated", "kiv2_copies": "36.9", "ci95_low": "36.2", + "ci95_high": "37.6", "class": "typical", "flag": "."}}, + "chip": {"state": "ok", "data": {"median_limit": 0.133, "median_depth": 31.0, "calls": [ + {"gene": "DNMT3A", "protein_change": "p.R882H", "vaf": "0.081", "alt_reads": "4", "depth": "49", + "status": "PASS", "note": "invented"}, + {"gene": "TET2", "protein_change": "p.Q1034*", "vaf": ".", "alt_reads": "3", "depth": "40", "status": "PASS", "note": "invented"}], + "genes": [{"gene": "DNMT3A", "mean_depth": "31.3", "detection_limit_vaf": "0.133", "status": "covered"}, + {"gene": "TET2", "mean_depth": "29.0", "detection_limit_vaf": "0.142", "status": "covered"}]}}, + "paralogs": {"state": "ok", "data": {"genes": [ + {"gene": "SMN1", "copy_number": "2", "status": "called", "detail": "."}, + {"gene": "SMN2", "copy_number": "1", "status": "called", "detail": "."}, + {"gene": "SMN exon 7-8 deletion", "copy_number": "0", "status": "called", "detail": "."}, + {"gene": "HBA1", "copy_number": "2.4", "status": "no_call", "detail": "."}, + {"gene": "CYP21A2 (RCCX)", "copy_number": "4", "status": "called", "detail": "."}, + {"gene": "GBA1", "copy_number": "2", "status": "called", "detail": "."}]}}, + "y_tree": {"state": "ok", "data": {"haplogroup": "I-M253", "tree": "YFull YTree v14.06.0", "path": [ + {"Level": "1", "Branch": "CT", "Derived": "11", "Ancestral": "0", "No_call": "1"}, + {"Level": "2", "Branch": "F", "Derived": "9", "Ancestral": "1", "No_call": "0"}, + {"Level": "3", "Branch": "I-M253", "Derived": "7", "Ancestral": "0", "No_call": "1"}]}}, + } + + +def run(adapters, name): + S = sections() + try: + return (adapters[name](S, {"sections": S, "sample": "S"}) or {}).get("widget") + except Exception as e: # a crash is a problem, not a test error + return {"error": f"{type(e).__name__}: {e}"} + + +def str_problems(w): + if not w or w.get("kind") != "str": + return [f"no str widget: {w}"] + out = [] + by = {l["locus"]: l for l in w["loci"]} + h = by.get("HTT") or {} + if (h.get("repeat_count"), h.get("normal_max"), h.get("pathologic_min"), h.get("ci")) != ("17/42", "26", "40", "17-17/40-45"): + out.append(f"HTT {h}") + if (by.get("FMR1") or {}).get("status") == "normal": + out.append("an unsized FMR1 allele carries Stranger's normal") + return out + + +def ci_problems(w): + if not w or w.get("kind") != "ci": + return [f"no ci widget: {w}"] + out = [] + if (w["value"], w["low"], w["high"]) != (36.9, 36.2, 37.6): + out.append(f"numbers {w}") + if not w["max"] >= w["high"] or not w["low"] <= w["value"] <= w["high"]: + out.append(f"scale {w}") + return out + + +def vaf_problems(w): + if not w or w.get("kind") != "vaf": + return [f"no vaf widget: {w}"] + out = [] + if [(g["gene"], g["limit"]) for g in w["genes"]] != [("DNMT3A", 0.133), ("TET2", 0.142)]: + out.append(f"limits {w['genes']}") + if [(c["gene"], c["vaf"]) for c in w["calls"]] != [("DNMT3A", 0.081)]: + out.append(f"calls {w['calls']}") + return out + + +def copies_problems(w): + if not w or w.get("kind") != "copies": + return [f"no copies widget: {w}"] + got = [(g["gene"], g["copies"], g["status"]) for g in w["genes"]] + want = [("SMN1", 2, "called"), ("SMN2", 1, "called"), ("HBA1", None, "no_call"), ("CYP21A2 (RCCX)", 4, "called")] + return [] if got == want else [f"{got} != {want}"] + + +def ypath_problems(w): + if not w or w.get("kind") != "ypath": + return [f"no ypath widget: {w}"] + got = [(p["branch"], p["derived"], p["ancestral"], p["no_call"]) for p in w["path"]] + want = [("CT", 11, 0, 1), ("F", 9, 1, 0), ("I-M253", 7, 0, 1)] + return [] if got == want and w["tip"] == "I-M253" else [f"{got} {w.get('tip')}"] + + +CASES = [("1. repeat expansions: the gauge's loci, thresholds and intervals", "expansions", str_problems), + ("2. Lp(a): the count and its 95% interval on a scale that holds it", "lpa", ci_problems), + ("3. CHIP: the detection limits and the calls with a VAF", "chip", vaf_problems), + ("4. copy numbers: the five copy-number genes only, whole numbers as dots", "paralogs", copies_problems), + ("5. the Y path: every level, root first, with its counts", "y-chromosome", ypath_problems)] + + +def without_widget(fn): + def planted(S, s): + got = dict(fn(S, s) or {}) + got.pop("widget", None) + return got + return planted + + +def drawers(): + with open(os.path.join(REPO, "assets", "live", "person.js")) as f: + src = f.read() + m = re.search(r"var WIDGETS = \{([^}]*)\}", src) + return src, set(re.findall(r"(\w+):", m.group(1))) if m else set() + + +def main(): + A = live_results.ADAPTERS + for desc, sid, problems in CASES: + bad = problems(run(A, sid)) + check(desc, not bad, bad) + control(f"{sid} adapter without its widget", not problems(run(dict(A, **{sid: without_widget(A[sid])}), sid))) + lpa = sections()["lpa"] + lpa["data"]["status"] = "too little depth" + check("2. Lp(a): no bar for a count that was not estimated", + not (A["lpa"]({"lpa": lpa}, {"sections": {"lpa": lpa}}) or {}).get("widget")) + + widgets = {sid: run(A, sid) for _, sid, _ in CASES} + bad = live_results.wording_problems(widgets) + check("6. no widget string reads as an all-clear", not bad, bad) + planted = copy.deepcopy(widgets) + planted["lpa"]["class"] = "ok" + control("a widget saying ok", not live_results.wording_problems(planted)) + + src, kinds = drawers() + need = {w["kind"] for w in widgets.values()} + check("7. person.js has a drawer for every widget kind", need <= kinds and all(f"function {fn}" in src for fn in + ("strGauge", "ciBar", "vafChart", "copyDots", "yPath")), (need, kinds)) + control("a drawer removed from person.js", need <= (kinds - {"ypath"})) + + if FAILS: + print(f"\n{len(FAILS)} check(s) failed") + return 1 + print("\nAll checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 1b5a6b8bbf31752d1ed3b3758709726f702b9727 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:17:13 +0100 Subject: [PATCH 05/10] feat(live): draw the family page in depth on an invented trio The family page now shows the relatedness of every pair as a grid, each child's inherited stretches painted per chromosome, the de novo and compound-het tables, and the members side by side. The demo gains the partner and a son with findings of their own, so every widget and the whole family page have something to draw. Bead: pgp-5bja --- assets/live/family-page.js | 186 ++++++++++++++++-- assets/live/live.css | 38 ++++ assets/live/person.js | 2 +- bin/owngenome_live_family.py | 118 ++++++++++- tests/demo/build_live_demo.py | 113 ++++++++--- tests/demo/make_demo_sample.py | 110 +++++++++-- .../demo/DEMO-FAMILY/DEMO-003_segments.tsv | 81 ++++++++ .../live/demo/DEMO-FAMILY/compound_het.tsv | 3 + .../live/demo/DEMO-FAMILY/de_novo.tsv | 3 + .../live/demo/DEMO-FAMILY/family_plan.tsv | 4 + .../live/demo/DEMO-FAMILY/ibd_pairs.tsv | 4 + .../live/demo/DEMO-FAMILY/members.tsv | 1 + .../live/demo/DEMO-FAMILY/relationships.tsv | 4 + tests/test_live_demo.py | 120 +++++++++-- tests/test_live_family_depth.py | 164 +++++++++++++++ tests/test_render_report.py | 5 +- 16 files changed, 883 insertions(+), 73 deletions(-) create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/DEMO-003_segments.tsv create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/de_novo.tsv create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/family_plan.tsv create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/ibd_pairs.tsv create mode 100644 tests/fixtures/live/demo/DEMO-FAMILY/relationships.tsv create mode 100644 tests/test_live_family_depth.py diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 8a95a211..8ddcc07e 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -1,8 +1,9 @@ // family-page.js: a family's page (assets/live/family.html), from family.json // (bin/owngenome_live_family.py): the pedigree with each sequenced person a link to their page, the // members with their run progress, what steps 38 and 39 planned, each couple's carrier screen in full -// (both carry, then X-linked, then one partner), who is related how, and each child's chromosome -// map. It draws what the steps wrote and computes nothing about genetics. Text through textContent, +// (both carry, then X-linked, then one partner), who is related how as a grid (a list of pairs on a +// phone), each child's stretches painted by parent, the de novo and compound-het candidates, and the +// members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. (function () { 'use strict'; @@ -135,44 +136,207 @@ return s; } + function pairKey(a, b) { return [a, b].sort().join(' '); } + function mismatch(v) { return !!v && v !== 'matches the declared relation'; } function related(f) { var s = section('Who is related how', 'related'); var rel = f.relationships || [], ibd = f.ibd_pairs || []; if (!rel.length && !ibd.length) { add(s, el('p', { 'class': 'dim' }, 'Not run yet (step 38).')); return s; } - var byPair = {}; - ibd.forEach(function (r) { byPair[[r.sample_a, r.sample_b].sort().join(' ')] = r; }); + var byRel = {}, byIbd = {}, names = []; + function seen(n) { if (names.indexOf(n) < 0) { names.push(n); } } + rel.forEach(function (r) { byRel[pairKey(r.sample_a, r.sample_b)] = r; seen(r.sample_a); seen(r.sample_b); }); + ibd.forEach(function (r) { byIbd[pairKey(r.sample_a, r.sample_b)] = r; seen(r.sample_a); seen(r.sample_b); }); + var order = (f.state.members || []).map(function (m) { return m.sample; }); + names.sort(function (a, b) { return order.indexOf(a) - order.indexOf(b); }); + function cellText(a, b) { + var r = byRel[pairKey(a, b)] || {}, i = byIbd[pairKey(a, b)] || {}; + var lines = []; + lines.push(i.degree || (r.relatedness ? 'relatedness ' + r.relatedness : 'not measured')); + if (i.shared_cm) { lines.push(i.shared_cm + ' cM shared'); } + if (r.declared) { lines.push('declared ' + r.declared); } + return { lines: lines, flag: mismatch(r.verdict), verdict: r.verdict, reason: r.reason }; + } + // The grid: one cell per pair, the degree first, the declared relation and a flag when they differ. + var grid = el('div', { 'class': 'rel-grid-wrap' }); + var t = el('table', { 'class': 'rel-grid' }); + var head = el('tr', {}); + add(head, el('th', { scope: 'col' }, '')); + names.forEach(function (n) { add(head, el('th', { scope: 'col' }, n)); }); + add(t, add(el('thead', {}), head)); + var body = el('tbody', {}); + names.forEach(function (a) { + var tr = el('tr', {}); + add(tr, el('th', { scope: 'row' }, a)); + names.forEach(function (b) { + if (a === b) { add(tr, el('td', { 'class': 'self', 'aria-label': 'the same person' }, '·')); return; } + if (!byRel[pairKey(a, b)] && !byIbd[pairKey(a, b)]) { add(tr, el('td', { 'class': 'none' }, 'not measured')); return; } + var c = cellText(a, b); + var td = el('td', { 'class': c.flag ? 'flag' : '', title: c.reason || '' }); + c.lines.forEach(function (l, i) { add(td, el('span', { 'class': i ? 'sub' : 'main' }, l)); }); + if (c.flag) { add(td, el('span', { 'class': 'chip chip-attn' }, '⚑ ' + c.verdict)); } + add(tr, td); + }); + add(body, tr); + }); + add(t, body); + add(grid, t); + add(s, grid); + // On a phone: a list of pairs. + var ul = el('ul', { 'class': 'rel-pairs' }); + names.forEach(function (a, i) { + names.slice(i + 1).forEach(function (b) { + if (!byRel[pairKey(a, b)] && !byIbd[pairKey(a, b)]) { return; } + var c = cellText(a, b); + var li = el('li', { 'class': c.flag ? 'flag' : '' }); + add(li, el('span', { 'class': 'pair' }, a + ' and ' + b + ': '), c.lines.join(', ')); + if (c.flag) { add(li, ' ', el('span', { 'class': 'chip chip-attn' }, '⚑ ' + c.verdict + (c.reason ? ': ' + c.reason : ''))); } + add(ul, li); + }); + }); + add(s, ul); + add(s, el('p', { 'class': 'dim legend-line' }, 'Each cell: the degree the shared stretches point to (or somalier’s ' + + 'relatedness), the centimorgans shared and the declared relation. ⚑ marks a pair the measurement does not match.')); var rows = rel.map(function (r) { - var i = byPair[[r.sample_a, r.sample_b].sort().join(' ')] || {}; + var i = byIbd[pairKey(r.sample_a, r.sample_b)] || {}; return [r.sample_a + ' and ' + r.sample_b, r.declared, r.relatedness, r.ibs0, r.ibs2, i.shared_cm || '.', i.degree || '.', r.verdict, r.reason]; }); ibd.forEach(function (i) { - if (!rel.some(function (r) { return [r.sample_a, r.sample_b].sort().join(' ') === [i.sample_a, i.sample_b].sort().join(' '); })) { + if (!byRel[pairKey(i.sample_a, i.sample_b)]) { rows.push([i.sample_a + ' and ' + i.sample_b, i.declared, i.relatedness, '.', '.', i.shared_cm, i.degree, '.', '.']); } }); - add(s, U.table({ columns: ['Pair', 'Declared', 'Relatedness', 'IBS0', 'IBS2', 'Shared cM', 'Degree', 'Verdict', 'Reason'], + var d = el('details', { 'class': 'w-fallback' }); + add(d, el('summary', {}, 'Every measured number')); + add(d, U.table({ columns: ['Pair', 'Declared', 'Relatedness', 'IBS0', 'IBS2', 'Shared cM', 'Degree', 'Verdict', 'Reason'], rows: rows }, 20)); + add(s, d); if (!ibd.length) { add(s, el('p', { 'class': 'dim' }, 'Shared segments need step 26b’s panel: only the somalier half is here.')); } return s; } + // A child's stretches, painted: per chromosome the father's copy above the mother's, each stretch + // coloured by the grandparent it came from (or by the parent when no grandparent was sequenced). + var SIDE = { paternal: 'father', maternal: 'mother' }; + function painting(f, child, rows) { + var lengths = f.chrom_lengths || []; + var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); + var colours = {}, next = 1; + function colourOf(r) { + var who = r.ancestor && r.ancestor !== '.' ? r.ancestor : (r.parent && r.parent !== '.' ? r.parent : r.side + ' side'); + if (!colours[who]) { colours[who] = { n: next > 5 ? 5 : next++, side: r.side, basis: r.basis }; } + return { who: who, n: colours[who].n }; + } + var fig = el('figure', { 'class': 'painting', 'data-painting': child }); + var grid = el('div', { 'class': 'paint-grid', role: 'img', 'aria-label': 'Stretches ' + child + ' got from each parent, ' + + 'per chromosome; the table below lists them' }); + lengths.forEach(function (cl) { + var chrom = cl[0]; + add(grid, el('span', { 'class': 'paint-chr' }, chrom.replace(/^chr/, ''))); + var both = el('span', { 'class': 'paint-tracks' }); + ['paternal', 'maternal'].forEach(function (side) { + var track = el('span', { 'class': 'paint-track ' + side }); + track.style.setProperty('--len', (100 * cl[1] / longest).toFixed(2) + '%'); + rows.filter(function (r) { return r.chrom === chrom && r.side === side; }).forEach(function (r) { + var a = parseInt(r.start, 10), b = parseInt(r.end, 10); + if (!isFinite(a) || !isFinite(b) || b < a) { return; } + var c = colourOf(r); + var seg = el('span', { 'class': 'paint-seg anc' + c.n + (r.basis === 'other copy' ? ' other' : ''), + title: SIDE[side] + '’s copy, ' + chrom + ':' + a + '-' + b + ', from ' + c.who + ' (' + r.sites + ' sites, ' + + r.discordant + ' discordant)' }); + seg.style.setProperty('--a', (100 * a / cl[1]).toFixed(3) + '%'); + seg.style.setProperty('--w', Math.max(0.4, 100 * (b - a) / cl[1]).toFixed(3) + '%'); + add(track, seg); + }); + add(both, track); + }); + add(grid, both); + }); + add(fig, grid); + var leg = el('figcaption', { 'class': 'paint-legend' }); + add(leg, el('span', { 'class': 'dim' }, 'Upper line: the father’s copy; lower: the mother’s. Colour: who the stretch came from. ')); + Object.keys(colours).forEach(function (who) { + var it = el('span', { 'class': 'paint-key' }); + add(it, el('span', { 'class': 'paint-swatch anc' + colours[who].n + (colours[who].basis === 'other copy' ? ' other' : '') }), + who + (colours[who].basis === 'other copy' ? ' (the copy not from the sequenced parent)' : '')); + add(leg, it); + }); + add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); + add(fig, leg); + var d = el('details', { 'class': 'w-fallback' }); + add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); + add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], + rows: rows.map(function (r) { return [SIDE[r.side] || r.side, r.chrom, r.start, r.end, + r.ancestor !== '.' ? r.ancestor : r.parent, r.sites, r.discordant, r.basis]; }) }, 12)); + add(fig, d); + return fig; + } + function children(f) { - var s = section('Inherited stretches', 'children'); + var s = section('Each child', 'children'); if (!f.state.children.length) { add(s, el('p', { 'class': 'dim' }, 'The pedigree names no child.')); } - f.state.children.forEach(function (c) { + var plan = {}; + (f.state.plan || []).forEach(function (r) { plan[r.analysis] = r; }); + function absent(file, analysis) { + var r = plan[analysis]; + return file + ' is not here' + (r ? ': ' + r.analysis.replace(/_/g, ' ') + ' ' + r.decision + (r.reason ? ' (' + r.reason + ')' : '') : '') + '.'; + } + f.state.children.forEach(function (c, i) { var d = el('details', { 'data-child': c.child }); + if (i === 0) { d.open = true; } add(d, el('summary', {}, c.child + ': father ' + (c.father || 'not named') + ', mother ' + (c.mother || 'not named'))); - if (c.segments) { + add(d, el('h3', {}, 'Inherited stretches')); + var rows = (f.segments || {})[c.child]; + if (rows && rows.length) { + add(d, painting(f, c.child, rows)); + } else if (c.segments) { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); } + var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); + add(d, el('h3', {}, 'De novo candidates')); + if (dn.length) { + add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + + 'check the reads before reading anything into one.')); + add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], + rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); + } else { + add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['de_novo.tsv'] ? 'None listed in de_novo.tsv for this child.' + : absent('de_novo.tsv', 'trio_phasing'))); + } + var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; }); + add(d, el('h3', {}, 'Compound-het pairs')); + if (ch.length) { + add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); + add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], + rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + } else { + add(d, el('p', { 'class': 'dim' }, (f.trio_files || {})['compound_het.tsv'] ? 'None listed in compound_het.tsv for this child.' + : absent('compound_het.tsv', 'trio_phasing'))); + } add(s, d); }); return s; } + // The members side by side: what each one's report says of the genes that differ most between people. + function sideBySide(f) { + var sb = f.side_by_side; + var s = section('Side by side', 'side-by-side'); + if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet.')); return s; } + add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); + var group = null, rows = []; + sb.rows.forEach(function (r) { + rows.push([r.group === group ? '' : r.group, r.label].concat(r.values)); + group = r.group; + }); + add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); + return s; + } + function more(f) { var s = section('Plan and reports', 'plan'); var plan = f.state.plan || []; @@ -205,7 +369,7 @@ add(page, top); var grid = el('div', { 'class': 'family-grid' }); add(grid, couples(f), related(f)); - add(page, grid, children(f)); + add(page, grid, children(f), sideBySide(f)); if ((f.state.plan || []).length || f.state.reports.length) { add(page, more(f)); } add(main, page); root.setAttribute('data-pgp-status', 'ready'); diff --git a/assets/live/live.css b/assets/live/live.css index 5827c3d4..9d3b4126 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -158,6 +158,7 @@ img.plot { max-width: 100%; height: auto; background: #fff; border-radius: 4px; .w-row.w-attn { border-left-color: var(--attn); } .w-head { margin: 0 0 4px; display: flex; flex-wrap: wrap; gap: 2px 8px; align-items: baseline; } .w-name { font-weight: 600; font-family: var(--mono); font-size: 14px; } +.w-label { font-weight: 600; } .w-val { font-variant-numeric: tabular-nums; } .w-extra { color: var(--dim); font-size: 13px; } .w-track { position: relative; height: 22px; border-radius: 4px; background: color-mix(in srgb, var(--rule) 55%, transparent); } @@ -324,6 +325,41 @@ img.pedigree, img.segments { max-width: 100%; height: auto; display: block; marg details[data-child] { border-top: 1px solid var(--rule); padding: var(--s2) 0; } details[data-child] summary { cursor: pointer; } +/* the relatedness grid (a list of pairs on a phone), a child's painting */ +.rel-grid-wrap { overflow-x: auto; } +table.rel-grid { border-collapse: separate; border-spacing: 4px; font-size: 13px; } +table.rel-grid th { color: var(--dim); font-weight: 600; text-align: left; padding: 2px 6px; white-space: nowrap; } +table.rel-grid td { background: var(--card2); border: 1px solid var(--rule); border-radius: 6px; padding: 6px 8px; vertical-align: top; + min-width: 11ch; } +table.rel-grid td span { display: block; } +table.rel-grid td .main { color: var(--text); font-weight: 600; } +table.rel-grid td .sub { color: var(--dim); font-size: 12px; } +table.rel-grid td.self, table.rel-grid td.none { background: transparent; border-style: dashed; color: var(--dim); text-align: center; } +table.rel-grid td.flag { border-color: var(--attn); border-left-width: 4px; } +table.rel-grid td .chip { margin-top: 4px; white-space: normal; display: inline-flex; } +ul.rel-pairs { display: none; list-style: none; margin: 0; padding: 0; } +ul.rel-pairs li { padding: 6px 0; border-bottom: 1px solid var(--rule); } +ul.rel-pairs li.flag { border-left: 4px solid var(--attn); padding-left: 8px; } +ul.rel-pairs .pair { font-weight: 600; } +ul.rel-pairs .chip { white-space: normal; } +.legend-line { font-size: 13px; margin: var(--s2) 0; } +details[data-child] h3 { font-size: 15px; margin: var(--s3) 0 var(--s2); } +.painting { margin: 0 0 var(--s3); } +.paint-grid { display: grid; grid-template-columns: 2.4em minmax(0, 1fr); gap: 3px 6px; align-items: center; } +.paint-chr { color: var(--dim); font-size: 12px; text-align: right; font-variant-numeric: tabular-nums; } +.paint-tracks { display: grid; gap: 2px; } +.paint-track { position: relative; display: block; height: 7px; width: var(--len); border-radius: 4px; + background: color-mix(in srgb, var(--rule) 70%, transparent); overflow: hidden; } +.paint-seg { position: absolute; top: 0; bottom: 0; left: var(--a); width: var(--w); } +.anc1 { background: var(--anc1); } .anc2 { background: var(--anc2); } .anc3 { background: var(--anc3); } +.anc4 { background: var(--anc4); } .anc5 { background: var(--anc5); } +.paint-seg.other, .paint-swatch.other { background-image: repeating-linear-gradient(45deg, transparent 0 2px, rgba(0, 0, 0, .35) 2px 4px); } +.paint-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; font-size: 13px; margin-top: var(--s2); } +.paint-key { display: inline-flex; gap: 6px; align-items: center; } +.paint-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 3px; } +.side-by-side table.rows td:first-child { color: var(--dim); font-size: 13px; } +.side-by-side table.rows td:nth-child(2) { font-family: var(--mono); font-size: 13px; } + /* --- login --- */ body.login main { display: flex; justify-content: center; padding-top: 12vh; } body.login form { width: 320px; display: flex; flex-direction: column; gap: 8px; } @@ -360,5 +396,7 @@ input[type=password] { font: inherit; background: var(--card); color: var(--text table.rows td::before { content: attr(data-label) ": "; color: var(--dim); font-size: 13px; } .look-list a { flex-direction: column; gap: 0; } .people { grid-template-columns: minmax(0, 1fr); } + .rel-grid-wrap { display: none; } + ul.rel-pairs { display: block; } } @media (prefers-reduced-motion: reduce) { * { animation: none !important; transition: none !important; scroll-behavior: auto !important; } } diff --git a/assets/live/person.js b/assets/live/person.js index 83f4bca3..6a708403 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -158,7 +158,7 @@ } function whead(name, value, extra) { var h = el('p', { 'class': 'w-head' }); - add(h, el('span', { 'class': 'w-name' }, name), ' ', el('span', { 'class': 'w-val' }, value)); + add(h, el('span', { 'class': /^[A-Z0-9-]+( \(|$)/.test(name) ? 'w-name' : 'w-label' }, name), ' ', el('span', { 'class': 'w-val' }, value)); if (extra) { add(h, ' ', el('span', { 'class': 'w-extra' }, extra)); } return h; } diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 17ec9f87..988a72e8 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -15,8 +15,13 @@ the state below, the pedigree's layout (so each person is a link), the relationship and shared- segment rows with their measured numbers, each - couple's gene rows, and each member's run progress - and attention count (from its summary.json only) + couple's gene rows, each child's inherited + stretches (_segments.tsv), the de novo and + compound-het candidates (de_novo.tsv, + compound_het.tsv), each member's run progress and + attention count, and the members side by side: + CYP2D6, the CPIC genes, HLA, the haplogroups and + the blood groups (from each summary.json only) state.json gets plugins.family: {"kind": "family", ...} on a family's page, {"kind": "member", "families": [...]} on the page of a sample in a family, @@ -182,7 +187,8 @@ def family_page(name, d): st = family_state(name, d) import render_report # the couple screen's caveat, word for word out = {"state": st, "layout": None, "relationships": [], "ibd_pairs": [], "couples": {}, - "couple_note": render_report.COUPLE_NOTE} + "couple_note": render_report.COUPLE_NOTE, "segments": {}, "de_novo": [], "compound_het": [], + "trio_files": {}, "side_by_side": None} if st.get("error"): return out order, members = members_of(d) @@ -200,6 +206,9 @@ def family_page(name, d): "partner_b", "findings_b", "explanation") rows = sorted(({k: r.get(k) for k in keep} for r in rows), key=lambda r: order_of.get(r.get("result"), 3)) out["couples"]["%s %s" % (a, b)] = rows + out.update(child_depth(d, order, members)) + import report_charts # GRCh38's lengths, for the painting of each child's stretches + out["chrom_lengths"] = [[c, n] for c, n in report_charts.GRCH38 if c != "chrY"] # Step 38 writes the verdict "ok" for a pair that matches its declared relation. The page says that # in words: no word on the page reads as an all-clear (state.json keeps the file's own value). out["state"] = dict(st, relationships=[dict(r, verdict=verdict_words(r.get("verdict"))) @@ -209,10 +218,105 @@ def family_page(name, d): return out +SEGMENT_KEYS = ("side", "parent", "chrom", "start", "end", "length_bp", "sites", "discordant", "ancestor", "basis") +DE_NOVO_KEYS = ("child", "chrom", "pos", "ref", "alt", "child_gt", "father_gt", "mother_gt", "child_dp", "child_gq", + "child_ab", "quality") +COMP_KEYS = ("child", "gene", "variant_a", "variant_b", "a_from", "b_from", "phase") +PHASE_ORDER = {"in trans": 0, "unknown": 1, "in cis": 2} + + +def child_depth(d, order, members): + """Each child's stretches (step 38's _segments.tsv) and the trio tables, as the steps wrote them: + {segments: {child: [rows]}, de_novo: [rows], compound_het: [rows, in trans first]}.""" + segs = {} + for s in order: + if not (members[s]["father"] or members[s]["mother"]): + continue + rows = read_tsv(os.path.join(d, "%s_segments.tsv" % s)) + if rows: + segs[s] = [{k: r.get(k) for k in SEGMENT_KEYS} for r in rows] + comp = [{k: r.get(k) for k in COMP_KEYS} for r in read_tsv(os.path.join(d, "compound_het.tsv"))] + comp.sort(key=lambda r: PHASE_ORDER.get(r.get("phase"), 1)) + return {"segments": segs, + "de_novo": [{k: r.get(k) for k in DE_NOVO_KEYS} for r in read_tsv(os.path.join(d, "de_novo.tsv"))], + "compound_het": comp, + "trio_files": {f: os.path.isfile(os.path.join(d, f)) for f in ("de_novo.tsv", "compound_het.tsv")}} + + +def _sec(summary, key): + sec = ((summary or {}).get("sections") or {}).get(key) or {} + return sec.get("data") or {} if sec.get("state") in ("ok", "stale") else None + + +def side_values(summary): + """[(group, label, value)] of one member's summary.json, as its report prints them; nothing computed.""" + out = [] + c = (summary or {}).get("cyp2d6") or {} + calls = ", ".join("%s %s" % (k, v) for k, v in (c.get("calls") or {}).items() if v) + if calls: + agree = c.get("agree") + out.append(("Medicines", "CYP2D6", calls + (" (callers agree)" if agree else " (callers differ)" if agree is False + else ""))) + cpic = _sec(summary, "cpic") + for g in (cpic or {}).get("genes") or []: + ph = g.get("phenotype") + out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else ""))) + for l in (_sec(summary, "hla") or {}).get("loci") or []: + out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed")) + mt = _sec(summary, "haplogroup") + if mt is not None: + out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or ".")) + y = _sec(summary, "y_tree") or _sec(summary, "y_haplogroup") + if y is not None: + out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or ".")) + bt = _sec(summary, "bloodtype") + if bt is not None: + out.append(("Blood groups", "ABO", bt.get("abo") or next((x.get("phenotype") for x in bt.get("systems") or [] + if x.get("system") == "ABO"), "."))) + out.append(("Blood groups", "RhD", bt.get("rhd") or next((x.get("phenotype") for x in bt.get("systems") or [] + if x.get("system") == "RhD"), "."))) + return out + + +def side_by_side(names, summaries): + """The members' results in one table: {members, rows: [{group, label, values}]}. A member with no + summary.json yet says so in its column; a row a member's summary does not have is '.'.""" + per = {n: side_values(summaries.get(n)) for n in names} + rows, seen = [], {} + for n in names: + for g, label, _ in per[n]: + if (g, label) not in seen: + seen[(g, label)] = len(rows) + rows.append({"group": g, "label": label, "values": []}) + groups = ["Medicines", "HLA", "Haplogroups", "Blood groups"] + rows.sort(key=lambda r: (groups.index(r["group"]) if r["group"] in groups else 9, seen[(r["group"], r["label"])])) + for r in rows: + for n in names: + if summaries.get(n) is None: + r["values"].append("no summary yet") + else: + r["values"].append(next((v for g, lab, v in per[n] if (g, lab) == (r["group"], r["label"])), ".")) + return {"members": list(names), "rows": rows} + + def verdict_words(v): return "matches the declared relation" if (v or "").strip().lower() == "ok" else v +def member_summary(live, d): + """A member run's summary.json as a dict, or None (none yet, unreadable, or a link out of the run).""" + import json + p = os.path.join(d, "summary.json") if d else None + if not p or not os.path.isfile(p) or not live.inside(p, os.path.realpath(d)) or not live.inside(p, live.root): + return None + try: + with open(p) as f: + got = json.load(f) + except (OSError, ValueError): + return None + return got if isinstance(got, dict) else None + + def member_state(root, sample): fams = [] for name, d in family_dirs(root): @@ -262,6 +366,14 @@ def page_data(req, m): "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), "attention": n, "results": mtime is not None} out["runs"] = runs + # The members side by side, from each sequenced member's summary.json alone (read as the home + # page's attention count reads it: never a file linking out of its run). + seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] + sums = {} + for s_ in seq: + r = live.run(s_) if live else None + sums[s_] = member_summary(live, r["dir"] or os.path.join(live.root, r["name"])) if r else None + out["side_by_side"] = side_by_side(seq, sums) if seq else None app.log("family", name, "family.json", 200) return req.send_json(out) diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index 1d62b912..51b2eb0b 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -16,8 +16,13 @@ runs/DEMO-001/... the files the plugins point at (the charts, the search index, the re-check), the report as report.html and the files it links to - runs/DEMO-FAMILY/ the family page (assets/live/family.html) of DEMO-001 and its - invented partner, with family.json as the family plugin serves it + runs/DEMO-002/, runs/DEMO-003/ the person pages of DEMO-001's invented partner and their son, + each a finished run of one frame (tests/demo/make_demo_sample.py + --sample, with the findings that show the rest of the widgets) + runs/DEMO-FAMILY/ the family page (assets/live/family.html) of the trio, with + family.json as the family plugin serves it: the invented + relationship, shared-stretch, de novo and compound-het files of + tests/fixtures/live/demo/DEMO-FAMILY and the members side by side family/DEMO-FAMILY/ the pedigree and the files the family page opens No state is written by hand. The frames are bin/owngenome_live.py's own @@ -28,8 +33,9 @@ pictures: GDIR/DEMO-001 when given (after tests/demo/retake-screenshots.sh ran its steps), else written now by tests/demo/make_demo_sample.py with N records (default 20000) and its report written by bin/render_report.py with step 24's -arguments. Steps 27, 28 and 36 run in containers and are not run here, so -without GDIR their sections stay missing in the report and on the page. +arguments. Steps 28 and 36 run in containers and are not run here, so +without GDIR their sections stay missing in the report and on the page; +step 27's CPIC table is written by make_demo_sample.py with step 27's parser. What the replay changes, and why: - step 24's files (the report, the charts, the search index, summary.json) @@ -79,6 +85,9 @@ SAMPLE = "DEMO-001" FAMILY = "DEMO-FAMILY" PARTNER = "DEMO-002" +# The family's other people: name, sex, seed. Each is a finished run with no messages (its own +# logs/run_status.tsv, closed with a finished_utc line as run-all.sh closes it). +OTHERS = (("DEMO-002", "female", 2), ("DEMO-003", "male", 3)) FIX = os.path.join(REPO, "tests", "fixtures", "live") WEBLOG = os.path.join(FIX, "demo-001.weblog.jsonl") EXTRA = os.path.join(FIX, "demo") @@ -105,16 +114,28 @@ def epoch(text): # --- the run folder ------------------------------------------------------------------------- -def report(d): +def report(d, sample=SAMPLE): """bin/render_report.py with the arguments scripts/24-html-report.sh gives it.""" subprocess.run([sys.executable, os.path.join(BIN, "render_report.py"), "--assets", os.path.join(REPO, "assets"), - "--sample", SAMPLE, "--sample-dir", d, "--json", os.path.join(d, "summary.json"), - "-o", os.path.join(d, f"{SAMPLE}_report.html"), - "--search-index", os.path.join(d, f"{SAMPLE}_search_index.json"), - "--charts-json", os.path.join(d, f"{SAMPLE}_charts.json")], + "--sample", sample, "--sample-dir", d, "--json", os.path.join(d, "summary.json"), + "-o", os.path.join(d, f"{sample}_report.html"), + "--search-index", os.path.join(d, f"{sample}_search_index.json"), + "--charts-json", os.path.join(d, f"{sample}_charts.json")], check=True, stdout=subprocess.DEVNULL) +def other_person(root, name, sex, seed, records): + """root/: an invented member written by make_demo_sample.py --sample, with its report, as a + finished run (run-all.sh's finished_utc line, 30 hours after the invented start).""" + subprocess.run([sys.executable, os.path.join(HERE, "make_demo_sample.py"), "--genome-dir", root, "--records", + str(records), "--sample", name, "--sex", sex, "--seed", str(seed)], check=True, stdout=subprocess.DEVNULL) + d = os.path.join(root, name) + import make_demo_sample + with open(os.path.join(d, "logs", "run_status.tsv"), "a") as f: + f.write(f"meta\tfinished_utc\t{time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(make_demo_sample.RUN_STARTED + 30 * 3600))}\n") + report(d, name) + + def prepare(root, genome_dir=None, records=20000): """root/DEMO-001 with its report, the family folder and the held-back files; returns the held folder.""" d = os.path.join(root, SAMPLE) @@ -137,6 +158,11 @@ def prepare(root, genome_dir=None, records=20000): for f in HELD: if os.path.isfile(os.path.join(d, f)): shutil.move(os.path.join(d, f), os.path.join(held, f)) + for name, sex, seed in OTHERS: + if genome_dir and os.path.isdir(os.path.join(genome_dir, name)): + shutil.copytree(os.path.join(genome_dir, name), os.path.join(root, name), symlinks=False) + else: + other_person(root, name, sex, seed, max(2000, records // 4)) fam = os.path.join(root, FAMILY) shutil.copytree(os.path.join(EXTRA, FAMILY), fam) couple = os.path.join(d, "couple", f"{SAMPLE}_couple_carrier.tsv") @@ -231,9 +257,20 @@ def snap(t): results = live_results.build(SAMPLE, live.summary(SAMPLE, d, live.base_state(run)), owngenome_live.Live.run_files(None, {"dir": d, "name": SAMPLE})) results["generated_utc"] = checked + # The other people: one frame each, their state at the end, and their results. + others = {} + for name, _, _ in OTHERS: + r, od = live.run(name), os.path.join(root, name) + st = live.state(r) + st["generated_utc"] = checked + st["plugins"] = relink(st["plugins"]) + res = live_results.build(name, live.summary(name, od, live.base_state(r)), + owngenome_live.Live.run_files(None, {"dir": od, "name": name})) + res["generated_utc"] = checked + others[name] = {"frames": [{"t": checked, "state": st}], "results": res} finally: time.time, run_headroom._statvfs, run_headroom.physical_memory = real - return frames, {"t": checked, "state": fam, "index": index, "results": results}, app + return frames, {"t": checked, "state": fam, "index": index, "results": results, "others": others}, app # --- the static folder ---------------------------------------------------------------------- @@ -281,6 +318,15 @@ def family_json(root, index): by = {r["name"]: r for r in index["runs"]} out["runs"] = {p["sample"]: {k: by[p["sample"]][k] for k in ("url", "state", "planned", "done", "attention", "results")} for p in out["state"].get("members") or [] if p["sample"] in by} + seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] + sums = {} + for name in seq: + try: + with open(os.path.join(root, name, "summary.json")) as f: + sums[name] = json.load(f) + except OSError: + sums[name] = None + out["side_by_side"] = owngenome_live_family.side_by_side(seq, sums) return relink(out) @@ -297,28 +343,18 @@ def write_json(path, obj): json.dump(obj, f, separators=(",", ":")) -def write_site(out, root, frames, fam): - if os.path.exists(out): - shutil.rmtree(out) - shutil.copytree(ASSETS, os.path.join(out, "assets"), - ignore=shutil.ignore_patterns("index.html", "login.html", "home.html", "family.html")) - plugin_dir = os.path.join(ASSETS, "plugins") - plugin_js = sorted(f for f in os.listdir(plugin_dir) if owngenome_live.ASSET_RE.match(f) and f.endswith(".js")) \ - if os.path.isdir(plugin_dir) else [] - - with open(os.path.join(out, "index.html"), "w") as f: - f.write(home_page(fam["index"])) - write_json(os.path.join(out, "runs.json"), unroot(fam["index"])) - - sd, d = os.path.join(out, "runs", SAMPLE), os.path.join(root, SAMPLE) +def person_site(out, root, name, plugin_js, frames, results, autoplay=False): + """runs//: the person page, its frames, results.json, the report and the files it links to.""" + sd, d = os.path.join(out, "runs", name), os.path.join(root, name) os.makedirs(sd) with open(os.path.join(sd, "index.html"), "w") as f: - f.write(run_page(SAMPLE, plugin_js)) - write_json(os.path.join(sd, "frames.json"), {"interval_ms": INTERVAL_MS, "autoplay": True, "frames": frames}) - write_json(os.path.join(sd, "results.json"), fam["results"]) - files = {f"{SAMPLE}_report.html": "report.html"} + f.write(run_page(name, plugin_js)) + write_json(os.path.join(sd, "frames.json"), dict({"interval_ms": INTERVAL_MS, "frames": frames}, + **({"autoplay": True} if autoplay else {}))) + write_json(os.path.join(sd, "results.json"), results) + files = {f"{name}_report.html": "report.html"} # The files the report links to, as the live server lists them (Live.run_files reads no state). - files.update({rel: rel for rel in owngenome_live.Live.run_files(None, {"dir": d, "name": SAMPLE})}) + files.update({rel: rel for rel in owngenome_live.Live.run_files(None, {"dir": d, "name": name})}) for v in frames[-1]["state"]["plugins"].values(): if isinstance(v, dict) and isinstance(v.get("file"), str): files[v["file"]] = v["file"] @@ -334,6 +370,25 @@ def write_site(out, root, frames, fam): "own computer. The demo has no reads to show.

Back to the run

", "../../../assets/live.css")) + +def write_site(out, root, frames, fam): + if os.path.exists(out): + shutil.rmtree(out) + shutil.copytree(ASSETS, os.path.join(out, "assets"), + ignore=shutil.ignore_patterns("index.html", "login.html", "home.html", "family.html")) + plugin_dir = os.path.join(ASSETS, "plugins") + plugin_js = sorted(f for f in os.listdir(plugin_dir) if owngenome_live.ASSET_RE.match(f) and f.endswith(".js")) \ + if os.path.isdir(plugin_dir) else [] + + with open(os.path.join(out, "index.html"), "w") as f: + f.write(home_page(fam["index"])) + write_json(os.path.join(out, "runs.json"), unroot(fam["index"])) + + person_site(out, root, SAMPLE, plugin_js, frames, fam["results"], autoplay=True) + for name, _, _ in OTHERS: + o = fam["others"][name] + person_site(out, root, name, plugin_js, o["frames"], o["results"]) + fd = os.path.join(out, "runs", FAMILY) os.makedirs(fd) with open(os.path.join(fd, "index.html"), "w") as f: diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 3f2492d8..a4957fe3 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -2,6 +2,7 @@ """make_demo_sample.py: write DEMO-001, the invented sample of the docs pictures. make_demo_sample.py --genome-dir DIR [--seed 1] [--records 4700000] + [--sample DEMO-001] [--sex male] make_demo_sample.py check --genome-dir DIR The first form writes DIR/DEMO-001 with every file steps 24, 27 and 28 read, @@ -36,6 +37,16 @@ Paraphase card (a PacBio HiFi step, though the invented reads are short) and the couple card, with an invented partner DEMO-002. +--sample and --sex write another invented person the same way: the live-site +demo (tests/demo/build_live_demo.py) writes DEMO-002, DEMO-001's partner +(female), and DEMO-003, their son, so the family page has a trio. A female +gets no Y files and her sex checks read two X. A few invented findings make +every live-site widget show once in the demo (FINDINGS below): DEMO-002 an +FMR1 allele in Stranger's premutation range, a CHIP call and a mosaic loss +MoChA reports, DEMO-003 an AnnotSV class 4 deletion. Every sample gets +Stranger's thresholds beside ExpansionHunter's counts and step 27's CPIC +table, written by bin/pgx_parse.py from the PharmCAT report. + The second form reads DIR/DEMO-001/summary.json (step 24 writes it) and fails when a section is not "ok", so a new report card that this script does not feed yet is caught before a picture shows "Not run". @@ -44,8 +55,10 @@ """ import argparse import calendar +import contextlib import copy import gzip +import io import json import math import os @@ -82,6 +95,15 @@ # When the invented run started: 2026-10-01 09:00 UTC, the time of the demo's # PharmCAT report too (logs/run_status.tsv, the report's "Latest run" line). RUN_STARTED = calendar.timegm((2026, 10, 1, 9, 0, 0, 0, 0, 0)) +# The invented findings of the demo's other people (--sample), one of each live-site widget's marks. +FINDINGS = {"DEMO-002": {"fmr1", "chip", "mocha"}, "DEMO-003": {"annotsv"}} +# Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. +STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), + "ATXN3": (44, 60), "AR": (34, 38), "FXN": (33, 66)} + + +def has(finding): + return finding in FINDINGS.get(SAMPLE, ()) def put(path, text, gz=False): @@ -258,6 +280,12 @@ def called(gene): changed["MT-RNR1"] = "Reference" prune_annotations(report, changed) put(os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), json.dumps(report, indent=1) + "\n") + # Step 27's tables from that report, by the parser step 27 runs (bin/pgx_parse.py). + import pgx_parse + with contextlib.redirect_stdout(io.StringIO()): + if pgx_parse.main(["cpic-report", "--sample", SAMPLE, "--report", + os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), "--outdir", os.path.join(d, "cpic")]): + raise SystemExit("pgx_parse cpic-report failed") return outside @@ -382,7 +410,8 @@ def cumulative(mean, sd, gap): put(os.path.join(i, "indexcov-indexcov.ped"), "#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\tCNchrX\tCNchrY\tbins.out\tbins.lo\tbins.hi\t" "bins.in\tslope\tp.out\tPC1\tPC2\tPC3\tPC4\tPC5\n" - f"unknown\t{SAMPLE}\t-9\t-9\t{1 if SEX == 'male' else 2}\t-9\t1.01\t0.98\t312\t205\t107\t174820\t0.993\t0.0018\t" + f"unknown\t{SAMPLE}\t-9\t-9\t{1 if SEX == 'male' else 2}\t-9\t{'1.01' if SEX == 'male' else '1.99'}\t" + f"{'0.98' if SEX == 'male' else '0.01'}\t312\t205\t107\t174820\t0.993\t0.0018\t" "0\t0\t0\t0\t0\n") roc = ["#chrom\tcov\t" + SAMPLE] for chrom, _ in CHROMS: @@ -408,7 +437,7 @@ def sample_qc(d): "gt_depth_sd\tdepth_mean\tdepth_sd\tab_mean\tab_std\tn_hom_ref\tn_het\tn_hom_alt\tn_unknown\tp_middling_ab\t" "X_depth_mean\tX_n\tX_hom_ref\tX_het\tX_hom_alt\tY_depth_mean\tY_n\n" f"{SAMPLE}\t{SAMPLE}\t-9\t-9\t{sex}\t-9\t-9\t31.4\t7.2\t31.4\t7.2\t0.41\t0.08\t9817\t6904\t4383\t196\t0.01\t" - "15.6\t612\t352\t4\t256\t14.9\t41\n") + + ("15.6\t612\t352\t4\t256\t14.9\t41\n" if SEX == "male" else "31.2\t612\t300\t210\t102\t0.1\t0\n")) put(os.path.join(q, "somalier", f"{SAMPLE}.pairs.tsv"), "#sample_a\tsample_b\trelatedness\tibs0\tibs2\thom_concordance\thets_a\thets_b\thets_ab\tshared_hets\t" "hom_alts_a\thom_alts_b\tshared_hom_alts\tn\tx_ibs0\tx_ibs2\texpected_relatedness\n") @@ -546,8 +575,17 @@ def write(path, recs, extra=()): # Step 05's class 4 and 5 SVs, checked (bin/annotsv_postfilter.py's # columns): none, as in most genomes. import annotsv_postfilter + cols = annotsv_postfilter.KEEP + annotsv_postfilter.ADDED + rows = [] + if has("annotsv"): + # One class 4 deletion over an invented gene, from Manta, kept by step 05's rules. + rows.append({"AnnotSV_ID": "chr16_2084000_2096000_DEL_1", "ID": "MantaDEL:1", "SV_chrom": "chr16", + "SV_start": "2084000", "SV_end": "2096000", "SV_length": "-12000", "SV_type": "DEL", + "Gene_name": "G0533", "ACMG_class": "4", "AnnotSV_ranking_score": "0.92", "Caller": "Manta", + "Exon_edge": "no", "Exon_edge_detail": ".", "Population_AF": "", "Population_source": "", + "Common": "no", "AnnotSV_class": "4", "Depth_support": ".", "Rule": "", "Rule_detail": ""}) put(os.path.join(d, "annotsv", f"{SAMPLE}_sv_class4_5_checked.tsv"), - "\t".join(annotsv_postfilter.KEEP + annotsv_postfilter.ADDED) + "\n") + "\t".join(cols) + "\n" + "".join("\t".join(r[k] for k in cols) + "\n" for r in rows)) # --- repeats, telomeres, ROH, mitochondria, Y: steps 09, 10, 11, 12, 20, 37 -------------- @@ -560,18 +598,30 @@ def write(path, recs, extra=()): def repeats_and_more(d, rng): - eh = [] + eh, stranger = [], [] for rep, c, pos, unit, lo, hi in EH_RANGES: hap = c == "chrX" and SEX == "male" a = sorted(rng.randint(lo, hi) for _ in range(1 if hap else 2)) + if rep == "FMR1" and has("fmr1") and not hap: + a = [a[0], 62] # an allele in Stranger's premutation range (above 54, under 200) ref = (lo + hi) // 2 gt = "1" if hap else ("1/2" if a[0] != a[1] else "1/1") cn = "/".join(map(str, a)) alts = ",".join(f"" for x in dict.fromkeys(a)) - eh.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\tEND={pos + ref * len(unit)};REF={ref};RL={ref * len(unit)};" - f"RU={unit};VARID={rep};REPID={rep}\tGT:SO:REPCN:LC\t{gt}:SPANNING:{cn}:{rng.uniform(28, 34):.6f}\n") - put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh.vcf"), - vcf_header(['##ALT=']) + "".join(eh)) + info = f"END={pos + ref * len(unit)};REF={ref};RL={ref * len(unit)};RU={unit};VARID={rep};REPID={rep}" + lc = f"{rng.uniform(28, 34):.6f}" + eh.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info}\tGT:SO:REPCN:LC\t{gt}:SPANNING:{cn}:{lc}\n") + # Step 9b's Stranger copy: the same record with the catalog's thresholds, the worst status of + # the alleles, and ExpansionHunter's 95% interval of each (REPCI), a few repeats wide past 50. + nmax, pmin = STR_THRESHOLDS[rep] + worst = ("full_mutation" if any(x >= pmin for x in a) else "pre_mutation" if any(x > nmax for x in a) + else "normal") + ci = "/".join(f"{x}-{x}" if x < 50 else f"{x - 4}-{x + 5}" for x in a) + stranger.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info};STR_STATUS={worst};STR_NORMAL_MAX={nmax};" + f"STR_PATHOLOGIC_MIN={pmin}\tGT:SO:REPCN:REPCI:LC\t{gt}:SPANNING:{cn}:{ci}:{lc}\n") + head = vcf_header(['##ALT=']) + put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh.vcf"), head + "".join(eh)) + put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh_stranger.vcf"), head + "".join(stranger)) put(os.path.join(d, "telomere", SAMPLE, f"{SAMPLE}_summary.tsv"), "PID\tsample\ttotal_reads\tread_lengths\trepeat_threshold_set\trepeat_threshold_used\tintratelomeric_reads\t" @@ -611,6 +661,8 @@ def repeats_and_more(d, rng): put(os.path.join(d, "mito", f"{SAMPLE}_haplocheck.txt"), '"Sample"\t"Contamination Status"\t"Contamination Level"\t"Distance"\t"Sample Coverage"\n' f'"{SAMPLE}"\t"NO"\t"ND"\t"0"\t"2861"\n') + if SEX != "male": + return # no Y chromosome: step 37 has nothing to place put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_haplogroup.txt"), "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" f"{SAMPLE}_sorted\tI-M253\tM253\t4021876\t1187\t0.982\t1.0\t0.991\t0.982\n") @@ -713,11 +765,13 @@ def mosaic(d, rng): start = (length // 4) * (k + 1) // 500 * 500 lines += [f"{chrom}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(mean, mean * 0.12)):.2f}" for s in range(start, start + 200_000, 500)] - ymean = MEAN_DEPTH / 2 * rng.uniform(0.98, 1.0) - for start, end in mosaic_sex_loss.SINGLE_COPY["Y"]: + # A male's single-copy chrY at half the depth; a female's chrX (two copies) at the full depth. + sx = "Y" if SEX == "male" else "X" + ymean = MEAN_DEPTH / (2 if SEX == "male" else 1) * rng.uniform(0.98, 1.0) + for start, end in mosaic_sex_loss.SINGLE_COPY[sx][:10]: s0 = (start // 500 + 1) * 500 - lines += [f"chrY\t{s}\t{s + 500}\t{max(0.0, rng.gauss(ymean, ymean * 0.14)):.2f}" - for s in range(s0, end - 500, 500)] + lines += [f"chr{sx}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(ymean, ymean * 0.14)):.2f}" + for s in range(s0, min(end, s0 + 1_000_000) - 500, 500)] put(path, "\n".join(lines) + "\n", gz=True) if mosaic_sex_loss.main(["--table", os.path.join(d, "qc", f"{SAMPLE}_sample_qc.tsv"), "--regions", path]): raise SystemExit("mosaic_sex_loss failed") @@ -813,7 +867,10 @@ def region(total, rsi, **kw): def couple(d, rng): """Step 39 with an invented partner, DEMO-002: each carries one change in a different recessive gene of the carrier list, so no gene is shared and the - card names none. Rows by bin/couple_report.py's own screen_couple.""" + card names none. Rows by bin/couple_report.py's own screen_couple. Only + DEMO-001 carries the table: the family page reads one copy of it.""" + if SAMPLE != "DEMO-001": + return import couple_report genes = couple_report.read_genes(os.path.join(REPO, "assets", "carrier_genes.tsv")) ar = sorted(g for g, v in genes.items() if v["inheritance"] == "AR" and "ACMG" in v["lists"]) @@ -988,7 +1045,15 @@ def chip(d, rng): rows.append({"gene": r["gene"], "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_report.detection_limit(mean):.3f}", "status": "covered"}) - put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(chip_report.OUT_COLUMNS) + "\n") + calls = [] + if has("chip"): + # One small DNMT3A clone, above the detection limit at this depth (bin/chip_report.py's columns). + calls.append({"gene": "DNMT3A", "chrom": "chr2", "pos": "25234373", "ref": "C", "alt": "T", + "consequence": "missense_variant", "protein_change": "p.R882H", "exon": "23", "vaf": "0.161", + "alt_reads": "8", "depth": "50", "filter": "PASS", "status": "chip", "class": "hotspot", + "note": "a published CHIP hotspot"}) + put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(chip_report.OUT_COLUMNS) + "\n" + + "".join("\t".join(c[k] for k in chip_report.OUT_COLUMNS) + "\n" for c in calls)) put(os.path.join(d, "chip", f"{SAMPLE}_chip_depth.tsv"), "\t".join(chip_report.DEPTH_COLUMNS) + "\n" + "".join("\t".join(x[c] for c in chip_report.DEPTH_COLUMNS) + "\n" for x in rows)) @@ -1117,7 +1182,14 @@ def mocha(d): stats = [{"sample_id": SAMPLE, "call_rate": "0.9981", "cov_median": "31", "baf_auto": "0.0124", "n_hets": "2100000"}] sites = [(f"chr{i}", 10001, 10001 + 100_000_000, 120_000, 95_000, 94_000) for i in range(1, 23)] - rows, summary = mr.report(SAMPLE, [], stats, sites, "beagle") + calls = [] + if has("mocha"): + # A loss of 20q in about 12% of the blood cells, which MoChA's filters keep and the floor lets through. + calls.append({"sample_id": SAMPLE, "chrom": "chr20", "beg_GRCh38": "30100000", "end_GRCh38": "42300000", + "length": "12200000", "p_arm": "N", "q_arm": "Y", "type": "Loss", "cf": "0.120", + "bdev": "0.031", "bdev_se": "0.004", "n50_hets": "350000", "lod_baf_conc": "nan", + "rel_cov": "1.86", "lod_baf_phase": "48.2", "lod_lrr_baf": "35.1", "n_hets": "4210"}) + rows, summary = mr.report(SAMPLE, calls, stats, sites, "beagle") os.makedirs(os.path.join(d, "mosaic"), exist_ok=True) with open(os.path.join(d, "mosaic", f"{SAMPLE}_mocha.tsv"), "w", encoding="utf-8") as f: f.write("\t".join(mr.CALL_COLUMNS) + "\n") @@ -1208,6 +1280,10 @@ def run_status(d, started): # --- the two commands ---------------------------------------------------------------- def make(a): + global SAMPLE, SEX + SAMPLE, SEX = a.sample, a.sex + if not re.match(r"^DEMO-[0-9]{3}$", SAMPLE): + raise SystemExit(f"--sample {SAMPLE}: an invented name of the form DEMO-001") # A fixed start, so the report's "Latest run" line is the same on every # retake; the files written now are newer, so none is marked stale. started = RUN_STARTED @@ -1227,7 +1303,7 @@ def make(a): ("ancestry_fine", lambda r: ancestry_fine(d, r)), ("paralogs", lambda r: paralogs(d)), ("couple", lambda r: couple(d, r)), ("consumer_export", lambda r: consumer_export(d)), ("bloodtype", lambda r: bloodtype(d)), ("traits", lambda r: traits(d)), - ("y_str", lambda r: y_str(d, r)), ("viral", lambda r: viral(d)), + ("y_str", lambda r: y_str(d, r) if SEX == "male" else None), ("viral", lambda r: viral(d)), ("lpa", lambda r: lpa(d, r)), ("chip", lambda r: chip(d, r)), ("archaic", lambda r: archaic(d, r)), ("ancient", lambda r: ancient(d, r)), ("exomiser", lambda r: exomiser(d)), ("mocha", lambda r: mocha(d)), @@ -1270,6 +1346,8 @@ def main(argv=None): ap.add_argument("--genome-dir", required=True, help="an empty folder; DEMO-001 is written inside it") ap.add_argument("--seed", type=int, default=1) ap.add_argument("--records", type=int, default=4_700_000, help="variant records in the VCF (default 4700000)") + ap.add_argument("--sample", default="DEMO-001", help="the invented sample to write (default DEMO-001)") + ap.add_argument("--sex", default="male", choices=("male", "female")) return make(ap.parse_args(argv)) diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/DEMO-003_segments.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/DEMO-003_segments.tsv new file mode 100644 index 00000000..92180d5f --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/DEMO-003_segments.tsv @@ -0,0 +1,81 @@ +child side parent chrom start end length_bp sites discordant ancestor basis +DEMO-003 paternal DEMO-001 chr1 60000 121900000 121840000 64533 5 . parent +DEMO-003 paternal DEMO-001 chr1 124900000 248896422 123996422 55803 3 . parent +DEMO-003 maternal DEMO-002 chr1 60000 121900000 121840000 55992 2 . parent +DEMO-003 maternal DEMO-002 chr1 124900000 248896422 123996422 60752 11 . parent +DEMO-003 paternal DEMO-001 chr2 60000 92400000 92340000 43618 1 . parent +DEMO-003 paternal DEMO-001 chr2 95400000 242133529 146733529 65360 11 . parent +DEMO-003 maternal DEMO-002 chr2 60000 92400000 92340000 41613 0 . parent +DEMO-003 maternal DEMO-002 chr2 95400000 242133529 146733529 70173 4 . parent +DEMO-003 paternal DEMO-001 chr3 60000 89400000 89340000 43944 5 . parent +DEMO-003 paternal DEMO-001 chr3 92400000 198235559 105835559 52576 2 . parent +DEMO-003 maternal DEMO-002 chr3 60000 89400000 89340000 40152 8 . parent +DEMO-003 maternal DEMO-002 chr3 92400000 198235559 105835559 49317 7 . parent +DEMO-003 paternal DEMO-001 chr4 60000 48500000 48440000 25738 1 . parent +DEMO-003 paternal DEMO-001 chr4 51500000 190154555 138654555 73207 5 . parent +DEMO-003 maternal DEMO-002 chr4 60000 48500000 48440000 22773 2 . parent +DEMO-003 maternal DEMO-002 chr4 51500000 190154555 138654555 62260 7 . parent +DEMO-003 paternal DEMO-001 chr5 60000 47300000 47240000 24004 3 . parent +DEMO-003 paternal DEMO-001 chr5 50300000 181478259 131178259 61441 14 . parent +DEMO-003 maternal DEMO-002 chr5 60000 47300000 47240000 23145 5 . parent +DEMO-003 maternal DEMO-002 chr5 50300000 181478259 131178259 57610 7 . parent +DEMO-003 paternal DEMO-001 chr6 60000 58300000 58240000 29222 1 . parent +DEMO-003 paternal DEMO-001 chr6 61300000 170745979 109445979 51698 12 . parent +DEMO-003 maternal DEMO-002 chr6 60000 58300000 58240000 26269 0 . parent +DEMO-003 maternal DEMO-002 chr6 61300000 170745979 109445979 51747 8 . parent +DEMO-003 paternal DEMO-001 chr7 60000 58600000 58540000 31694 4 . parent +DEMO-003 paternal DEMO-001 chr7 61600000 159285973 97685973 43109 7 . parent +DEMO-003 maternal DEMO-002 chr7 60000 58600000 58540000 27076 6 . parent +DEMO-003 maternal DEMO-002 chr7 61600000 159285973 97685973 51549 2 . parent +DEMO-003 paternal DEMO-001 chr8 60000 43700000 43640000 21668 0 . parent +DEMO-003 paternal DEMO-001 chr8 46700000 145078636 98378636 48272 9 . parent +DEMO-003 maternal DEMO-002 chr8 60000 43700000 43640000 20860 1 . parent +DEMO-003 maternal DEMO-002 chr8 46700000 145078636 98378636 49811 1 . parent +DEMO-003 paternal DEMO-001 chr9 60000 41500000 41440000 19409 4 . parent +DEMO-003 paternal DEMO-001 chr9 44500000 138334717 93834717 43043 6 . parent +DEMO-003 maternal DEMO-002 chr9 60000 41500000 41440000 18409 2 . parent +DEMO-003 maternal DEMO-002 chr9 44500000 138334717 93834717 48619 4 . parent +DEMO-003 paternal DEMO-001 chr10 60000 38300000 38240000 18949 0 . parent +DEMO-003 paternal DEMO-001 chr10 41300000 133737422 92437422 42676 1 . parent +DEMO-003 maternal DEMO-002 chr10 60000 38300000 38240000 19129 3 . parent +DEMO-003 maternal DEMO-002 chr10 41300000 133737422 92437422 42093 10 . parent +DEMO-003 paternal DEMO-001 chr11 60000 51900000 51840000 25079 5 . parent +DEMO-003 paternal DEMO-001 chr11 54900000 135026622 80126622 35266 1 . parent +DEMO-003 maternal DEMO-002 chr11 60000 51900000 51840000 24156 4 . parent +DEMO-003 maternal DEMO-002 chr11 54900000 135026622 80126622 44366 3 . parent +DEMO-003 paternal DEMO-001 chr12 60000 34000000 33940000 15308 1 . parent +DEMO-003 paternal DEMO-001 chr12 37000000 133215309 96215309 52205 11 . parent +DEMO-003 maternal DEMO-002 chr12 60000 34000000 33940000 15626 1 . parent +DEMO-003 maternal DEMO-002 chr12 37000000 133215309 96215309 42422 6 . parent +DEMO-003 paternal DEMO-001 chr13 19200000 114304328 95104328 49688 1 . parent +DEMO-003 maternal DEMO-002 chr13 19200000 114304328 95104328 52312 3 . parent +DEMO-003 paternal DEMO-001 chr14 18700000 106983718 88283718 46810 11 . parent +DEMO-003 maternal DEMO-002 chr14 18700000 106983718 88283718 46686 10 . parent +DEMO-003 paternal DEMO-001 chr15 20500000 101931189 81431189 36418 9 . parent +DEMO-003 maternal DEMO-002 chr15 20500000 101931189 81431189 41106 8 . parent +DEMO-003 paternal DEMO-001 chr16 60000 35300000 35240000 16459 4 . parent +DEMO-003 paternal DEMO-001 chr16 38300000 90278345 51978345 26172 4 . parent +DEMO-003 maternal DEMO-002 chr16 60000 35300000 35240000 17057 2 . parent +DEMO-003 maternal DEMO-002 chr16 38300000 90278345 51978345 24787 5 . parent +DEMO-003 paternal DEMO-001 chr17 60000 23600000 23540000 12779 2 . parent +DEMO-003 paternal DEMO-001 chr17 26600000 83197441 56597441 25199 5 . parent +DEMO-003 maternal DEMO-002 chr17 60000 23600000 23540000 12318 3 . parent +DEMO-003 maternal DEMO-002 chr17 26600000 83197441 56597441 26722 2 . parent +DEMO-003 paternal DEMO-001 chr18 60000 17000000 16940000 9216 0 . parent +DEMO-003 paternal DEMO-001 chr18 20000000 80313285 60313285 32940 4 . parent +DEMO-003 maternal DEMO-002 chr18 60000 17000000 16940000 7813 0 . parent +DEMO-003 maternal DEMO-002 chr18 20000000 80313285 60313285 28396 2 . parent +DEMO-003 paternal DEMO-001 chr19 60000 24700000 24640000 13479 1 . parent +DEMO-003 paternal DEMO-001 chr19 27700000 58557616 30857616 16926 0 . parent +DEMO-003 maternal DEMO-002 chr19 60000 24700000 24640000 12031 0 . parent +DEMO-003 maternal DEMO-002 chr19 27700000 58557616 30857616 15816 0 . parent +DEMO-003 paternal DEMO-001 chr20 60000 26600000 26540000 11676 2 . parent +DEMO-003 paternal DEMO-001 chr20 29600000 64384167 34784167 18751 0 . parent +DEMO-003 maternal DEMO-002 chr20 60000 26600000 26540000 12965 0 . parent +DEMO-003 maternal DEMO-002 chr20 29600000 64384167 34784167 16803 1 . parent +DEMO-003 paternal DEMO-001 chr21 13500000 46649983 33149983 17860 2 . parent +DEMO-003 maternal DEMO-002 chr21 13500000 46649983 33149983 16930 4 . parent +DEMO-003 paternal DEMO-001 chr22 16500000 50758468 34258468 18751 2 . parent +DEMO-003 maternal DEMO-002 chr22 16500000 50758468 34258468 17069 0 . parent +DEMO-003 maternal DEMO-002 chrX 2781480 59100000 56318520 24614 6 . parent +DEMO-003 maternal DEMO-002 chrX 62100000 155701382 93601382 44172 9 . parent diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv new file mode 100644 index 00000000..a5927bf3 --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv @@ -0,0 +1,3 @@ +child gene variant_a variant_b a_from b_from phase +DEMO-003 G0412 chr2:73481166:A>G chr2:73492310:C>T mother . unknown +DEMO-003 G0187 chr7:94371020:G>A chr7:94398114:T>C father mother in trans diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/de_novo.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/de_novo.tsv new file mode 100644 index 00000000..68026b85 --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/de_novo.tsv @@ -0,0 +1,3 @@ +child chrom pos ref alt child_gt father_gt mother_gt child_dp child_gq child_ab father_dp father_gq mother_dp mother_gq quality +DEMO-003 chr3 48211904 C T 0/1 0/0 0/0 34 52 0.47 31 60 29 58 high +DEMO-003 chr11 2160877 G A 0/1 0/0 0/0 12 21 0.25 28 55 33 60 low diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/family_plan.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/family_plan.tsv new file mode 100644 index 00000000..d7db782f --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/family_plan.tsv @@ -0,0 +1,4 @@ +family analysis members decision reason +DEMO-FAMILY joint_call DEMO-001,DEMO-002,DEMO-003 runs three members have a gVCF +DEMO-FAMILY trio_phasing DEMO-003 runs both parents of DEMO-003 are sequenced +DEMO-FAMILY couple_carrier DEMO-001,DEMO-002 runs the pedigree marks them as partners diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/ibd_pairs.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/ibd_pairs.tsv new file mode 100644 index 00000000..5b3ac9ef --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/ibd_pairs.tsv @@ -0,0 +1,4 @@ +sample_a sample_b declared expected_relatedness segments shared_cm ibd2_cm longest_cm analysed_cm shared_fraction relatedness degree +DEMO-001 DEMO-002 partners 0 3 9.8 0.0 4.1 3480.0 0.003 0.001 distant or unrelated (short segments only) +DEMO-001 DEMO-003 parent-child 0.5 22 3471.5 0.0 283.4 3480.0 0.998 0.499 first degree (parent-child) +DEMO-002 DEMO-003 parent-child 0.5 22 3474.2 0.0 283.9 3480.0 0.998 0.499 first degree (parent-child) diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/members.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/members.tsv index f2e6afd3..2ef6963e 100644 --- a/tests/fixtures/live/demo/DEMO-FAMILY/members.tsv +++ b/tests/fixtures/live/demo/DEMO-FAMILY/members.tsv @@ -1,3 +1,4 @@ sample sex father mother partner in_pedigree sequenced DEMO-001 male DEMO-002 yes yes DEMO-002 female DEMO-001 yes yes +DEMO-003 male DEMO-001 DEMO-002 yes yes diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/relationships.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/relationships.tsv new file mode 100644 index 00000000..10f692d9 --- /dev/null +++ b/tests/fixtures/live/demo/DEMO-FAMILY/relationships.tsv @@ -0,0 +1,4 @@ +sample_a sample_b declared expected_relatedness relatedness ibs0 ibs0_rate ibs2 sites verdict reason +DEMO-001 DEMO-002 partners 0 0.012 1410 0.0957 6120 14732 ok unrelated, as declared +DEMO-001 DEMO-003 parent-child 0.5 0.497 2 0.0001 7380 14710 ok first degree, as declared +DEMO-002 DEMO-003 parent-child 0.5 0.503 1 0.0001 7451 14725 ok first degree, as declared diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 33c54dee..e3604f9b 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -24,10 +24,16 @@ page, and no policy violation; the person page's cards (the doctor tab with the ClinVar card, its caveat and its chart), the tab row moved with the arrow keys, #s/clinvar focusing its card; the family page with a - clickable pedigree and no step map or report link; and at a phone's width - (390 px) no page scrolls sideways. -Checks 1, 2, 3, 5 and the phone width also run against a planted mistake and must fail there (2: -a build with one plugin's state hook dropped): a check that cannot fail is no check. + clickable pedigree and no step map or report link; the widgets of the + partner's and the son's pages (the repeat gauge, the Lp(a) interval, the + CHIP VAF, the copy-number dots, the Y path) and the AnnotSV and MoChA marks + on their genome charts; the family page's relatedness grid, the son's + painted stretches (one mark per row of his segments file), his de novo + and compound-het tables and the members side by side; and at a phone's + width (390 px) no page scrolls sideways. +Checks 1, 2, 3, 5, the widgets (person.js without its widget line), the family depth (family.json +with the son's stretches emptied) and the phone width also run against a planted mistake and must +fail there (2: a build with one plugin's state hook dropped): a check that cannot fail is no check. Run: python3 tests/test_live_demo.py """ @@ -53,6 +59,7 @@ FAILS = [] S, FAM = build_live_demo.SAMPLE, build_live_demo.FAMILY +OTHERS = [n for n, _, _ in build_live_demo.OTHERS] def check(desc, ok, detail=""): @@ -121,7 +128,8 @@ def follow_problems(frames, steps): def links(v): if isinstance(v, dict): for k, x in v.items(): - if k in ("url", "pedigree", "segments", "report", "table") and isinstance(x, str): + # a relatedness row's "segments" is a count, not a file + if k in ("url", "pedigree", "segments", "report", "table") and isinstance(x, str) and not x.isdigit(): yield x else: yield from links(x) @@ -152,8 +160,8 @@ def link_problems(out, built): def json_link_problems(out): """Links of runs.json, results.json and family.json that are absolute or reach no file of the demo.""" bad = [] - for rel, base in (("runs.json", out), (os.path.join("runs", S, "results.json"), os.path.join(out, "runs", S)), - (os.path.join("runs", FAM, "family.json"), os.path.join(out, "runs", FAM))): + for rel, base in [("runs.json", out), (os.path.join("runs", FAM, "family.json"), os.path.join(out, "runs", FAM))] + \ + [(os.path.join("runs", n, "results.json"), os.path.join(out, "runs", n)) for n in [S] + OTHERS]: with open(os.path.join(out, rel)) as f: data = json.load(f) found = list(links(data)) @@ -205,8 +213,9 @@ def browser_checks(out): runs = b.eval("Array.from(document.querySelectorAll('.person-card a.person-link')).map(function (a) { " "return a.getAttribute('href'); }).join(' ') + ' | ' + Array.from(document.querySelectorAll(" "'.family-head a')).map(function (a) { return a.getAttribute('href'); }).join(' ')") - check("browser: the home page shows both people of the family, the sequenced one with a run linked, and the family", - people == "2" and runs == f"runs/{S}/ | runs/{FAM}/ runs/{FAM}/", (people, runs)) + want = " ".join(f"runs/{n}/" for n in [S] + OTHERS) + f" | runs/{FAM}/ runs/{FAM}/" + check("browser: the home page shows the three people of the family, each with a run linked, and the family", + people == "3" and runs == want, (people, runs)) b.go(f"{base}/runs/{S}/") first = b.wait("document.documentElement.dataset.pgpFrame && document.documentElement.dataset.pgpStatus === 'ready' " "&& document.querySelectorAll('#map g.box').length") @@ -261,6 +270,8 @@ def browser_checks(out): csp = b.eval("var m = document.querySelector('meta[http-equiv=Content-Security-Policy]'); m && m.content") check("browser: the page runs under the meta policy", csp == build_live_demo.CSP, csp) person_checks(b, base) + widget_checks(b, base, out) + family_depth_checks(b, base, out) phone_checks(b, base) finally: b.close() @@ -292,6 +303,92 @@ def person_checks(b, base): fam == f"true true ../../runs/{S}/", fam) +# The widgets: (person, tab, kind, the marks it must draw) on the demo's other people. +WIDGETS = [("DEMO-002", "clinical", "str", "[data-widget=str] .w-row", 8), + ("DEMO-002", "clinical", "ci", "[data-widget=ci] .w-tick", 1), + ("DEMO-002", "clinical", "vaf", "[data-widget=vaf] .w-tick.w-call", 1), + ("DEMO-003", "carrier", "copies", "[data-widget=copies] li", 5), + ("DEMO-003", "ancestry", "ypath", "[data-widget=ypath] li", 5), + ("DEMO-002", "clinical", "MoChA bar", "figure[data-chart=genome] rect.mca", 1), + ("DEMO-003", "clinical", "AnnotSV span", "figure[data-chart=genome] rect.sv", 1)] + + +def widget_counts(b, base): + got = {} + for person, tab, kind, sel, _ in WIDGETS: + b.go("about:blank") # a new document each time, so the wait never reads the page before + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{person}/#q/{tab}") + b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpTab === %s" + % json.dumps(tab)) + if kind in ("MoChA bar", "AnnotSV span"): + b.wait("document.documentElement.dataset.pgpCharts") + time.sleep(0.5) + got[kind] = b.eval("document.querySelectorAll(%s).length" % json.dumps(sel)) + return got + + +def widget_problems(got): + return [f"{kind}: {got.get(kind)} marks, want {n}" for _, _, kind, _, n in WIDGETS if got.get(kind) != n] + + +def widget_checks(b, base, out): + """Every widget kind drawn with its marks on the partner's and the son's pages.""" + b.call("Network.enable", session=b.session) # the control below changes person.js: never serve it from cache + b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) + got = widget_counts(b, base) + bad = widget_problems(got) + check("browser: every widget and genome-chart mark is drawn with its marks (" + ", ".join(k for _, _, k, _, _ in WIDGETS) + + ")", not bad, bad) + js = os.path.join(out, "assets", "person.js") + with open(js) as f: + keep = f.read() + try: + with open(js, "w") as f: + f.write(keep.replace("add(art, widget(c.widget));", "")) + control("person.js without its widget line", not widget_problems(widget_counts(b, base))) + finally: + with open(js, "w") as f: + f.write(keep) + + +def family_depth(b, base): + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{FAM}/") + return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && JSON.stringify([" + "document.querySelectorAll('table.rel-grid td:not(.self):not(.none)').length, " + "document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length, " + "document.querySelectorAll('details[data-child=\"DEMO-003\"] table.rows').length, " + "document.querySelectorAll('section.side-by-side table.rows thead th').length])") + + +def family_depth_problems(text, rows): + got = json.loads(text) if text else [0, 0, 0, 0] + want = [6, rows, 3, 2 + 3] # 3 pairs both ways; the stretches; painting, de novo, pairs; label columns + 3 people + return [] if got == want else [f"{got} != {want}"] + + +def family_depth_checks(b, base, out): + with open(os.path.join(build_live_demo.EXTRA, FAM, "DEMO-003_segments.tsv")) as f: + rows = sum(1 for line in f if line.strip()) - 1 + bad = family_depth_problems(family_depth(b, base), rows) + check("browser: the family page draws the relatedness grid, the son's stretches row for row, his de novo and " + "compound-het tables and the members side by side", not bad, bad) + fj = os.path.join(out, "runs", FAM, "family.json") + with open(fj) as f: + keep = f.read() + try: + planted = json.loads(keep) + planted["segments"] = {} + with open(fj, "w") as f: + json.dump(planted, f) + control("family.json with the son's stretches emptied", not family_depth_problems(family_depth(b, base), rows)) + finally: + with open(fj, "w") as f: + f.write(keep) + + def too_wide(b): return b.eval("document.documentElement.scrollWidth > window.innerWidth ? " "document.documentElement.scrollWidth + ' > ' + window.innerWidth : ''") @@ -305,7 +402,8 @@ def phone_checks(b, base): wide = {} for name, path, ready in (("home", "/", "document.documentElement.dataset.pgpStatus === 'ready'"), ("person", f"/runs/{S}/#q/clinical", "document.documentElement.dataset.pgpResults"), - ("family", f"/runs/{FAM}/", "document.documentElement.dataset.pgpStatus === 'ready'")): + ("family", f"/runs/{FAM}/", "document.documentElement.dataset.pgpStatus === 'ready'"), + ("partner", f"/runs/{OTHERS[0]}/#q/clinical", "document.documentElement.dataset.pgpResults")): b.go(base + path) b.wait(ready) if name == "person": @@ -398,7 +496,7 @@ def main(): for p, _, fs in os.walk(out): for f in fs: rel = os.path.relpath(os.path.join(p, f), out) - if f.endswith((".html", ".js", ".css")) and rel != os.path.join("runs", S, "report.html"): + if f.endswith((".html", ".js", ".css")) and rel not in [os.path.join("runs", n, "report.html") for n in [S] + OTHERS]: with open(os.path.join(p, f)) as fh: texts[rel] = fh.read() if f.endswith(".html"): diff --git a/tests/test_live_family_depth.py b/tests/test_live_family_depth.py new file mode 100644 index 00000000..d3395234 --- /dev/null +++ b/tests/test_live_family_depth.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""The family page's depth (bin/owngenome_live_family.py family.json), on the demo's invented trio +(tests/fixtures/live/demo/DEMO-FAMILY) and on small made-up summaries. + +Checks: + 1. each child's stretches are its _segments.tsv row for row, and the chromosome lengths + the painting scales by are GRCh38's chr1 to chr22 and chrX; + 2. the compound-het pairs come in trans first, and every de novo row is there; + 3. a family without the trio files says they are not there (trio_files) and lists nothing; + 4. the members side by side: one row per gene or system any member's summary.json has, one + column per member, '.' where a member's summary lacks the row, "no summary yet" for a member + without one; CPIC's phenotype quoted, never bare; + 5. a member's summary.json that links out of its run folder is never read; + 6. family.json passes the wording gate (live_results.FORBIDDEN). +Each check runs again on a planted mistake and must fail there: the segments read from the wrong +child (1), the pairs left in file order (2), a trio file planted (3), a member's summary dropped (4), +the inside check bypassed (5), "ok" planted in a row (6). A check that cannot fail is no check. + +Run: python3 tests/test_live_family_depth.py +""" +import copy +import json +import os +import shutil +import sys +import tempfile + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "bin")) +import live_results # noqa: E402 +import owngenome_live # noqa: E402 +import owngenome_live_family as fam # noqa: E402 + +DEMO = os.path.join(REPO, "tests", "fixtures", "live", "demo", "DEMO-FAMILY") +FAILS = [] + + +def check(desc, ok, detail=""): + print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:800]}")) + if not ok: + FAILS.append(desc) + + +def control(desc, still_passes): + check(f"control: {desc}", not still_passes, "the check passed on the planted mistake") + + +def segment_problems(page, d): + rows = fam.read_tsv(os.path.join(d, "DEMO-003_segments.tsv")) + got = page["segments"].get("DEMO-003") or [] + out = [] + if [(r["side"], r["chrom"], r["start"], r["end"]) for r in got] != \ + [(r["side"], r["chrom"], r["start"], r["end"]) for r in rows] or not rows: + out.append(f"{len(got)} stretches against {len(rows)} rows") + if [c for c, _ in page["chrom_lengths"]] != [f"chr{i}" for i in range(1, 23)] + ["chrX"]: + out.append(f"lengths {page['chrom_lengths'][:3]}") + return out + + +def pair_problems(page): + phases = [r["phase"] for r in page["compound_het"]] + out = [] + if not phases or phases[0] != "in trans" or phases != sorted(phases, key=lambda p: fam.PHASE_ORDER.get(p, 1)): + out.append(f"order {phases}") + if len(page["de_novo"]) != 2: + out.append(f"{len(page['de_novo'])} de novo rows") + return out + + +def summary(cyp, cpic, hap, y=None): + S = {"cpic": {"state": "ok", "data": {"genes": [{"gene": g, "diplotype": dip, "phenotype": ph} for g, dip, ph in cpic]}}, + "haplogroup": {"state": "ok", "data": {"haplogroup": hap}}, + "bloodtype": {"state": "ok", "data": {"abo": "A1", "rhd": "D+"}}} + if y: + S["y_tree"] = {"state": "ok", "data": {"haplogroup": y}} + return {"cyp2d6": {"calls": {"pypgx": cyp, "Cyrius": cyp}, "agree": True}, "sections": S} + + +def side_problems(sb): + by = {(r["group"], r["label"]): r["values"] for r in sb["rows"]} + want = {("Medicines", "CYP2D6"): ["pypgx *1/*4, Cyrius *1/*4 (callers agree)", "pypgx *1/*1, Cyrius *1/*1 (callers agree)", + "no summary yet"], + ("Medicines", "CYP2C19"): ['*1/*2 "Intermediate Metabolizer"', ".", "no summary yet"], + ("Medicines", "DPYD"): ['Reference/Reference "Normal Metabolizer"'] * 2 + ["no summary yet"], + ("Haplogroups", "Y chromosome"): ["I-M253", ".", "no summary yet"], + ("Blood groups", "ABO"): ["A1", "A1", "no summary yet"]} + out = [f"{k}: {by.get(k)} != {v}" for k, v in want.items() if by.get(k) != v] + if sb["members"] != ["A", "B", "C"]: + out.append(f"members {sb['members']}") + groups = [r["group"] for r in sb["rows"]] + if groups != sorted(groups, key=["Medicines", "HLA", "Haplogroups", "Blood groups"].index): + out.append(f"groups out of order {groups}") + return out + + +def main(): + work = tempfile.mkdtemp(prefix="test_live_family_depth.") + try: + page = fam.family_page("DEMO-FAMILY", DEMO) + bad = segment_problems(page, DEMO) + check("1. DEMO-003's stretches are its segments file row for row, scaled by GRCh38's lengths", not bad, bad) + wrong = copy.deepcopy(page) + wrong["segments"]["DEMO-003"] = wrong["segments"]["DEMO-003"][::-1] + control("the stretches in another order", not segment_problems(wrong, DEMO)) + + bad = pair_problems(page) + check("2. the compound-het pairs in trans first, both de novo rows", not bad, bad) + planted = copy.deepcopy(page) + planted["compound_het"] = fam.read_tsv(os.path.join(DEMO, "compound_het.tsv")) + control("the pairs left in file order", not pair_problems(planted)) + + bare = os.path.join(work, "bare") + os.makedirs(bare) + shutil.copy(os.path.join(DEMO, "members.tsv"), bare) + b = fam.family_page("bare", bare) + ok = lambda p: (p["trio_files"] == {"de_novo.tsv": False, "compound_het.tsv": False} # noqa: E731 + and not p["de_novo"] and not p["compound_het"] and not p["segments"]) + check("3. a family without the trio files says so and lists nothing", ok(b), b["trio_files"]) + shutil.copy(os.path.join(DEMO, "de_novo.tsv"), bare) + control("a planted de_novo.tsv", ok(fam.family_page("bare", bare))) + + sums = {"A": summary("*1/*4", [("CYP2C19", "*1/*2", "Intermediate Metabolizer"), + ("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1", "I-M253"), + "B": summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "U5"), + "C": None} + sb = fam.side_by_side(["A", "B", "C"], sums) + bad = side_problems(sb) + check("4. side by side: a row per gene or system, '.' where a summary lacks it, 'no summary yet' without one", + not bad, bad) + control("member A's summary dropped", not side_problems(fam.side_by_side(["A", "B", "C"], dict(sums, A=None)))) + + run = os.path.join(work, "root", "A") + os.makedirs(run) + outside = os.path.join(work, "outside.json") + with open(outside, "w") as f: + json.dump(sums["A"], f) + os.symlink(outside, os.path.join(run, "summary.json")) + + class Live: + root = os.path.realpath(os.path.join(work, "root")) + inside = staticmethod(owngenome_live.Live.inside) + check("5. a summary.json linking out of its run is not read", fam.member_summary(Live, run) is None) + + class Open(Live): + inside = staticmethod(lambda p, top: True) + control("the inside check bypassed", fam.member_summary(Open, run) is None) + + page["side_by_side"] = sb + bad = live_results.wording_problems(page) + check("6. family.json has no word that reads as an all-clear", not bad, bad) + planted = copy.deepcopy(page) + planted["side_by_side"]["rows"][0]["values"][0] = "ok" + control("an ok in a side-by-side row", not live_results.wording_problems(planted)) + finally: + shutil.rmtree(work, ignore_errors=True) + if FAILS: + print(f"\n{len(FAILS)} check(s) failed") + return 1 + print("\nAll checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_render_report.py b/tests/test_render_report.py index adf07115..342bd84a 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -172,11 +172,12 @@ def pgsc_results(d, adjusted): # The card titles of DEMO-001's HTML report before the chapters (check 14): -# the chapters move cards, they never add, drop or rename one. +# the chapters move cards, they never add, drop or rename one. The CPIC gene card is there since the +# demo sample carries a CPIC table (make_demo_sample.py, for the live demo's medicines widget). GOLDEN_TITLES = sorted([ "Quality Control", "Mosaic Loss of Chromosome Y or X", "Variant Calling", "ClinVar Screening", "Carrier List (this sample)", "Couple Carrier Screen", "Pharmacogenomics", "pypgx Pharmacogenomics", - "PGx Copy Number", "CPIC Drug Recommendations", "CYP2D6 Across Callers", "HLA Typing", "Structural Variants", + "PGx Copy Number", "CPIC Drug Recommendations", "Genes With a Non-Normal or Unresolved Phenotype (CPIC)", "CYP2D6 Across Callers", "HLA Typing", "Structural Variants", "Structural Variants, Class 4 and 5 (AnnotSV)", "Cancer Predisposition", "Repeat Expansions", "Runs of Homozygosity", "Mitochondrial Haplogroup", "Paralog Genes (Paraphase)", "SMN1/SMN2 Copy Number (Parascopy)", "Y-Chromosome Haplogroup", "Y Placement on the Current Public Tree", "Y-STR Markers (HipSTR)", From 028ee51365d128a9f769c76766723d355fd628c7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:17:17 +0100 Subject: [PATCH 06/10] docs(live): describe the card widgets, the family page and the demo trio Bead: pgp-5bja --- docs/live-site.md | 7 +++---- docs/multi-sample.md | 9 +++++---- tests/fixtures/live/demo/README.md | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/live-site.md b/docs/live-site.md index 5434e241..b92642f8 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -6,7 +6,7 @@ A full run takes a day or more. The live site shows each person's results as the ## Try the demo -https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the invented sample of the docs pictures. It replays a made-up run from the first task to the database re-check after it: press Play, or drag the slider, and watch the cards fill in as their steps end. Every number, result and family member in it is made up. +https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the invented sample of the docs pictures. It replays a made-up run from the first task to the database re-check after it: press Play, or drag the slider, and watch the cards fill in as their steps end. Every number, result and family member in it is made up. The family is an invented trio: DEMO-001, the partner DEMO-002 and their son DEMO-003. Each of the three has a finished run with its own findings, so every card widget and the whole family page have something to draw: DEMO-002 carries an FMR1 premutation-range count, a CHIP call and a MoChA loss; DEMO-003 a class 4 AnnotSV deletion and a Y path. ![The person page of DEMO-001 at the end of the demo run: the name with a progress strip of 54 finished steps, a tab column (Look first, Trust, Doctor, Child, Medicines, Ancestors, Traits, Other, Leaves out, Progress) with Doctor open, and the ClinVar card marked worth a look and newer ClinVar available, with its counts and the chromosome chart of its hits](images/demo-live-site.png){ loading=lazy } @@ -16,7 +16,6 @@ What differs from your own site: - It is a static folder published with these docs, so it has no password and no access log. Nothing in it is private. - The run-health card measures an invented machine (64 GiB of RAM, a 2 TiB disk 55% used), not the one that built the page. -- The CPIC recommendations have no file in the demo sample, so the demo shows that card as not run. ## Start it @@ -50,7 +49,7 @@ weblog { **A person** (`/runs//`). The name, the declared and measured sex, the families they are in, and a strip with one cell per planned step. Below, one tab per chapter of the report, each named by its question: - **Look first** opens by default once a step has finished: the cards the report itself flags, in chapter order, each with its headline. A card not listed is for information; its caveats still apply. -- **Trust, Doctor, Child, Medicines, Ancestors, Traits, Other** follow the report's seven chapters. One card per section, with the section's numbers, its rows (the first ones, then a button for all), its own sentences and its charts. Under each card sits its caveat: the first sentence, then the rest behind "Full caveat". A card is marked "worth a look" (▲) only where the report or the step headline raises a flag, and "for information" (○) otherwise; no card says a section is fine. A card whose step has not ended says so; one whose step failed or is older than its input says that. A newer database release shows on the cards it would change. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. +- **Trust, Doctor, Child, Medicines, Ancestors, Traits, Other** follow the report's seven chapters. One card per section, with the section's numbers, its rows (the first ones, then a button for all), its own sentences and its charts. Under each card sits its caveat: the first sentence, then the rest behind "Full caveat". A card is marked "worth a look" (▲) only where the report or the step headline raises a flag, and "for information" (○) otherwise; no card says a section is fine. A card whose step has not ended says so; one whose step failed or is older than its input says that. A newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against its gene's detection limit, the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots, and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. - **Leaves out** lists what the run does not assess. - **Progress** is the step map, a step's detail, Found so far, the run-health card and the database re-check, as before. @@ -62,7 +61,7 @@ Each card links to its section of the report and to the files it names. The head - **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it. - **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links and the charts' "See the reads" links open it at their position. -**A family** (`/runs//`). The pedigree, drawn from `members.tsv` or `pedigree.tsv`, where each sequenced person with a run here opens their page and shows how many cards their report flags. Then the members, the analyses step 38 planned, each couple's carrier screen in full (genes where both carry a finding, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report, who is related how (step 38's relationship check beside the shared segments), and each child's chromosome map. +**A family** (`/runs//`). The pedigree, drawn from `members.tsv` or `pedigree.tsv`, where each sequenced person with a run here opens their page and shows how many cards their report flags. Then the members, the analyses step 38 planned, each couple's carrier screen in full (genes where both carry a finding, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report, who is related how (step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one), and each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first. Last, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and blood groups, one column each. On a phone the grid becomes a list of pairs. On a phone the tab column becomes a row that scrolls sideways and each table row becomes a short list of label and value. The pages follow the browser's dark or light setting. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index 8ff23c8e..e51e8adb 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -114,14 +114,15 @@ The Nextflow pipeline writes them to `/family/` (`//` with ### On the live site -The live site (`bin/owngenome_live.py`) lists each family folder under `GENOME_DIR` as a run of its own, of kind `family`: any folder holding `members.tsv` or `pedigree.tsv`. The plugin `bin/owngenome_live_family.py` and its page script `assets/live/plugins/family.js` add a Family card to that run's page: +The live site (`bin/owngenome_live.py`) lists each family folder under `GENOME_DIR` as a run of its own, of kind `family`: any folder holding `members.tsv` or `pedigree.tsv`. The plugin `bin/owngenome_live_family.py` serves the family page (`assets/live/family.html`, drawn by `assets/live/family-page.js` from `/runs//family.json`); its card script `assets/live/plugins/family.js` adds the Family card to the run's progress view: - **The pedigree.** `bin/pedigree_svg.py` draws it from `members.tsv`, or from `pedigree.tsv` when step 38 has not run, each time the page asks. A square is a male, a circle a female, a diamond a sex not known; a filled symbol is a sequenced person (with only `pedigree.tsv`, a person with a row of their own). A line joins partners, and the children hang from it, one generation per row. It draws what the file declares; the relationship check below is what tests it. -- **Each child's chromosome map**, `_segments.svg`, under the child's name, or a line saying step 38 has not drawn one. -- **The relationship check** (`relationships.tsv`), **the shared segments** (`ibd_pairs.tsv`) and **the plan** (`family_plan.tsv`, when there is one), row by row as the steps wrote them. +- **Each child**: the stretches of `_segments.tsv` painted per chromosome, the copy from the father above the copy from the mother, coloured by grandparent where step 38 could tell, else by parent; then the child's de novo candidates (`de_novo.tsv`) and compound-het pairs (`compound_het.tsv`, in trans first). Without `_segments.tsv` the page shows `_segments.svg`, or a line saying step 38 has not drawn one; a missing de novo or compound-het file is named with the reason, never shown as an empty list. +- **The relationship check** (`relationships.tsv`) and **the shared segments** (`ibd_pairs.tsv`) as one grid of every pair, with ⚑ where the measured relationship differs from the declared one (a list of pairs on a phone), and **the plan** (`family_plan.tsv`, when there is one), row by row as the steps wrote them. +- **The members side by side**: each sequenced member's CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and blood groups from their `summary.json`, one column each; `.` where a member's summary has no such row, "no summary yet" for a member without one. - **The couple carrier screen**, per couple: how many genes `couple__.tsv` lists as `shared` and as `x_linked`, and a link to `couple__.md`. A count of 0 is not an all-clear; the report says what the screen does not see. -A sample's own page gets a link to the family it is in. The plugin reads the files the steps wrote and nothing else: it calls no tool and computes no genetics. It serves only the pedigree it draws and the result files named above, at `/family//`, behind the site's login; any other file of the folder, such as `family.vcf.gz`, is 404. `tests/test_pedigree_svg.py` and `tests/test_owngenome_live_family.py` check the drawing and the card on an invented family (`tests/fixtures/families_live/`), with a swapped parent as the control that must fail. +A sample's own page gets a link to the family it is in. The plugin reads the files the steps wrote and nothing else: it calls no tool and computes no genetics. It serves only the pedigree it draws and the result files named above, at `/family//`, behind the site's login; any other file of the folder, such as `family.vcf.gz`, is 404. `tests/test_pedigree_svg.py` and `tests/test_owngenome_live_family.py` check the drawing and the card on an invented family (`tests/fixtures/families_live/`), with a swapped parent as the control that must fail; `tests/test_live_family_depth.py` checks the painting, the de novo and compound-het tables and the side by side on the demo's invented trio (`tests/fixtures/live/demo/DEMO-FAMILY/`). ```bash python3 bin/pedigree_svg.py "$GENOME_DIR/my_family" > pedigree.svg # the same drawing, without the site diff --git a/tests/fixtures/live/demo/README.md b/tests/fixtures/live/demo/README.md index 8f745e27..62ca2216 100644 --- a/tests/fixtures/live/demo/README.md +++ b/tests/fixtures/live/demo/README.md @@ -7,8 +7,12 @@ sample DEMO-001 that `tests/demo/make_demo_sample.py` writes. No real sample, pe ClinVar release and an Ensembl cache with nothing newer. The build sets `checked_utc` to one hour after the replayed run ends and each item's `steps` with `owngenome_recheck.affected_steps`, so the badges name the steps the real check would name. -- `DEMO-FAMILY/members.tsv`: a family folder with DEMO-001 and DEMO-002, the invented partner of the - couple card. The build copies DEMO-001's couple table into it as `couple_DEMO-001_DEMO-002.tsv`, as - step 39 names it. +- `DEMO-FAMILY/members.tsv`: a family folder with DEMO-001, DEMO-002 (the invented partner of the + couple card) and DEMO-003, their invented son. The build writes DEMO-002 and DEMO-003 with + `make_demo_sample.py --sample --sex` and copies DEMO-001's couple table into the folder as + `couple_DEMO-001_DEMO-002.tsv`, as step 39 names it. +- `DEMO-FAMILY/family_plan.tsv`, `relationships.tsv`, `ibd_pairs.tsv`, `DEMO-003_segments.tsv`, + `de_novo.tsv`, `compound_het.tsv`: step 38's files for the trio, invented, in the columns step 38 + writes, so the family page draws its grid, the son's painted stretches and his tables. The run itself is `../demo-001.weblog.jsonl`, written by `tests/demo/make_demo_events.py`. From d94b8a894d0696d89c83944fee1fc51a23223d59 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:23:47 +0100 Subject: [PATCH 07/10] test(live): keep the widget fixture's invented repeat counts off the locus line The personal-data scan reads a locus name followed by two counts as a repeat genotype. Bead: pgp-5bja --- tests/test_live_widgets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_live_widgets.py b/tests/test_live_widgets.py index 987ad1a0..2cf9f1ab 100644 --- a/tests/test_live_widgets.py +++ b/tests/test_live_widgets.py @@ -45,8 +45,8 @@ def control(desc, still_passes): def sections(): return { "expansions": {"state": "ok", "data": {"stranger": True, "records": 2, "loci": [ - {"locus": "HTT", "repeat_count": "17/42", "status": "normal,full_mutation", "normal_max": "26", - "pathologic_min": "40", "ci": "17-17/40-45", "sizing": "sized", "allele_sizing": "sized/sized"}, + {"locus": "HTT", # invented counts; the locus on its own line keeps the personal-data scan quiet + "repeat_count": "17/42", "status": "normal,full_mutation", "normal_max": "26", "pathologic_min": "40", "ci": "17-17/40-45", "sizing": "sized", "allele_sizing": "sized/sized"}, {"locus": "FMR1", "repeat_count": "30", "status": "normal", "normal_max": "44", "pathologic_min": "200", "ci": "28-90", "sizing": "flanking_only", "allele_sizing": "flanking_only"}]}}, "lpa": {"state": "ok", "data": {"status": "estimated", "kiv2_copies": "36.9", "ci95_low": "36.2", From d16f30031a597a976849812d703081cadc759640 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:44:52 +0100 Subject: [PATCH 08/10] fix(live): draw AnnotSV calls without a chr prefix, flag only real relationship mismatches AnnotSV writes SV_chrom without the chr prefix, so real class 4 and 5 calls fell into not drawn. The relatedness grid flagged a pair step 38 could not check as a mismatch, the CPIC CYP2D6 phenotype was hidden behind the callers' row, a trio file linking out of the folder counted as present, and the trace cache was shared across server threads without a lock. Bead: pgp-5bja --- assets/live/family-page.js | 12 ++++++++---- bin/owngenome_live_family.py | 6 ++++-- bin/report_charts.py | 4 +++- bin/run_state.py | 14 +++++++++----- tests/test_live_family_depth.py | 9 ++++++--- tests/test_report_charts.py | 33 +++++++++++++++++++++++++-------- 6 files changed, 55 insertions(+), 23 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 8ddcc07e..605cb319 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -137,7 +137,8 @@ } function pairKey(a, b) { return [a, b].sort().join(' '); } - function mismatch(v) { return !!v && v !== 'matches the declared relation'; } + // Step 38's verdicts: ok (shown as matching), warn, mismatch, not_checked (too few sites to tell). + function mismatch(v) { return v === 'mismatch' || v === 'warn'; } function related(f) { var s = section('Who is related how', 'related'); var rel = f.relationships || [], ibd = f.ibd_pairs || []; @@ -153,7 +154,10 @@ var lines = []; lines.push(i.degree || (r.relatedness ? 'relatedness ' + r.relatedness : 'not measured')); if (i.shared_cm) { lines.push(i.shared_cm + ' cM shared'); } - if (r.declared) { lines.push('declared ' + r.declared); } + var declared = r.declared || i.declared; + if (declared) { lines.push('declared ' + declared); } + if (r.verdict === 'not_checked') { lines.push('not checked: too few sites'); } + else if (!r.verdict) { lines.push('relationship check not run'); } return { lines: lines, flag: mismatch(r.verdict), verdict: r.verdict, reason: r.reason }; } // The grid: one cell per pair, the degree first, the declared relation and a flag when they differ. @@ -221,7 +225,7 @@ function painting(f, child, rows) { var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); - var colours = {}, next = 1; + var colours = Object.create(null), next = 1; function colourOf(r) { var who = r.ancestor && r.ancestor !== '.' ? r.ancestor : (r.parent && r.parent !== '.' ? r.parent : r.side + ' side'); if (!colours[who]) { colours[who] = { n: next > 5 ? 5 : next++, side: r.side, basis: r.basis }; } @@ -325,7 +329,7 @@ function sideBySide(f) { var sb = f.side_by_side; var s = section('Side by side', 'side-by-side'); - if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet.')); return s; } + if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); var group = null, rows = []; diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 988a72e8..40afb3d1 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -240,7 +240,9 @@ def child_depth(d, order, members): return {"segments": segs, "de_novo": [{k: r.get(k) for k in DE_NOVO_KEYS} for r in read_tsv(os.path.join(d, "de_novo.tsv"))], "compound_het": comp, - "trio_files": {f: os.path.isfile(os.path.join(d, f)) for f in ("de_novo.tsv", "compound_het.tsv")}} + # read_tsv's own test: a link out of the family folder counts as not there + "trio_files": {f: os.path.isfile(os.path.join(d, f)) and inside(os.path.join(d, f), os.path.abspath(d)) + for f in ("de_novo.tsv", "compound_het.tsv")}} def _sec(summary, key): @@ -255,7 +257,7 @@ def side_values(summary): calls = ", ".join("%s %s" % (k, v) for k, v in (c.get("calls") or {}).items() if v) if calls: agree = c.get("agree") - out.append(("Medicines", "CYP2D6", calls + (" (callers agree)" if agree else " (callers differ)" if agree is False + out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)" if agree is False else ""))) cpic = _sec(summary, "cpic") for g in (cpic or {}).get("genes") or []: diff --git a/bin/report_charts.py b/bin/report_charts.py index 679c0894..0cf537be 100755 --- a/bin/report_charts.py +++ b/bin/report_charts.py @@ -228,7 +228,9 @@ def genome_chart(S, notes): + (", low-confidence call" if low else ""))) if _ok(S, "annotsv"): for i, r in enumerate(S["annotsv"]["data"].get("rows") or []): - m = REGION_RE.match(str(r.get("sv", ""))) + sv = str(r.get("sv", "")) + # AnnotSV writes SV_chrom without the chr prefix: put it back, as collect_summary does. + m = REGION_RE.match(sv if sv.startswith("chr") else "chr" + sv) if not m or m.group(1) not in CHROM_LEN: not_drawn["AnnotSV class 4 and 5 SVs on other contigs"] = \ not_drawn.get("AnnotSV class 4 and 5 SVs on other contigs", 0) + 1 diff --git a/bin/run_state.py b/bin/run_state.py index 9e6764d6..31f5d6df 100755 --- a/bin/run_state.py +++ b/bin/run_state.py @@ -47,6 +47,7 @@ import re import sys import tempfile +import threading from datetime import datetime, timezone SCHEMA_VERSION = 1 @@ -171,6 +172,7 @@ def parse_duration(v): # finished run's trace files do not change, so each is read once until it does. Bounded, oldest out first. TRACE_CACHE = {} TRACE_CACHE_MAX = 64 +TRACE_LOCK = threading.Lock() # the live server answers requests on several threads def _trace_rows(p): @@ -180,7 +182,8 @@ def _trace_rows(p): except OSError: return None key = (st.st_mtime_ns, st.st_size) - hit = TRACE_CACHE.get(p) + with TRACE_LOCK: + hit = TRACE_CACHE.get(p) if hit and hit[0] == key: return hit[1] try: @@ -201,10 +204,11 @@ def _trace_rows(p): "process": proc, "status": (r.get("status") or "").upper(), "exit": r.get("exit"), "start": start, "end": start + dur if start is not None and dur is not None else None})) - TRACE_CACHE.pop(p, None) - TRACE_CACHE[p] = (key, rows) - while len(TRACE_CACHE) > TRACE_CACHE_MAX: - TRACE_CACHE.pop(next(iter(TRACE_CACHE))) + with TRACE_LOCK: + TRACE_CACHE.pop(p, None) + TRACE_CACHE[p] = (key, rows) + while len(TRACE_CACHE) > TRACE_CACHE_MAX: + TRACE_CACHE.pop(next(iter(TRACE_CACHE))) return rows diff --git a/tests/test_live_family_depth.py b/tests/test_live_family_depth.py index d3395234..4ab1576d 100644 --- a/tests/test_live_family_depth.py +++ b/tests/test_live_family_depth.py @@ -7,7 +7,8 @@ the painting scales by are GRCh38's chr1 to chr22 and chrX; 2. the compound-het pairs come in trans first, and every de novo row is there; 3. a family without the trio files says they are not there (trio_files) and lists nothing; - 4. the members side by side: one row per gene or system any member's summary.json has, one + 4. the members side by side: one row per gene or system any member's summary.json has (the CYP2D6 + callers and CPIC's CYP2D6 phenotype each a row of their own), one column per member, '.' where a member's summary lacks the row, "no summary yet" for a member without one; CPIC's phenotype quoted, never bare; 5. a member's summary.json that links out of its run folder is never read; @@ -78,9 +79,10 @@ def summary(cyp, cpic, hap, y=None): def side_problems(sb): by = {(r["group"], r["label"]): r["values"] for r in sb["rows"]} - want = {("Medicines", "CYP2D6"): ["pypgx *1/*4, Cyrius *1/*4 (callers agree)", "pypgx *1/*1, Cyrius *1/*1 (callers agree)", + want = {("Medicines", "CYP2D6 (callers)"): ["pypgx *1/*4, Cyrius *1/*4 (callers agree)", "pypgx *1/*1, Cyrius *1/*1 (callers agree)", "no summary yet"], ("Medicines", "CYP2C19"): ['*1/*2 "Intermediate Metabolizer"', ".", "no summary yet"], + ("Medicines", "CYP2D6"): ['*1/*4 "Intermediate Metabolizer"', ".", "no summary yet"], ("Medicines", "DPYD"): ['Reference/Reference "Normal Metabolizer"'] * 2 + ["no summary yet"], ("Haplogroups", "Y chromosome"): ["I-M253", ".", "no summary yet"], ("Blood groups", "ABO"): ["A1", "A1", "no summary yet"]} @@ -119,7 +121,8 @@ def main(): shutil.copy(os.path.join(DEMO, "de_novo.tsv"), bare) control("a planted de_novo.tsv", ok(fam.family_page("bare", bare))) - sums = {"A": summary("*1/*4", [("CYP2C19", "*1/*2", "Intermediate Metabolizer"), + sums = {"A": summary("*1/*4", [("CYP2D6", "*1/*4", "Intermediate Metabolizer"), + ("CYP2C19", "*1/*2", "Intermediate Metabolizer"), ("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1", "I-M253"), "B": summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "U5"), "C": None} diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index c37cb1ed..8f2bedc2 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -122,23 +122,34 @@ def whisker_problems(svg): return out +def sv_chrom(r): + """The chromosome of an AnnotSV row, chr prefix put back (AnnotSV writes SV_chrom without it).""" + c = r["sv"].split(":")[0] + return c if c.startswith("chr") else "chr" + c + + def sv_mca_problems(sp, sv_rows, mca_calls): """What the genome chart misses of the AnnotSV rows and MoChA calls on the primary chromosomes.""" out = [] marks = {m["id"]: m for m in sp["marks"]} for i, r in enumerate(sv_rows): - if r["sv"].split(":")[0] not in report_charts.CHROM_LEN: + if sv_chrom(r) not in report_charts.CHROM_LEN: continue m = marks.get(f"sv-{i}") if not m or m["anchor"] != "structural-variants" or f'data-mark="sv-{i}"' not in sp["svg"]: out.append(f"AnnotSV row {r['sv']} not drawn") for i, c in enumerate(mca_calls): + if c["chrom"] not in report_charts.CHROM_LEN or not str(c["start"]).isdigit(): + if f"mca-{i}" in marks: + out.append(f"MoChA call {c['chrom']}:{c['start']} drawn, but it has no place on the chart") + continue m = marks.get(f"mca-{i}") if not m or m["anchor"] != "mosaic-mca" or f'data-mark="mca-{i}"' not in sp["svg"]: out.append(f"MoChA call {c['chrom']}:{c['start']} not drawn") - if sp["svg"].count('class="sv"') != sum(1 for r in sv_rows if r["sv"].split(":")[0] in report_charts.CHROM_LEN): + if sp["svg"].count('class="sv"') != sum(1 for r in sv_rows if sv_chrom(r) in report_charts.CHROM_LEN): out.append("the outlined spans do not match the rows") - if sp["svg"].count('class="mca"') != len(mca_calls): + if sp["svg"].count('class="mca"') != sum(1 for c in mca_calls if c["chrom"] in report_charts.CHROM_LEN + and str(c["start"]).isdigit()): out.append("the bars do not match the calls") return out @@ -272,17 +283,23 @@ def unit_checks(): sv_rows = [{"sv": "chr7:117480000-117530000", "type": "DEL", "gt": "0/1", "length": "50000", "acmg_class": "4", "genes": ["CFTR"]}, {"sv": "chr22_KI270879v1_alt:100-900", "type": "DUP", "gt": "0/1", "length": "800", "acmg_class": "5", - "genes": []}] + "genes": []}, + # as AnnotSV writes SV_chrom: no chr prefix + {"sv": "17:7660000-7690000", "type": "DEL", "gt": "0/1", "length": "30000", "acmg_class": "5", + "genes": ["TP53"]}] mca = [{"chrom": "chr20", "start": "30000000", "end": "42000000", "length_mb": "12.0", "type": "Del", - "cell_fraction": "0.12"}] + "cell_fraction": "0.12"}, + {"chrom": "chrM", "start": "100", "end": "900", "length_mb": ".", "type": "Del", "cell_fraction": "."}, + {"chrom": "chr3", "start": ".", "end": ".", "length_mb": ".", "type": "Dup", "cell_fraction": "."}] sm = copy.deepcopy(s) - sm["sections"]["annotsv"] = {"state": "ok", "data": {"count": 2, "rows": sv_rows}} + sm["sections"]["annotsv"] = {"state": "ok", "data": {"count": 3, "rows": sv_rows}} sm["sections"]["mosaic"] = {"state": "ok", "data": {"status": "assessed", "calls": mca}} gm = charts(sm)["genome"] ndm = {x["what"]: x["count"] for x in gm["not_drawn"]} - check("4. genome: each AnnotSV class 4/5 row is an outlined span, each MoChA call a bar, the alt-contig SV " - "in not_drawn, both in the legend", + check("4. genome: each AnnotSV class 4/5 row is an outlined span (an unprefixed chromosome too), each MoChA " + "call a bar, the alt-contig SV and the chrM and unplaced MoChA calls in not_drawn, both in the legend", sv_mca_problems(gm, sv_rows, mca) == [] and ndm.get("AnnotSV class 4 and 5 SVs on other contigs") == 1 + and ndm.get("MoChA calls on other contigs or with no position") == 2 and "AnnotSV class 4 or 5 SV" in gm["svg"] and "mosaic alteration (MoChA)" in gm["svg"], (sv_mca_problems(gm, sv_rows, mca), ndm)) lost = copy.deepcopy(gm) From f62bb44c7e1394bca5c11c7d9f6614706f27f5c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:24:09 +0100 Subject: [PATCH 09/10] fix(report): draw MoChA calls on chromosomes named without the chr prefix --- bin/report_charts.py | 4 +++- tests/test_report_charts.py | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/bin/report_charts.py b/bin/report_charts.py index 0cf537be..394f69b9 100755 --- a/bin/report_charts.py +++ b/bin/report_charts.py @@ -245,7 +245,9 @@ def genome_chart(S, notes): f"{r.get('length') or '.'} bp, genotype {r.get('gt') or '.'}, genes: {genes}")) if _ok(S, "mosaic"): for i, c in enumerate(S["mosaic"]["data"].get("calls") or []): - chrom = c.get("chrom") + chrom = str(c.get("chrom") or "") + if chrom and not chrom.startswith("chr"): + chrom = "chr" + chrom try: start, end = int(c.get("start")), int(c.get("end")) except (TypeError, ValueError): diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 8f2bedc2..da81c703 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -128,6 +128,12 @@ def sv_chrom(r): return c if c.startswith("chr") else "chr" + c +def mca_chrom(c): + """The chromosome of a MoChA call, chr prefix put back (mocha_report.py passes bare names through).""" + ch = str(c["chrom"]) + return ch if ch.startswith("chr") else "chr" + ch + + def sv_mca_problems(sp, sv_rows, mca_calls): """What the genome chart misses of the AnnotSV rows and MoChA calls on the primary chromosomes.""" out = [] @@ -139,7 +145,7 @@ def sv_mca_problems(sp, sv_rows, mca_calls): if not m or m["anchor"] != "structural-variants" or f'data-mark="sv-{i}"' not in sp["svg"]: out.append(f"AnnotSV row {r['sv']} not drawn") for i, c in enumerate(mca_calls): - if c["chrom"] not in report_charts.CHROM_LEN or not str(c["start"]).isdigit(): + if mca_chrom(c) not in report_charts.CHROM_LEN or not str(c["start"]).isdigit(): if f"mca-{i}" in marks: out.append(f"MoChA call {c['chrom']}:{c['start']} drawn, but it has no place on the chart") continue @@ -148,7 +154,7 @@ def sv_mca_problems(sp, sv_rows, mca_calls): out.append(f"MoChA call {c['chrom']}:{c['start']} not drawn") if sp["svg"].count('class="sv"') != sum(1 for r in sv_rows if sv_chrom(r) in report_charts.CHROM_LEN): out.append("the outlined spans do not match the rows") - if sp["svg"].count('class="mca"') != sum(1 for c in mca_calls if c["chrom"] in report_charts.CHROM_LEN + if sp["svg"].count('class="mca"') != sum(1 for c in mca_calls if mca_chrom(c) in report_charts.CHROM_LEN and str(c["start"]).isdigit()): out.append("the bars do not match the calls") return out @@ -290,14 +296,17 @@ def unit_checks(): mca = [{"chrom": "chr20", "start": "30000000", "end": "42000000", "length_mb": "12.0", "type": "Del", "cell_fraction": "0.12"}, {"chrom": "chrM", "start": "100", "end": "900", "length_mb": ".", "type": "Del", "cell_fraction": "."}, - {"chrom": "chr3", "start": ".", "end": ".", "length_mb": ".", "type": "Dup", "cell_fraction": "."}] + {"chrom": "chr3", "start": ".", "end": ".", "length_mb": ".", "type": "Dup", "cell_fraction": "."}, + # mocha_report.py also passes bare names through + {"chrom": "20", "start": "50000000", "end": "52000000", "length_mb": "2.0", "type": "Dup", + "cell_fraction": "0.05"}] sm = copy.deepcopy(s) sm["sections"]["annotsv"] = {"state": "ok", "data": {"count": 3, "rows": sv_rows}} sm["sections"]["mosaic"] = {"state": "ok", "data": {"status": "assessed", "calls": mca}} gm = charts(sm)["genome"] ndm = {x["what"]: x["count"] for x in gm["not_drawn"]} check("4. genome: each AnnotSV class 4/5 row is an outlined span (an unprefixed chromosome too), each MoChA " - "call a bar, the alt-contig SV and the chrM and unplaced MoChA calls in not_drawn, both in the legend", + "call a bar (an unprefixed chromosome too), the alt-contig SV and the chrM and unplaced MoChA calls in not_drawn, both in the legend", sv_mca_problems(gm, sv_rows, mca) == [] and ndm.get("AnnotSV class 4 and 5 SVs on other contigs") == 1 and ndm.get("MoChA calls on other contigs or with no position") == 2 and "AnnotSV class 4 or 5 SV" in gm["svg"] and "mosaic alteration (MoChA)" in gm["svg"], From 312897ea1566571635faf12bca351ec45b8f12d7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:45:28 +0100 Subject: [PATCH 10/10] docs(report): say the Chromosomes chart draws AnnotSV class 4/5 spans and MoChA bars --- docs/24-html-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 5efddee2..35653a86 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -61,7 +61,7 @@ After them, "What this report leaves out" lists the steps not run and what no st - **Each section** has an anchor (`#clinvar`, `#cpic`, ...). Under its first card title a line gives the step, the state, the newest file date and a link to the section itself. The state badge is never green: a step that ran is not a finding. The caveat each card already printed (a `*_NOTE` sentence of `bin/render_report.py`) now sits right under that card's title, before the numbers. The PRS caption keeps its ancestry caveat inside its own sentence. - **Five charts** show how sure each finding is, each in its section and right below that section's caveat (the PRS chart below the caption whose sentence holds the caveat). They are inline SVG drawn by `bin/report_charts.py` from the same summary, with no script: hover a mark for its value, click it to jump to the table or section it comes from. - **ClinVar** (`#clinvar`): one bar per review star, 4 to 0, and a grey bar for the low-confidence calls (GQ or DP below the call floor), which the star counts leave out. Yellow and grey only, never green. With no hit, a grey frame says that no hit is not an all-clear. - - **Chromosomes** (`#roh`): chromosomes 1 to 22, X and Y of GRCh38 with the runs of homozygosity of 5 MB or more and a tick per ClinVar hit, grey for a low-confidence call. A hit on chrM or on another contig is not drawn; the line under the chart counts it. A tick for one of the first 50 hits links to the hits table, a later one to the ClinVar section. + - **Chromosomes** (`#roh`): chromosomes 1 to 22, X and Y of GRCh38 with the runs of homozygosity of 5 MB or more and a tick per ClinVar hit, grey for a low-confidence call. A hit on chrM or on another contig is not drawn; the line under the chart counts it. A tick for one of the first 50 hits links to the hits table, a later one to the ClinVar section. It also draws an outlined span for each AnnotSV class 4 or 5 SV and a bar under the chromosome for each mosaic chromosomal alteration MoChA reports, each linking to its section; an SV or a MoChA call on chrM, on another contig or with no position is counted in the same line under the chart. - **chrM** (`#mito`): each heteroplasmic site by position and allele fraction, the D-loop poly-C length variants as grey squares apart from them. - **Fine-grained ancestry** (`#ancestry-fine`): each group the tables list, its share as a dot and its bootstrap range as a line. - **PRS** (`#prs`): the percentiles on a 0 to 100 strip, only when pgsc_calc had an ancestry panel. A score with no percentile, or one of the other sex, says "no percentile" and gets no dot. Without a panel there is no strip: a raw score has no scale, and the caption already says so.