From 169677db3ea67b636bd54bf11b0c88b9f41e0726 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 12:48:34 -0400 Subject: [PATCH 001/155] first pass format manual variants for response --- seqr/views/utils/variant_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 19088364e6..46b17e2787 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -14,7 +14,7 @@ from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Dataset, RnaSample, VariantTag, VariantTagType from seqr.utils.gene_utils import get_genes_for_variants -from seqr.utils.xpos_utils import parse_variant_id +from seqr.utils.xpos_utils import parse_variant_id, get_chrom_pos from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ @@ -382,7 +382,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False, genome_version=None): additional_model_fields = [] if response_variants is None: - additional_model_fields = ['dataset_type', 'saved_variant_json'] + additional_model_fields = ['dataset_type', 'main_transcript'] if not genome_version: additional_model_fields.append('family__project__genome_version') response = get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=additional_model_fields) \ @@ -454,9 +454,7 @@ def _get_clickhouse_variant_annotations(variants, genome_version): for variant in variants: dataset_type = variant.pop('datasetType') gv = genome_version or variant.pop('familyProjectGenomeVersion') - variant_json = variant.pop('savedVariantJson') variants_by_id[variant['variantId']] = { - **variant_json, **variants_by_id[variant['variantId']], **variant, 'familyGuids': variant['familyGuids'] + variants_by_id[variant['variantId']].get('familyGuids', []), @@ -464,6 +462,15 @@ def _get_clickhouse_variant_annotations(variants, genome_version): } if variant['key']: variant_keys_by_genome_version_dataset_type[gv][dataset_type].add(variant['key']) + else: + chrom, pos = get_chrom_pos(variant['xpos']) + transcripts = {gene_id: [{'geneId': gene_id}] for gene_id in variant['geneIds']} + if variant['mainTranscript'] and (variant['mainTranscript'].get('geneId') or len(variant['geneIds']) == 1): + main_gene_id = variant['mainTranscript'].get('geneId') or variant['geneIds'][0] + transcripts[main_gene_id] = [variant['mainTranscript']] + variants_by_id[variant['variantId']].update({ + 'chrom': chrom, 'pos': pos, 'genomeVersion': gv, 'transcripts': transcripts, + }) for gv, gv_keys in variant_keys_by_genome_version_dataset_type.items(): for dataset_type, keys in gv_keys.items(): From 0b7cc43add8dc0a79c6a4456fd8dd1431eae79b9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 13:04:02 -0400 Subject: [PATCH 002/155] clean up --- seqr/views/utils/variant_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 46b17e2787..b3ab4b0667 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -465,7 +465,7 @@ def _get_clickhouse_variant_annotations(variants, genome_version): else: chrom, pos = get_chrom_pos(variant['xpos']) transcripts = {gene_id: [{'geneId': gene_id}] for gene_id in variant['geneIds']} - if variant['mainTranscript'] and (variant['mainTranscript'].get('geneId') or len(variant['geneIds']) == 1): + if variant['mainTranscript']: main_gene_id = variant['mainTranscript'].get('geneId') or variant['geneIds'][0] transcripts[main_gene_id] = [variant['mainTranscript']] variants_by_id[variant['variantId']].update({ From 1d82f81e052c431aee3c58bfba9bc82d4efdb4ff Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 14:11:04 -0400 Subject: [PATCH 003/155] clean up setting saved variant json --- seqr/views/apis/individual_api.py | 4 +--- seqr/views/apis/saved_variant_api.py | 28 +++++++++++----------------- seqr/views/utils/variant_utils.py | 1 - 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 75846a1bb1..9ff831be05 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -893,9 +893,7 @@ def _parse_new_aip_saved_variants(new_variant_keys, family_variant_data): if variant_id in variants_by_id: variant.update(variants_by_id[variant_id]) else: - variant.update({'key': None, 'saved_variant_json': {k: v for k, v in variant.items() if k in { - 'chrom', 'pos', 'ref', 'alt', 'variantId', 'xpos', 'genomeVersion', 'genotypes', 'transcripts', 'mainTranscriptId', - }}}) + variant.update({'key': None}) new_variant_data[key] = variant return new_variant_data diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 81a3c0c713..099d210914 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -56,34 +56,28 @@ def create_manual_saved_variant_handler(request, family_guid): tags = variant_json.pop('tags', []) saved_variant_guids = {guid for guid, is_selected in variant_json.pop('variants', {}).items() if is_selected} - genome_version = family.project.genome_version try: xpos = get_xpos(variant_json['chrom'], variant_json['pos']) variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" except (KeyError, ValueError) as e: return create_json_response({'error': str(e)}, status=400) - variant_json.update({ - 'genomeVersion': genome_version, - 'transcripts': {}, - 'variantId': variant_id, - 'xpos': xpos, - }) - gene_id = variant_json.pop('geneId', None) - if gene_id: - variant_json['transcripts'][gene_id] = [] - if variant_json.get('mainTranscriptId'): - variant_json['transcripts'][gene_id].append({ - 'transcriptId': variant_json['mainTranscriptId'], - 'hgvsc': variant_json.pop('hgvsc', None), - 'hgvsp': variant_json.pop('hgvsp', None), - }) + main_transcript = {} + if variant_json.get('mainTranscriptId'): + main_transcript = { + 'transcriptId': variant_json['mainTranscriptId'], + 'hgvsc': variant_json.pop('hgvsc', None), + 'hgvsp': variant_json.pop('hgvsp', None), + } - variant_json['saved_variant_json'] = {**variant_json} variant_json.update({ + 'variantId': variant_id, + 'xpos': xpos, 'key': None, 'dataset_type': Dataset.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Dataset.DATASET_TYPE_VARIANT_CALLS, + 'gene_ids': [gene_id] if gene_id else [], + 'main_transcript': main_transcript, }) model_json = parse_saved_variant_json(variant_json, family.id) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index b3ab4b0667..3b25244e95 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -59,7 +59,6 @@ def parse_saved_variant_json(variant_json, family_id): 'dataset_type': variant_json.get('dataset_type') or variant_dataset_type(variant_json), 'gene_ids': gene_ids, 'main_transcript': main_transcript, - 'saved_variant_json': variant_json.get('saved_variant_json', {}), } From c86a123afe38877984add481083d60900f5a4ca0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 15:12:52 -0400 Subject: [PATCH 004/155] drop saved variant json --- ..._remove_savedvariant_saved_variant_json.py | 93 +++++++++++++++++++ seqr/models.py | 1 - seqr/views/utils/variant_utils.py | 13 ++- 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 seqr/migrations/0089_remove_savedvariant_saved_variant_json.py diff --git a/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py b/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py new file mode 100644 index 0000000000..b2f8d21b6d --- /dev/null +++ b/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py @@ -0,0 +1,93 @@ +# Generated by Django 4.2.24 on 2026-05-04 18:13 + +from django.db import migrations +from pyliftover.liftover import LiftOver + +from seqr.utils.xpos_utils import get_xpos, get_chrom_pos + + +def populate_saved_variant_json_fields(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + saved_variants_q = SavedVariant.objects.using(db_alias) + update_variants = saved_variants_q.filter(key__isnull=True, sv_type__isnull=True) + + # Some edge cases were missed in the initial migration to set the main transcript + to_update = [] + transcript_variants = update_variants.filter( + gene_ids__len__gt=0, main_transcript={}, saved_variant_json__mainTranscriptId__isnull=False, + ) + for variant in transcript_variants: + main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') + if main_transcript_id: + variant.main_transcript = next( + t for ts in variant.saved_variant_json['transcripts'].values() for t in ts + if t['transcriptId'] == main_transcript_id + ) + to_update.append(variant) + if to_update: + updated = saved_variants_q.bulk_update(to_update, ['main_transcript']) + print(f'Updated main transcript for {updated} variants') + + # Some variants failed to lift to 38 when project build was updated, run liftover on these + to_update = [] + lifover_variants = update_variants.filter(saved_variant_json__genomeVersion='37', family__project__genome_version='38') + if lifover_variants.exists(): + VariantNote = apps.get_model('seqr', 'VariantNote') + note_q = VariantNote.objects.using(db_alias) + _liftover_variants(lifover_variants, saved_variants_q, note_q) + + # Some variants are inexplicably saved on 38 despite being in 37 projects, + # use existing liftover to map to the correct project genome version + to_update = [] + liftunder_variants = update_variants.filter(saved_variant_json__genomeVersion='38', family__project__genome_version='37') + for variant in liftunder_variants: + chrom = variant.saved_variant_json['liftedOverChrom'] + pos = variant.saved_variant_json['liftedOverPos'] + variant.xpos = get_xpos(chrom, pos) + variant.variant_id = f'{chrom}-{pos}-{variant.ref}-{variant.alt}' + to_update.append(variant) + if to_update: + updated = saved_variants_q.bulk_update(to_update, ['xpos', 'variant_id']) + print(f'Lifted under {updated} variants') + + +def _liftover_variants(lifover_variants, saved_variants_q, note_q): + to_update = [] + skipped = 0 + liftover = LiftOver('hg19', 'hg38') + for variant in lifover_variants: + chrom, pos = get_chrom_pos(variant.xpos) + lifted_coord = liftover.convert_coordinate(f'chr{chrom}', pos) + + if not lifted_coord: + note = note_q.create(note='Variant tagged on build GRCh37', guid=variant.guid.replace('SV', 'VN')) + note.saved_variants.add(variant) + skipped += 1 + continue + + lifted_pos = lifted_coord[0][1] + variant.xpos = get_xpos(chrom, lifted_pos) + variant.variant_id = f'{chrom}-{lifted_pos}-{variant.ref}-{variant.alt}' + to_update.append(variant) + + updated = saved_variants_q.bulk_update(to_update, ['xpos', 'variant_id']) + print(f'Lifted over {updated} variants') + if skipped: + print(f'Skipped liftover for {skipped} variants, added notes to affected variants') + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0088_rnasample_sequencing_type'), + ] + + operations = [ + migrations.RunPython(populate_saved_variant_json_fields, reverse_code=migrations.RunPython.noop), + migrations.RemoveField( + model_name='savedvariant', + name='saved_variant_json', + ), + ] diff --git a/seqr/models.py b/seqr/models.py index b2f53553cf..23e5943203 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -834,7 +834,6 @@ class SavedVariant(ModelWithGUID): key = models.PositiveBigIntegerField(null=True, blank=True) selected_main_transcript_id = models.CharField(max_length=20, null=True) - saved_variant_json = models.JSONField(default=dict) genotypes = models.JSONField(default=dict) main_transcript = models.JSONField(default=dict, encoder=DjangoJSONEncoderWithSets) gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 3b25244e95..92fa2ee706 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -453,6 +453,7 @@ def _get_clickhouse_variant_annotations(variants, genome_version): for variant in variants: dataset_type = variant.pop('datasetType') gv = genome_version or variant.pop('familyProjectGenomeVersion') + main_transcript = variant.pop('mainTranscript') variants_by_id[variant['variantId']] = { **variants_by_id[variant['variantId']], **variant, @@ -464,11 +465,15 @@ def _get_clickhouse_variant_annotations(variants, genome_version): else: chrom, pos = get_chrom_pos(variant['xpos']) transcripts = {gene_id: [{'geneId': gene_id}] for gene_id in variant['geneIds']} - if variant['mainTranscript']: - main_gene_id = variant['mainTranscript'].get('geneId') or variant['geneIds'][0] - transcripts[main_gene_id] = [variant['mainTranscript']] + if main_transcript: + main_gene_id = main_transcript.get('geneId') or variant['geneIds'][0] + transcripts[main_gene_id] = [main_transcript] variants_by_id[variant['variantId']].update({ - 'chrom': chrom, 'pos': pos, 'genomeVersion': gv, 'transcripts': transcripts, + 'chrom': chrom, + 'pos': pos, + 'genomeVersion': gv, + 'transcripts': transcripts, + 'mainTranscriptId': main_transcript.get('transcriptId'), }) for gv, gv_keys in variant_keys_by_genome_version_dataset_type.items(): From b63feedc1efc99df7b0703468f923f4c6e30bc47 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:02:34 -0400 Subject: [PATCH 005/155] update test fixtures --- seqr/fixtures/1kg_project.json | 142 --------------------------------- 1 file changed, 142 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 33a5913f17..19ad76f818 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1616,31 +1616,6 @@ "gene_ids": ["ENSG00000135953"], "main_transcript": {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, "genotypes": {"I000003_na19679": {"sampleId": "NA19679", "ab": 0.0, "ad": "45,0", "gq": 99.0, "dp": "45", "pl": "0,135,1525", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000002_na19678": {"sampleId": "NA19678", "ab": 0.0, "ad": "42,0", "gq": 99.0, "dp": "43", "pl": "0,126,1479", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000001_na19675": {"sampleId": "NA19675_1", "ab": 0.7021276595744681, "ad": "14,33", "gq": 46.0, "dp": "50", "pl": "46,0,686", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 1}}, - "saved_variant_json": { - "variantId": "21-3343353-GAGA-G", - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "", - "familyGuids": ["F000001_1", "F000002_2"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000258436", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 2.4418633044922146e-05}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.000242306760358614}, "topmed": {"ac": null, "an": null, "af": null}}, - "genomeVersion": "37", - "pos": 3343353, - "predictions": {"eigen": null, "revel": null, "sift": null, "cadd": "14.33", "metasvm": "", "mpc": null, "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "selectedMainTranscriptId": null, - "liftedOverChrom": "", - "transcripts": {"ENSG00000135953": [ - {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, - {"transcriptId": "ENST00000411991", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000411991.1:c.*211_*213delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "558-560"}, - {"transcriptId": "ENST00000421966", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "downstream_gene_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": null}, {"transcriptId": "ENST00000428085", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "", "hgvsp": "ENSP00000413641.1:p.Leu48del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000428085.1:c.141_143delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "141-143"}, - {"transcriptId": "ENST00000437075", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000437075.2:c.*176_*178delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "541-543"}, - {"transcriptId": "ENST00000438943", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000438943.1:c.*211_*213delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "558-560"}]}, - "chrom": "21", - "genotypes": {"I000003_na19679": {"sampleId": "NA19679", "ab": 0.0, "ad": "45,0", "gq": 99.0, "dp": "45", "pl": "0,135,1525", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000002_na19678": {"sampleId": "NA19678", "ab": 0.0, "ad": "42,0", "gq": 99.0, "dp": "43", "pl": "0,126,1479", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000001_na19675": {"sampleId": "NA19675_1", "ab": 0.7021276595744681, "ad": "14,33", "gq": 46.0, "dp": "50", "pl": "46,0,686", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 1}}, - "CAID": null - }, "family": 1 } }, @@ -1668,7 +1643,6 @@ } }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": {}, "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": null, "consequenceTerms": [], "geneId": "ENSG00000135953", "hgvsc": null, "hgvsp": null, "loftee": [null, []], "majorConsequence": null, "transcriptId": "ENST00000371839", "transcriptRank": 1}, "selected_main_transcript_id": "ENST00000371839", "family": 2 @@ -1695,30 +1669,6 @@ "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 6, "dp": 2, "pl": null, "numAlt": 0}, "I000002_na19675": {"sampleId": "NA19675", "ab": 0.125, "ad": null, "gq": 7, "dp": 8, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "38", - "familyGuids": ["F000001_1"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000505820", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 8.142526788913136e-06}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "topmed": {"ac": null, "an": null, "af": null}}, - "pos": 1560662, - "predictions": {"eigen": null, "revel": null, "sift": "damaging", "cadd": "31", "metasvm": "D", "mpc": null, "splice_ai": null, "phastcons_100_vert": null, "mut_taster": "disease_causing", "fathmm": "tolerated", "polyphen": "probably_damaging", "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000197530": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "CELSR1", "majorConsequence": "splice_region_variant", "canonical": "YES", "hgvs": "ENST00000505820.2:c.1067-4G>A", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000505820.2:c.1067-4G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, - "chrom": "1", - "genotypes": { - "I000003_na19679": {"sampleId": "NA19679", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 0}, - "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 6, "dp": 2, "pl": null, "numAlt": 0}, - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.125, "ad": null, "gq": 7, "dp": 8, "pl": null, "numAlt": 1} - } - }, "family": 1 } }, @@ -1743,74 +1693,6 @@ "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 30, "dp": 10, "pl": null, "numAlt": 0}, "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "38", - "familyGuids": ["F000001_1"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000505820", - "populations": { - "callset": { - "ac": 11, - "af": 0.344, - "an": 32, - "hemi": null, - "hom": null - }, - "exac": { - "ac": 28869, - "af": 0.6478622327790974, - "an": 80512, - "hemi": 0, - "hom": 5122 - }, - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - }, - "gnomad_exomes": { - "ac": 67263, - "af": 0.6339977851605758, - "an": 216284, - "hemi": 0, - "hom": 12201 - }, - "gnomad_genomes": { - "ac": 10595, - "af": 0.6165093253511398, - "an": 30852, - "hemi": 0, - "hom": 2311 - }, - "topmed": { - "ac": 45912, - "af": 0.365635, - "an": 125568, - "hemi": null, - "hom": 10332 - } - }, - "genomeVersion": "38", - "pos": 1562437, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000197530": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, - "chrom": "1", - "genotypes": { - "I000003_na19679": {"sampleId": "NA19679", "ab": 0.71428573, "ad": null, "gq": 58, "dp": 7, "pl": null, "numAlt": 1}, - "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 30, "dp": 10, "pl": null, "numAlt": 0}, - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1} - }, - "CAID": null - }, "family": 1 } }, @@ -1835,7 +1717,6 @@ }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": "Gtg/Atg", "consequenceTerms": ["intron_variant"], "geneId": "ENSG00000240361", "hgvsc": "ENST00000262738.3:c.3955G>A", "hgvsp": "ENST00000505820.2:c.1586-17C>G", "loftee": [null, []], "majorConsequence": "intron_variant", "transcriptId": "ENST00000505820", "transcriptRank": 0}, - "saved_variant_json": {}, "family": 12 } }, @@ -1858,28 +1739,6 @@ "genotypes": { "I000017_na20889": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, - "saved_variant_json": { - "liftedOverGenomeVersion": null, - "pos": 249045487, - "end": 249045898, - "xpos": 1249045487, - "predictions": {"strvctvre": 0.374}, - "alt": null, - "numExon": 2, - "genotypeFilters": [], - "ref": null, - "genotypes": { - "I000017_na20889": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} - }, - "liftedOverPos": null, - "liftedOverChrom": null, - "svType": "DEL", - "variantId": "prefix_19107_DEL", - "chrom": "1", - "genomeVersion": "37", - "populations": {"sv_callset": {}}, - "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} - }, "family": 12 } }, @@ -1901,7 +1760,6 @@ "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} }, - "saved_variant_json": {}, "family": 14 } }, From 0ed19123f6d5bbeb2c96ede26296b6136bc3d5e6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:18:20 -0400 Subject: [PATCH 006/155] update test fixtures --- seqr/fixtures/report_variants.json | 82 ------------------------------ 1 file changed, 82 deletions(-) diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 3571f6b9c2..b35a39fce8 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -17,35 +17,6 @@ "genotypes": { "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", - "pos": 1912634, - "end": 1912634, - "xpos": 19001912634, - "ref": "C", - "alt": "T", - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} - }, - "populations": { - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - } - }, - "liftedOverPos": 1912633, - "liftedOverChrom": "19", - "variantId": "19-1912634-C-T", - "chrom": "19", - "transcripts": { - "ENSG00000240361": [] - }, - "CAID": "CA403171634" - }, "family": 2 } }, @@ -68,38 +39,6 @@ "genotypes": { "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", - "pos": 1912633, - "end": 1912633, - "xpos": 19001912633, - "ref": "G", - "alt": "T", - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} - }, - "populations": { - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - } - }, - "liftedOverPos": 1912632, - "liftedOverChrom": "19", - "variantId": "19-1912633-G-T", - "chrom": "19", - "mainTranscriptId": "ENST00000371839", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361"} - ] - }, - "CAID": "CA403171631" - }, "family": 2 } }, @@ -122,25 +61,6 @@ "genotypes": { "I000004_hg00731": {"numAlt": 1} }, - "saved_variant_json": { - "pos": 1912632, - "end": 1912632, - "xpos": 19001912632, - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"numAlt": 1} - }, - "variantId": "19-1912632-G-C", - "chrom": "19", - "mainTranscriptId": "ENST00000371839", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361", - "hgvsc": "c.586_587delinsTT", "hgvsp": "p.Ala196Leu"} - ] - }, - "CAID": null - }, "family": 2 } }, @@ -164,7 +84,6 @@ }, "gene_ids": ["ENSG00000198727"], "main_transcript": {"aminoAcids": "L", "biotype": "protein_coding", "canonical": 1, "codons": "Tta/Cta", "consequenceTerms": ["synonymous_variant"], "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "loftee": [null, []], "majorConsequence": "synonymous_variant", "transcriptId": "ENST00000361789", "transcriptRank": 0}, - "saved_variant_json": {}, "family": 14 } }, { @@ -187,7 +106,6 @@ "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], - "saved_variant_json": {}, "family": 14 } }, From 437ccb652944f66fcb2b3bb53da9e5bdd1fc4fe2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:21:12 -0400 Subject: [PATCH 007/155] update matchmaker saved variant json usage --- matchmaker/matchmaker_utils.py | 5 ++--- matchmaker/views/matchmaker_api.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/matchmaker/matchmaker_utils.py b/matchmaker/matchmaker_utils.py index 4ed0cce103..c47751c494 100644 --- a/matchmaker/matchmaker_utils.py +++ b/matchmaker/matchmaker_utils.py @@ -185,7 +185,7 @@ def _submission_genes_to_external_genomic_features(submission): def _submission_gene_to_external_genomic_features(submission_gene, individual): variant = submission_gene.saved_variant chrom, pos = get_chrom_pos(variant.xpos) - genome_version = variant.saved_variant_json.get('genomeVersion', individual.family.project.genome_version) + genome_version = individual.family.project.genome_version feature = { 'gene': {'id': submission_gene.gene_id}, @@ -204,8 +204,7 @@ def _submission_gene_to_external_genomic_features(submission_gene, individual): _, end = get_chrom_pos(variant.xpos_end) feature['variant']['end'] = end - genotypes = variant.genotypes or variant.saved_variant_json.get('genotypes', {}) - genotype = genotypes.get(individual.guid) + genotype = variant.genotypes.get(individual.guid) if genotype and genotype.get('numAlt', -1) > 0: feature['zygosity'] = genotype['numAlt'] diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 7c7abadc07..aae9cdc1a1 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -59,7 +59,7 @@ def get_individual_mme_matches(request, submission_guid): variants = get_json_for_saved_variants( SavedVariant.objects.filter(guid__in=variant_guids), additional_values={ - 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + 'genomeVersion': Value(project.genome_version), 'selectedMainTranscript': F('main_transcript'), 'xposEnd': F('xpos_end'), }, From ca4c0d6b7674b4f6a4c753b17373036d3177ff61 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:25:49 -0400 Subject: [PATCH 008/155] test clean up --- .../tag_seqr_prioritized_variants_tests.py | 12 ++++++------ seqr/views/apis/individual_api_tests.py | 19 +------------------ 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index f03bd95a59..f0cb6b62c2 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -72,28 +72,28 @@ def test_command(self, mock_datetime, mock_slack, mock_email): }) new_saved_variants = SavedVariant.objects.filter(key__in=[2, 3, 4, 18, 19]).order_by('key').values( - 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', 'main_transcript', 'sv_type', + 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'main_transcript', 'sv_type', ) self.maxDiff = None self.assertListEqual(list(new_saved_variants), [{ 'key': 2, 'variant_id': '1-38724419-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1038724419, 'xpos_end': 1038724419, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000177000', 'ENSG00000277258'], - 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT2['genotypes'], 'sv_type': None, 'main_transcript': VARIANT2['transcripts']['ENSG00000177000'][0], }, {'key': 3, 'variant_id': '1-91502721-G-A', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091502721, 'xpos_end': 1091502721, 'ref': 'G', 'alt': 'A', 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT3['genotypes'], 'sv_type': None, 'main_transcript': VARIANT3['transcripts']['ENSG00000097046'][0], }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, 'xpos_end': 1091511686, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000097046'], - 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT4['genotypes'], 'sv_type': None, 'main_transcript': VARIANT4['transcripts']['ENSG00000097046'][0], }, {'key': 18, 'variant_id': 'suffix_140593_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038717327, 'xpos_end': 17038719636, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023'], - 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DUP', + 'genotypes': GCNV_VARIANT3['genotypes'], 'main_transcript': {}, 'sv_type': 'DUP', }, {'key': 19, 'variant_id': 'suffix_140608_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038721781, 'xpos_end': 17038735703, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023', 'ENSG00000277258', 'ENSG00000277972'], - 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DEL', + 'genotypes': GCNV_VARIANT4['genotypes'], 'main_transcript': {}, 'sv_type': 'DEL', }]) expected_tags = { diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 027b5a8237..41813d526f 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1229,7 +1229,7 @@ def test_import_gregor_metadata(self): saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__name='GREGoR Finding' ).order_by('family_id', 'variant_id').distinct().values( - 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', + 'guid', 'variant_id', 'xpos', 'family__guid', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1237,7 +1237,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': 'F000012_12', - 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, @@ -1255,20 +1254,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-249045487-A-G', 'xpos': 1249045487, 'family__guid': 'F000012_12', - 'saved_variant_json': { - 'alt': 'G', - 'chrom': '1', - 'genomeVersion': '37', - 'genotypes': {created_individual_guid: {'numAlt': 1}}, - 'mainTranscriptId': None, - 'pos': 249045487, - 'ref': 'A', - 'variantId': '1-249045487-A-G', - 'xpos': 1249045487, - 'transcripts': { - 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], - }, - }, 'main_transcript': {}, 'key': None, 'dataset_type': 'SNV_INDEL', @@ -1281,7 +1266,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': new_family_guid, - 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, @@ -1339,7 +1323,6 @@ def test_import_gregor_metadata(self): saved_variant = SavedVariant.objects.get(family__guid=new_family_guid, variant_id='1-248367227-TC-T') self.assertEqual(saved_variant.key, 100) self.assertDictEqual(saved_variant.genotypes, new_family_genotypes) - self.assertDictEqual(saved_variant.saved_variant_json, {}) def test_get_hpo_terms(self): url = reverse(get_hpo_terms, args=['HP:0011458']) From 3d84ad6baa992b1008a5d0a84dd9b17f189900b1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:50:03 -0400 Subject: [PATCH 009/155] test clean up --- .../tests/check_for_new_samples_from_pipeline_tests.py | 6 ------ seqr/views/apis/report_api_tests.py | 7 +------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 7ae407db96..4a99f5d775 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -594,12 +594,6 @@ def test_command(self): self._test_call(error_logs=error_logs) self.assertEqual(Dataset.objects.filter(guid__in=NEW_DATASET_GUIDS + NEW_GCNV_DATASET_GUIDS).count(), 0) - # Update fixture data to allow testing edge cases - svs = SavedVariant.objects.filter(guid__in=['SV0000002_1248367227_r0390_100', 'SV0000006_1248367227_r0003_tes', 'SV0000007_prefix_19107_DEL_r00']) - for sv in svs: - sv.saved_variant_json['genomeVersion'] = '38' - sv.save() - # Test success self.mock_send_slack.reset_mock() self.mock_email.reset_mock() diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 60d214a668..d9dc603317 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -4,7 +4,7 @@ import responses from settings import AIRTABLE_URL -from seqr.models import Project, SavedVariant, RnaSample +from seqr.models import Project, RnaSample from seqr.views.apis.report_api import seqr_stats, anvil_export, gregor_export, family_metadata, variant_metadata from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest @@ -963,11 +963,6 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo project.consent_code = 'H' project.save() - # For SV variant, test reports in gene associated with OMIM condition even if not annotated - variant = SavedVariant.objects.get(id=7) - variant.saved_variant_json['transcripts'] = {'ENSG00000135953': []} - variant.save() - responses.calls.reset() responses.add(responses.GET, 'https://monarchinitiative.org/v3/api/entity/MONDO:0008788', status=200, json={ 'id': 'MONDO:0008788', From fa0bf2c731192ba77ad72c49d1a9ad5cedea996f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 16:58:53 -0400 Subject: [PATCH 010/155] fix saved variant tests --- seqr/views/apis/saved_variant_api_tests.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index a9bb075618..7663308e60 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -177,7 +177,10 @@ def test_saved_variant_data(self): variants = response_json['variantsById'] self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) variant = variants['21-3343353-GAGA-G'] - variant_fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'mainTranscriptId'} + variant_fields = { + *SAVED_VARIANT_FIELDS, 'mainTranscriptId', 'genomeVersion', 'tagGuids', 'functionalDataGuids', 'noteGuids', + 'genotypes', 'transcripts', 'acmgClassification', + } self.assertSetEqual(set(variant.keys()), variant_fields) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) @@ -378,10 +381,10 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals', 'transcriptsById'}) self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), fields) self.assertSetEqual(set(response_json['variantsById']['1-248367227-TC-T'].keys()), { - *variant_fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', + *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): @@ -436,7 +439,7 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None, has_saved_variant_json=False): + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) @@ -444,7 +447,6 @@ def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, da self.assertEqual(dataset_type, saved_variant.dataset_type) self.assertEqual(sv_type, saved_variant.sv_type) self.assertDictEqual(main_transcript or {}, saved_variant.main_transcript) - self.assertDictEqual(variant_json if has_saved_variant_json else {}, saved_variant.saved_variant_json) def test_create_saved_sv_variant(self): # SVs are only supported on build 38 @@ -615,10 +617,9 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, **{k: v for k, v in manual_variant_request_body.items() if k in {'alt', 'ref', 'chrom', 'pos', 'genotypes', 'mainTranscriptId'}}, } - self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True, main_transcript={ + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], main_transcript={ 'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627', }) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) base_variant_json = { 'variantGuid': variant_guid, @@ -671,7 +672,7 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000240361': []}, **{k: v for k, v in manual_variant_request_body.items() if k in {'chrom', 'pos', 'end', 'svName', 'svType', 'genotypes'}}, } - self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL', has_saved_variant_json=True) + self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL') sv_variant_json = { **{k: v for k, v in sv_variant_json.items() if k in SAVED_VARIANT_FIELDS}, From 98fe67118d5e7663b7bd11287382f40db928445a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 17:10:56 -0400 Subject: [PATCH 011/155] fix test --- clickhouse_search/search_tests.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 126235428b..11aa16e0ff 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -38,7 +38,7 @@ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual +from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -2231,10 +2231,6 @@ def test_gene_variant_lookup(self): ) def test_search_context(self): - sv = SavedVariant.objects.get(guid='SV0000001_2103343353_r0390_100') - sv.saved_variant_json['genomeVersion'] = '38' - sv.save() - expected_results = [ VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, From 6a03abb944cb30dd47c6d7b5b95ea0547c23d372 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 4 May 2026 17:27:15 -0400 Subject: [PATCH 012/155] update set key command --- .../commands/set_saved_variant_key.py | 26 ++++------ .../tests/set_saved_variant_key_tests.py | 51 ++++++++----------- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 0928b6d728..6b5eedabcd 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -19,7 +19,6 @@ class Command(BaseCommand): def handle(self, *args, **options): variant_ids = SavedVariant.objects.filter( key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh38, - saved_variant_json__populations__isnull=False, # Omit manual variants ).values_list('variant_id', flat=True).distinct() ids_by_dataset_type = { Dataset.DATASET_TYPE_VARIANT_CALLS: [], Dataset.DATASET_TYPE_MITO_CALLS: [], Dataset.DATASET_TYPE_SV_CALLS: [], @@ -59,11 +58,6 @@ def handle(self, *args, **options): if no_keys_37: self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37) - num_updated = SavedVariant.objects.filter(key__isnull=False).exclude(saved_variant_json={}).update( - saved_variant_json={}, - ) - logger.info(f'Cleared saved json for {num_updated} variants with keys') - logger.info('Done') @staticmethod @@ -98,32 +92,32 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ return no_key @classmethod - def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GENOME_VERSION_GRCh38): + def _query_missing_variants(cls, variant_ids, variant_fields=None, genome_version=GENOME_VERSION_GRCh38): missing_variants = SavedVariant.objects.filter( variant_id__in=variant_ids, family__project__genome_version=genome_version, ) num_missing = missing_variants.count() missing_with_data_qs = missing_variants.filter(family__individual__active_datasets__isnull=False).distinct() missing_with_search_data = missing_with_data_qs.values( - 'variant_id', *variant_fields, + 'variant_id', *(variant_fields or []), ).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id') return missing_with_search_data, num_missing @classmethod def _resolve_missing_variants(cls, variant_ids, genome_version): missing_with_search_data, num_missing = cls._query_missing_variants( - variant_ids, ['saved_variant_json__populations__seqr__ac'], genome_version, + variant_ids, genome_version=genome_version, ) num_data= len(missing_with_search_data) - in_backend = [ - f"{var['variant_id']} - {'; '.join(var['family_ids'])}" - for var in missing_with_search_data if var['saved_variant_json__populations__seqr__ac'] - ] + logger.info( - f'{num_missing} variants have no key, {num_missing - num_data} of which have no search data, {num_data - len(in_backend)} of which are absent from the hail backend.' + f'{num_missing} variants have no key, {num_missing - num_data} of which have no search data.' ) - if in_backend: - logger.info(f'{len(in_backend)} remaining variants: {", ".join(in_backend)}') + if missing_with_search_data: + summary = [ + f"{var['variant_id']} - {'; '.join(var['family_ids'])}" for var in missing_with_search_data + ] + logger.info(f'{len(missing_with_search_data)} remaining variants: {", ".join(summary)}') @classmethod def _resolve_reloaded_svs(cls, variant_ids): diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 9b3d07f479..032364f550 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -22,13 +22,6 @@ def setUpTestData(cls): dataset.dataset_type = 'SV' dataset.save() dataset.active_individuals.set({17}) - for sv in SavedVariant.objects.filter(key__isnull=False): - sv.saved_variant_json = { - 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, - } - if sv.guid == 'SV0000009_25000014783_r0004_no': - sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} - sv.save() SavedVariant.objects.update(key=None) SavedVariant.objects.filter(guid='SV0000007_prefix_19107_DEL_r00').update(variant_id='prefix_19107_DEL') @@ -62,25 +55,25 @@ def test_command(self, mock_subprocess): ('Updated batch of 1', None), ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), ('No key found for 6 variants', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 6 variants with keys', None), + ('6 variants have no key, 0 of which have no search data.', None), + ('6 remaining variants: 1-1562437-G-CA - 1, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) - saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')) + saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id')) expected_saved_variants = [ - {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'saved_variant_json': {}}, - {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'saved_variant_json': {}}, - {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'saved_variant_json': {}}, - {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G'}, + {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL'}, + {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C'}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746'}, + {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T'}, + {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T'}, + {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C'}, + {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A'}, + {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA'}, ] self.assertListEqual(saved_variants, expected_saved_variants) @@ -90,12 +83,12 @@ def test_command(self, mock_subprocess): self.assert_json_logs(user=None, expected=[ ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), + ('6 variants have no key, 0 of which have no search data.', None), + ('6 remaining variants: 1-1562437-G-CA - 1, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) self.assertListEqual( - list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')), + list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id')), expected_saved_variants, ) @@ -115,8 +108,8 @@ def test_command(self, mock_subprocess): ('Found 0 keys', None), ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), ('Found 0 keys', None), - ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), - ('1 remaining variants: M-14783-T-C - fam14', None), + ('3 variants have no key, 1 of which have no search data.', None), + ('2 remaining variants: 1-248367227-TC-T - 12; fam14, M-14783-T-C - fam14', None), ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), ('Found 0 keys', None), ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), @@ -126,8 +119,8 @@ def test_command(self, mock_subprocess): ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), - ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), + ('7 variants have no key, 0 of which have no search data.', None), + ('7 remaining variants: 1-1562437-G-CA - 1, 1-248367227-TC-T - 2, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) From e81dfd71dc66a080c300e4713e04706f685798f4 Mon Sep 17 00:00:00 2001 From: hanars Date: Tue, 5 May 2026 11:56:09 -0400 Subject: [PATCH 013/155] correctly return end for manual SVs --- seqr/views/utils/variant_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 92fa2ee706..6e055f7b7e 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -381,7 +381,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False, genome_version=None): additional_model_fields = [] if response_variants is None: - additional_model_fields = ['dataset_type', 'main_transcript'] + additional_model_fields = ['dataset_type', 'main_transcript', 'xpos_end'] if not genome_version: additional_model_fields.append('family__project__genome_version') response = get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=additional_model_fields) \ @@ -454,6 +454,7 @@ def _get_clickhouse_variant_annotations(variants, genome_version): dataset_type = variant.pop('datasetType') gv = genome_version or variant.pop('familyProjectGenomeVersion') main_transcript = variant.pop('mainTranscript') + xpos_end = variant.pop('xposEnd') variants_by_id[variant['variantId']] = { **variants_by_id[variant['variantId']], **variant, @@ -475,6 +476,11 @@ def _get_clickhouse_variant_annotations(variants, genome_version): 'transcripts': transcripts, 'mainTranscriptId': main_transcript.get('transcriptId'), }) + if xpos_end: + end_chrom, end = get_chrom_pos(xpos_end) + variants_by_id[variant['variantId']]['end'] = end + if end_chrom != chrom: + variants_by_id[variant['variantId']]['endChrom'] = end_chrom for gv, gv_keys in variant_keys_by_genome_version_dataset_type.items(): for dataset_type, keys in gv_keys.items(): From 17dd63c74016e2b82bec413a169943ef4b33d90e Mon Sep 17 00:00:00 2001 From: hanars Date: Tue, 5 May 2026 13:37:31 -0400 Subject: [PATCH 014/155] fix manual sv response --- seqr/views/utils/variant_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 6e055f7b7e..55cf0d932d 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -476,7 +476,7 @@ def _get_clickhouse_variant_annotations(variants, genome_version): 'transcripts': transcripts, 'mainTranscriptId': main_transcript.get('transcriptId'), }) - if xpos_end: + if variant['svType']: end_chrom, end = get_chrom_pos(xpos_end) variants_by_id[variant['variantId']]['end'] = end if end_chrom != chrom: From ae10fb16212d0368f53753ee7316994d2538ec02 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 13:54:31 -0400 Subject: [PATCH 015/155] genome version difff cleanup --- ui/shared/components/panel/family/FamilyReads.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/shared/components/panel/family/FamilyReads.jsx b/ui/shared/components/panel/family/FamilyReads.jsx index bba09bd91f..70d6a41a55 100644 --- a/ui/shared/components/panel/family/FamilyReads.jsx +++ b/ui/shared/components/panel/family/FamilyReads.jsx @@ -196,11 +196,11 @@ const applyUserTrackSettings = (tracks, options) => tracks.map(track => ({ } : {}, })) -const getVariantLocus = (variant, project) => { +const getVariantLocus = (variant) => { const size = variant.end && variant.end - variant.pos return getLocus( variant.chrom, - (variant.genomeVersion !== project.genomeVersion && variant.liftedOverPos) ? variant.liftedOverPos : variant.pos, + variant.pos, size ? Math.max(Math.round(size / 2), MIN_LOCUS_RANGE_SIZE) : MIN_LOCUS_RANGE_SIZE, size, ) @@ -309,7 +309,7 @@ class FamilyReads extends React.PureComponent { this.setState({ openFamily: familyGuid, sampleTypes, - locus: variant && getVariantLocus(variant, this.getProjectForFamily(familyGuid)), + locus: variant && getVariantLocus(variant), }) } @@ -416,7 +416,7 @@ class FamilyReads extends React.PureComponent { const project = openFamily && this.getProjectForFamily(openFamily) const geneLocus = project && variant && getGeneLocus(variant, genesById, project) const locusOptions = [ - { text: 'Variant', value: geneLocus && getVariantLocus(variant, project) }, + { text: 'Variant', value: geneLocus && getVariantLocus(variant) }, { text: 'Gene', value: geneLocus }, { text: 'Splice Outlier', From bac2004942c55f7153a52e8d3d752bfec6cdf71b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 14:11:02 -0400 Subject: [PATCH 016/155] loftee cleanup --- clickhouse_search/test_utils.py | 2 +- .../components/panel/variants/Annotations.jsx | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index b43764cd73..bd3c1dd1eb 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -154,7 +154,7 @@ }, 'transcripts': { 'ENSG00000177000': [ - {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, + {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'git config --global user.name': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, diff --git a/ui/shared/components/panel/variants/Annotations.jsx b/ui/shared/components/panel/variants/Annotations.jsx index fbb99e52fd..c41f434f0c 100644 --- a/ui/shared/components/panel/variants/Annotations.jsx +++ b/ui/shared/components/panel/variants/Annotations.jsx @@ -449,11 +449,9 @@ const svSizeDisplay = (size) => { return `${(size / 1000000).toFixed(2) / 1}Mb` } -const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) => { - const isNagnag = isLofNagnag || lofFlags === 'NAGNAG_SITE' - const filters = lofFilters || (lof === 'LC' && lofFilter && lofFilter.split(/&|,/g)) - return (filters || isNagnag) ? [ - ...(filters ? [...new Set(filters)] : []).map((lofFilterKey) => { +const getLofDetails = ({ isLofNagnag, lofFilters }) => ( + (lofFilters || isLofNagnag) ? [ + ...(lofFilters ? [...new Set(lofFilters)] : []).map((lofFilterKey) => { const filter = LOF_FILTER_MAP[lofFilterKey] || { message: lofFilterKey } return (
@@ -463,7 +461,7 @@ const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) =>
) }), - isNagnag ? ( + isLofNagnag ? (
LOFTEE: NAGNAG site
@@ -471,7 +469,7 @@ const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) =>
) : null, ] : null -} +) // Adapted from https://github.com/ImperialCardioGenetics/UTRannotator/blob/master/README.md#the-detailed-annotation-for-each-consequence const UTR_ANNOTATOR_DESCRIPTIONS = { @@ -533,7 +531,7 @@ const Annotations = React.memo(({ variant, mainGeneId, showMainGene, transcripts endChrom, CAID, } = variant const mainTranscript = getVariantMainTranscript(variant) - const lofDetails = getLofDetails(mainTranscript.loftee || mainTranscript) + const lofDetails = getLofDetails(mainTranscript.loftee || {}) const transcriptPopupProps = mainTranscript.transcriptId && { content: , From 8abbcf59422d6bce21b16aa63dd69d3a8be0d85a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 14:25:28 -0400 Subject: [PATCH 017/155] remiove callset pop support --- .../components/panel/variants/Frequencies.jsx | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/ui/shared/components/panel/variants/Frequencies.jsx b/ui/shared/components/panel/variants/Frequencies.jsx index ad508e295f..1d17c1117a 100644 --- a/ui/shared/components/panel/variants/Frequencies.jsx +++ b/ui/shared/components/panel/variants/Frequencies.jsx @@ -175,12 +175,11 @@ FreqSummary.propTypes = { const getGenePath = ({ variant }) => `gene/${getVariantMainGeneId(variant)}` -const gnomadLink = ({ fieldTitle, esVersion, variant, ...props }) => { - const isEs = !(variant || {}).populations?.seqr +const gnomadLink = ({ fieldTitle, variant, ...props }) => { const [prefix, detail] = fieldTitle.split(' ') return ( - +   {detail} @@ -191,9 +190,8 @@ gnomadLink.propTypes = { fieldTitle: PropTypes.string, } -const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN, variant }) => { - const isEs = !(variant || {}).populations?.seqr - return (!isEs && ac && an < (maxAN / 2)) ? ( +const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN }) => { + return (ac && an < (maxAN / 2)) ? ( } content={`This variant is covered in fewer than 50% of individuals in ${fieldTitle}. This may indicate a low-quality site.`} @@ -242,15 +240,11 @@ GlobalAcSampleTypeSummary.propTypes = { const HOM_SECTION = 'Homoplasmy' const HET_SECTION = 'Heteroplasmy' -const SV_CALLSET_POP = { field: 'sv_callset', fieldTitle: 'This Callset', acDisplay: 'AC', helpMessage: SV_CALLSET_CRITERIA_MESSAGE } -const CALLSET_POP = { field: 'callset', fieldTitle: 'This Callset', acDisplay: 'AC' } -const SEQR_POP = { ...CALLSET_POP, field: 'seqr', fieldTitle: 'seqr' } +const SEQR_POP = { field: 'seqr', fieldTitle: 'seqr', acDisplay: 'AC' } const POPULATIONS = [ - SV_CALLSET_POP, - { ...SV_CALLSET_POP, field: 'sv_seqr', fieldTitle: 'seqr' }, + { field: 'sv_seqr', fieldTitle: 'seqr', acDisplay: 'AC', helpMessage: SV_CALLSET_CRITERIA_MESSAGE }, { ...SEQR_POP, field: 'sv_seqr_affected', fieldTitle: 'seqr affected' }, - CALLSET_POP, SEQR_POP, { ...SEQR_POP, field: 'seqr_affected', fieldTitle: 'seqr affected' }, { @@ -259,7 +253,6 @@ const POPULATIONS = [ titleContainer: gnomadLink, warningContainer: gnomadAnWarning, maxAN: 730947 * 2, // From https://gnomad.broadinstitute.org/stats - esVersion: 'v2', conditionalQueryParams: populations => (populations.seqr ? GNOMAD_URL_INFO.queryParams : { [GENOME_VERSION_37]: 'dataset=gnomad_r2_1' }), ...GNOMAD_URL_INFO, }, @@ -269,7 +262,6 @@ const POPULATIONS = [ titleContainer: gnomadLink, warningContainer: gnomadAnWarning, maxAN: 76215 * 2, // From https://gnomad.broadinstitute.org/stats - esVersion: 'v3', conditionalQueryParams: populations => (populations.seqr ? GNOMAD_URL_INFO.queryParams : { [GENOME_VERSION_38]: 'dataset=gnomad_r3' }), precision: 3, ...GNOMAD_URL_INFO, @@ -293,10 +285,8 @@ const POPULATIONS = [ }, ] -const CALLSET_HET_POP = { - field: 'callset_heteroplasmy', - fieldTitle: 'This Callset', - acDisplay: 'AC', +const SEQR_HET_POP = { + ...SEQR_POP, titleContainer: sectionTitle, section: HET_SECTION, } @@ -308,20 +298,14 @@ const SEQR_HOM_POP = { } const MITO_POPULATIONS = [ - { - ...CALLSET_POP, - titleContainer: sectionTitle, - section: HOM_SECTION, - }, { ...SEQR_POP, titleContainer: sectionTitle, section: HOM_SECTION, }, { ...SEQR_HOM_POP, field: 'seqr_affected', fieldTitle: 'seqr affected' }, - CALLSET_HET_POP, - { ...CALLSET_HET_POP, field: 'seqr_heteroplasmy', fieldTitle: 'seqr' }, - { ...CALLSET_HET_POP, field: 'seqr_heteroplasmy_affected', fieldTitle: 'seqr affected' }, + { ...SEQR_HET_POP, field: 'seqr_heteroplasmy', fieldTitle: 'seqr' }, + { ...SEQR_HET_POP, field: 'seqr_heteroplasmy_affected', fieldTitle: 'seqr affected' }, { field: 'gnomad_mito', fieldTitle: 'gnomAD mito', @@ -388,8 +372,7 @@ const getValueDisplay = (pop, valueField, precision) => (valueField === 'ac' ? const Frequencies = React.memo(({ variant, totalSampleCounts }) => { const { populations = {}, svType } = variant - const callsetHetPop = populations.callset_heteroplasmy || populations.seqr_heteroplasmy - const isMito = callsetHetPop && callsetHetPop.ac !== null && callsetHetPop.ac !== undefined + const isMito = !!populations.seqr_heteroplasmy const popConfigs = isMito ? MITO_POPULATIONS : POPULATIONS let datasetType = isMito ? DATASET_TYPE_MITO_CALLS : DATASET_TYPE_SNV_INDEL_CALLS datasetType = svType ? DATASET_TYPE_SV_CALLS : datasetType From 9c9ff44e48e1aa5ffa99a5ab56a375c7c5b7e7aa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 15:06:32 -0400 Subject: [PATCH 018/155] pathogenicity clean up --- .../Report/components/VariantMetadata.jsx | 4 +- .../panel/variants/Pathogenicity.jsx | 43 ++++--------- .../components/panel/variants/Variants.jsx | 13 ++-- ui/shared/utils/constants.js | 61 +++++++------------ 4 files changed, 40 insertions(+), 81 deletions(-) diff --git a/ui/pages/Report/components/VariantMetadata.jsx b/ui/pages/Report/components/VariantMetadata.jsx index b077b70c36..a06596f36f 100644 --- a/ui/pages/Report/components/VariantMetadata.jsx +++ b/ui/pages/Report/components/VariantMetadata.jsx @@ -1,7 +1,7 @@ import React from 'react' import LoadReportTable from 'shared/components/table/LoadReportTable' -import { clinvarSignificance, VARIANT_METADATA_COLUMNS } from 'shared/utils/constants' +import { VARIANT_METADATA_COLUMNS } from 'shared/utils/constants' const VIEW_ALL_PAGES = [ { name: 'GREGoR', downloadName: 'GREGoR', path: 'gregor' }, @@ -14,7 +14,7 @@ const COLUMNS = [ { name: 'variant_type' }, { name: 'allele_balance_or_heteroplasmy_percentage' }, { name: 'Clinvar allele ID', format: ({ clinvar }) => clinvar?.alleleId }, - { name: 'ClinVar Clinical Significance', format: ({ clinvar }) => clinvarSignificance(clinvar).pathogenicity }, + { name: 'ClinVar Clinical Significance', format: ({ clinvar }) => clinvar?.pathogenicity }, { name: 'ClinVar gold star', format: ({ clinvar }) => clinvar?.goldStars }, { name: 'known_condition_name' }, { name: 'condition_id' }, diff --git a/ui/shared/components/panel/variants/Pathogenicity.jsx b/ui/shared/components/panel/variants/Pathogenicity.jsx index f84517a640..6489cf26d4 100644 --- a/ui/shared/components/panel/variants/Pathogenicity.jsx +++ b/ui/shared/components/panel/variants/Pathogenicity.jsx @@ -3,10 +3,10 @@ import PropTypes from 'prop-types' import { connect } from 'react-redux' import styled from 'styled-components' import { Label, Icon, Popup, List, ListItem } from 'semantic-ui-react' -import { HorizontalSpacer, VerticalSpacer } from 'shared/components/Spacers' +import { HorizontalSpacer } from 'shared/components/Spacers' import { getUser, getFamiliesByGuid, getProjectsByGuid } from 'redux/selectors' -import { clinvarSignificance, clinvarColor, getPermissionedHgmdClass } from '../../../utils/constants' +import { clinvarColor, getPermissionedHgmdClass } from '../../../utils/constants' import { snakecaseToTitlecase } from '../../../utils/stringUtils' const StarsContainer = styled.span` @@ -21,7 +21,7 @@ const StarIcon = styled(Icon).attrs({ name: 'star' })` const HGMD_CLASS_NAMES = { DM: 'Disease Causing (DM)', 'DM?': 'Disease Causing? (DM?)', - FPV: 'Frameshift or truncating variant (FTV)', + R: 'Removed', FP: 'In vitro/laboratory or in vivo functional polymorphism (FP)', DFP: 'Disease-associated polymorphism with additional supporting functional evidence (DFP)', DP: 'Disease-associated polymorphism (DP)', @@ -69,12 +69,6 @@ PathogenicityLink.propTypes = { popup: PropTypes.object, } -const clinvarUrl = (clinvar) => { - const baseUrl = 'http://www.ncbi.nlm.nih.gov/clinvar' - const variantPath = clinvar.alleleId ? `?term=${clinvar.alleleId}[alleleid]` : `/variation/${clinvar.variationId}` - return baseUrl + variantPath -} - const clinvarLabel = (pathogenicity, assertions, conflictingPathogenicities) => { let label = snakecaseToTitlecase(pathogenicity) if (conflictingPathogenicities && conflictingPathogenicities.length) { @@ -89,11 +83,8 @@ const clinvarLabel = (pathogenicity, assertions, conflictingPathogenicities) => return label } -const clinvarPopup = (clinvar) => { - const lastUpdated = ( -
{clinvar.version && `Last Updated: ${new Date(clinvar.version).toLocaleDateString()}`}
- ) - const conditions = clinvar.conditions && ( +const clinvarPopup = (clinvar) => ( + clinvar.conditions ? (
Conditions: @@ -102,29 +93,17 @@ const clinvarPopup = (clinvar) => { ))}
- ) - return ( -
- {lastUpdated} - {conditions && ( -
- - {conditions} -
- )} -
- ) -} + ) : null +) const Pathogenicity = React.memo(({ variant, showHgmd }) => { const clinvar = variant.clinvar || {} const pathogenicity = [] - if ((clinvar.clinicalSignificance || clinvar.pathogenicity) && (clinvar.variationId || clinvar.alleleId)) { - const { pathogenicity: clinvarPathogenicity, assertions, severity } = clinvarSignificance(clinvar) + if (clinvar.pathogenicity && clinvar.alleleId) { pathogenicity.push(['ClinVar', { - label: clinvarLabel(clinvarPathogenicity, assertions, clinvar.conflictingPathogenicities), - color: clinvarColor(severity, 'red', 'orange', 'green'), - href: clinvarUrl(clinvar), + label: clinvarLabel(clinvar.pathogenicity, clinvar.assertions, clinvar.conflictingPathogenicities), + color: clinvarColor(clinvar, 'red', 'orange', 'green'), + href: `http://www.ncbi.nlm.nih.gov/clinvar?term=${clinvar.alleleId}[alleleid]`, goldStars: clinvar.goldStars, popup: clinvarPopup(clinvar), submitters: clinvar.submitters, diff --git a/ui/shared/components/panel/variants/Variants.jsx b/ui/shared/components/panel/variants/Variants.jsx index 7bad475092..7279b82dc0 100644 --- a/ui/shared/components/panel/variants/Variants.jsx +++ b/ui/shared/components/panel/variants/Variants.jsx @@ -3,7 +3,7 @@ import PropTypes from 'prop-types' import styled from 'styled-components' import { Grid, Popup, Label, Button, Header, Tab } from 'semantic-ui-react' -import { GENOME_VERSION_37, clinvarSignificance, clinvarColor, getVariantMainGeneId } from 'shared/utils/constants' +import { GENOME_VERSION_37, clinvarColor, getVariantMainGeneId } from 'shared/utils/constants' import { VerticalSpacer } from '../../Spacers' import { TagFieldDisplay } from '../view-fields/TagFieldView' import FamilyReads from '../family/FamilyReads' @@ -16,16 +16,16 @@ import VariantGenes, { VariantGene } from './VariantGene' import VariantIndividuals from './VariantIndividuals' import { compHetGene, has37Coords } from './VariantUtils' -export const StyledVariantRow = styled(({ isSV, severity, ...props }) => )` +export const StyledVariantRow = styled(({ isSV, severityColor, ...props }) => )` .column { margin-top: 0em !important; margin-bottom: 0em !important; } color: #999; - background-color: ${({ severity, isSV }) => { - if (severity !== undefined) { - return clinvarColor(severity, '#eaa8a857', '#f5d55c57', '#21a92624') || 'inherit' + background-color: ${({ severityColor, isSV }) => { + if (severityColor) { + return severityColor } if (isSV) { return '#f3f8fa' @@ -137,10 +137,9 @@ export const Variant = React.memo(( { variant, mainGeneId, reads, showReads, dispatch, isCompoundHet, updateReads, ...props }, ) => { const variantMainGeneId = mainGeneId || getVariantMainGeneId(variant) - const { severity } = clinvarSignificance(variant.clinvar) return ( ({ ...acc, [path]: i }), {}) const HGMD_SEVERITY = { @@ -1298,27 +1298,8 @@ export const getPermissionedHgmdClass = (variant, user, familiesByGuid, projectB familyGuid => projectByGuid[familiesByGuid[familyGuid].projectGuid].enableHgmd, )) && variant.hgmd && variant.hgmd.class -export const clinvarSignificance = (clinvar) => { - let { pathogenicity, assertions } = clinvar || {} - const { clinicalSignificance } = clinvar || {} - if (clinicalSignificance && !pathogenicity) { - [pathogenicity, ...assertions] = clinicalSignificance.split(/[,|]/) - if (pathogenicity === 'Pathogenic/Likely_pathogenic/Pathogenic') { - pathogenicity = 'Pathogenic/Likely_pathogenic' - } else if (pathogenicity === 'Pathogenic/Pathogenic') { - pathogenicity = 'Pathogenic' - } - if (!(pathogenicity.replace(' ', '_').toLowerCase() in CLINVAR_PATHOGENICITIES)) { - assertions = [pathogenicity, ...assertions] - pathogenicity = CLINVAR_DEFAULT_PATHOGENICITY - } - assertions = assertions.map(a => a.replace(/^_/, '')) - } - - return { pathogenicity, assertions, severity: CLINVAR_PATHOGENICITIES[pathogenicity?.replace(' ', '_').toLowerCase()] } -} - -export const clinvarColor = (severity, pathColor, riskColor, benignColor) => { +export const clinvarColor = (clinvar, pathColor, riskColor, benignColor) => { + const severity = CLINVAR_PATHOGENICITIES[clinvar?.pathogenicity] if (severity > CLINVAR_PATHOGENICITIES[CLINVAR_MAX_RISK_PATHOGENICITY]) { return pathColor } @@ -1332,10 +1313,10 @@ export const clinvarColor = (severity, pathColor, riskColor, benignColor) => { } const clinsigSeverity = (variant, user, familiesByGuid, projectByGuid) => { - const { pathogenicity, severity } = clinvarSignificance(variant.clinvar) + const { pathogenicity } = variant.clinvar || {} const hgmdSignificance = getPermissionedHgmdClass(variant, user, familiesByGuid, projectByGuid) if (!pathogenicity && !hgmdSignificance) return -10 - const clinvarSeverity = pathogenicity ? severity + 1 : 0.1 + const clinvarSeverity = pathogenicity ? CLINVAR_PATHOGENICITIES[pathogenicity] + 1 : 0.1 const hgmdSeverity = HGMD_SEVERITY[hgmdSignificance] || 0 return clinvarSeverity + hgmdSeverity } From 1d6f55ddfab8f4ef772cb620c43eebf73f0cd4ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 15:30:56 -0400 Subject: [PATCH 019/155] predictor clean up --- ui/shared/utils/constants.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index d40595aa95..d6a060ecba 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1563,7 +1563,6 @@ export const ORDERED_PREDICTOR_FIELDS = [ { field: 'mut_pred', thresholds: [0.0101, 0.392, 0.737, 0.829, 0.932], fieldTitle: 'MutPred', citation: CLINGEN_CITATION }, { field: 'primate_ai', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, 0.484, 0.79, 0.867, undefined], fieldTitle: 'PrimateAI', citation: CLINGEN_CITATION }, { field: 'eigen', group: CODING_IN_SILICO_GROUP, thresholds: [undefined, undefined, 1, 2, undefined], max: 99 }, - { field: 'dann', displayOnly: true, thresholds: [undefined, undefined, 0.93, 0.96, undefined] }, { field: 'strvctvre', group: SV_IN_SILICO_GROUP, thresholds: [undefined, undefined, 0.5, 0.75, undefined] }, { field: 'polyphen', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, 0.114, 0.978, 0.999, undefined], fieldTitle: 'PolyPhen', citation: CLINGEN_CITATION }, { field: 'sift', reverseThresholds: true, thresholds: [undefined, 0, 0.002, 0.081, undefined], group: MISSENSE_IN_SILICO_GROUP, fieldTitle: 'SIFT', citation: CLINGEN_CITATION }, @@ -1577,7 +1576,7 @@ export const ORDERED_PREDICTOR_FIELDS = [ thresholds: [undefined, undefined, 2.18, 4, undefined], citation: CLINGEN_CITATION, }, - { field: 'haplogroup_defining', indicatorMap: { Y: { color: 'green', value: '' }, true: { color: 'green', value: '' } } }, + { field: 'haplogroup_defining', indicatorMap: { true: { color: 'green', value: '' } } }, { field: 'mitotip', indicatorMap: MITOTIP_MAP, fieldTitle: 'MitoTIP' }, { field: 'hmtvar', thresholds: [undefined, undefined, 0.35, 0.35, undefined], fieldTitle: 'HmtVar' }, { field: 'mlc', thresholds: [undefined, 0.5, 0.5, 0.75, undefined], fieldTitle: 'MLC' }, @@ -1609,7 +1608,7 @@ export const predictionFieldValue = ( ) } - return indicatorMap && (indicatorMap[value[0]] || indicatorMap[value]) + return indicatorMap && indicatorMap[value] } export const predictorColorRanges = (thresholds, { citation, reverseThresholds, thresholdMap, absValue }) => (
From 96431c72edfef9edf2451638fb424422ead792e3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 16:55:46 -0400 Subject: [PATCH 020/155] sort export cleanup --- clickhouse_search/search.py | 2 +- clickhouse_search/search_tests.py | 6 +++--- ui/shared/utils/constants.js | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2c45bec171..f74e887d51 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -831,7 +831,7 @@ def _sv_size(x): 'alphamissense': [ lambda x: -max(t.get('alphamissensePathogenicity') or MIN_SORT_RANK for t in x[TRANSCRIPT_CONSEQUENCES_FIELD]) if x.get(TRANSCRIPT_CONSEQUENCES_FIELD) else MIN_SORT_RANK, ] + _subfield_sort(SELECTED_TRANSCRIPT_FIELD, 'alphamissensePathogenicity', reverse=True, default=MIN_SORT_RANK), - 'callset_af': _subfield_sort('populations', ('seqr', 'sv_callset'), 'ac'), + 'seqr_ac': _subfield_sort('populations', ('seqr', 'sv_callset'), 'ac'), 'family_guid': [lambda x: sorted(x.get('familyGuids', ['z']))[0]], 'gnomad': _subfield_sort('populations', ('gnomad_genomes', 'gnomad_mito', 'gnomad_svs'), 'af'), 'gnomad_exomes': _subfield_sort('populations', 'gnomad_exomes', 'af'), diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 11aa16e0ff..8fd63e415e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1978,7 +1978,7 @@ def test_sort(self): self._assert_expected_search( [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, VARIANT4, MULTI_FAMILY_VARIANT, VARIANT1, VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1], - sort='callset_af', + sort='seqr_ac', ) self._assert_expected_search( @@ -2048,7 +2048,7 @@ def test_sort(self): self._assert_expected_search( [MITO_VARIANT3, [VARIANT4, VARIANT3], VARIANT2], - sort='callset_af', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ + sort='seqr_ac', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) @@ -2108,7 +2108,7 @@ def test_multi_data_type_comp_het_sort(self): self._assert_expected_search( [[VARIANT4, VARIANT3], [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='callset_af', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ + sort='seqr_ac', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index d6a060ecba..1b8c22e55c 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1281,7 +1281,7 @@ const SORT_BY_PRIORITIZED_GENE = 'PRIORITIZED_GENE' const SORT_BY_PROTEIN_CONSQ = 'PROTEIN_CONSEQUENCE' const SORT_BY_GNOMAD_GENOMES = 'GNOMAD' const SORT_BY_GNOMAD_EXOMES = 'GNOMAD_EXOMES' -const SORT_BY_CALLSET_AF = 'CALLSET_AF' +const SORT_BY_SEQR_AC = 'SEQR_AC' const SORT_BY_CONSTRAINT = 'CONSTRAINT' const SORT_BY_CADD = 'CADD' const SORT_BY_REVEL = 'REVEL' @@ -1390,7 +1390,7 @@ const VARIANT_SORT_OPTONS = [ }, { value: SORT_BY_GNOMAD_GENOMES, text: 'gnomAD Genomes Frequency', comparator: populationComparator('gnomad_genomes') }, { value: SORT_BY_GNOMAD_EXOMES, text: 'gnomAD Exomes Frequency', comparator: populationComparator('gnomad_exomes') }, - { value: SORT_BY_CALLSET_AF, text: 'Callset AF', comparator: populationComparator('callset') }, + { value: SORT_BY_SEQR_AC, text: 'seqr AC', comparator: (a, b) => a.populations?.seqr?.ac - b.populations?.seqr?.ac }, { value: SORT_BY_CADD, text: 'CADD', comparator: predictionComparator('cadd') }, { value: SORT_BY_REVEL, text: 'REVEL', comparator: predictionComparator('revel') }, { value: SORT_BY_EIGEN, text: 'Eigen', comparator: predictionComparator('eigen') }, @@ -1707,7 +1707,7 @@ export const VARIANT_EXPORT_DATA = [ { header: 'alt' }, { header: 'gene', getVal: getVariantGene }, { header: 'worst_consequence', getVal: variant => getVariantMainTranscript(variant).majorConsequence }, - { header: 'callset_freq', getVal: variant => getPopAf('callset')(variant) || getPopAf('seqr')(variant) }, + { header: 'callset_freq', getVal: variant => getPopAf('seqr')(variant) }, { header: 'exac_freq', getVal: getPopAf('exac') }, { header: 'gnomad_genomes_freq', getVal: getPopAf('gnomad_genomes') }, { header: 'gnomad_exomes_freq', getVal: getPopAf('gnomad_exomes') }, @@ -1723,7 +1723,7 @@ export const VARIANT_EXPORT_DATA = [ { header: 'rsid', getVal: variant => variant.rsid }, { header: 'hgvsc', getVal: variant => getVariantMainTranscript(variant).hgvsc }, { header: 'hgvsp', getVal: variant => getVariantMainTranscript(variant).hgvsp }, - { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).clinicalSignificance || (variant.clinvar || {}).pathogenicity }, + { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).pathogenicity }, { header: 'clinvar_gold_stars', getVal: variant => (variant.clinvar || {}).goldStars }, { header: 'project' }, { header: 'family' }, From 4576018221dcbb33aa0a66b79c6f6ca2b9612b53 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 5 May 2026 17:04:28 -0400 Subject: [PATCH 021/155] clean up genotype --- .../panel/variants/VariantIndividuals.jsx | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/ui/shared/components/panel/variants/VariantIndividuals.jsx b/ui/shared/components/panel/variants/VariantIndividuals.jsx index 35ad3bdaff..0558bbafa3 100644 --- a/ui/shared/components/panel/variants/VariantIndividuals.jsx +++ b/ui/shared/components/panel/variants/VariantIndividuals.jsx @@ -202,18 +202,10 @@ Alleles.propTypes = { const GENOTYPE_DETAILS = [ { title: 'Sample Type', field: 'sampleType' }, - { - title: 'Raw Alt. Alleles', - variantField: 'originalAltAlleles', - format: val => (val || []).join(', '), - shouldHide: (val, variant) => (val || []).length < 1 || ((val || []).length === 1 && val[0] === variant.alt), - }, - { title: 'Allelic Depth', field: 'ad' }, { title: 'Read Depth', field: 'dp' }, { title: 'Genotype Quality', field: 'gq' }, { title: 'Allelic Balance', field: 'ab', format: val => val && val.toPrecision(2) }, - { title: 'Filter', field: 'filters', variantField: 'genotypeFilters', shouldHide: val => (val || []).length < 1 }, - { title: 'Phred Likelihoods', field: 'pl' }, + { title: 'Filter', field: 'filters' }, { title: 'Quality Score', field: 'qs' }, { title: 'Mitochondrial Copy Number', @@ -236,10 +228,10 @@ const SV_GENOTYPE_DETAILS = [ }, ] -const formattedGenotypeDetails = (details, genotype, variant, genesById) => details.map( - ({ shouldHide, title, field, variantField, format, comment }) => { - const value = genotype[field] || variant[variantField] - return value && !(shouldHide && shouldHide(value, variant)) ? ( +const formattedGenotypeDetails = (details, genotype, genesById) => details.map( + ({ title, field, format, comment }) => { + const value = genotype[field] + return value ? (
{`${title}: `} {format ? format(value, genesById) : value} @@ -249,9 +241,9 @@ const formattedGenotypeDetails = (details, genotype, variant, genesById) => deta }, ).filter(val => val) -const genotypeDetails = (genotype, variant, genesById) => { - const details = formattedGenotypeDetails(GENOTYPE_DETAILS, genotype, variant) - const svDetails = formattedGenotypeDetails(SV_GENOTYPE_DETAILS, genotype, variant, genesById) +const genotypeDetails = (genotype, genesById) => { + const details = formattedGenotypeDetails(GENOTYPE_DETAILS, genotype) + const svDetails = formattedGenotypeDetails(SV_GENOTYPE_DETAILS, genotype, genesById) if (svDetails.length < 1) { return details } @@ -393,7 +385,7 @@ const GenotypeQuality = ({ genotype, variant, showSampleType }) => { const showSecondaryQuality = !variant.svType && genotype.numAlt >= 0 const secondaryQuality = genotype.ab || genotype.hl const quality = Number.isInteger(genotype.gq) ? genotype.gq : genotype.qs - const filters = genotype.filters?.join(', ') || variant.genotypeFilters + const filters = genotype.filters?.join(', ') return (
@@ -442,8 +434,8 @@ const Genotype = React.memo(({ variant, individual, isCompoundHet, genesById }) const details = genotypes.flatMap((genotype, index) => ( index === 0 ? - [genotypeDetails(genotype, variant, genesById)] : - [, genotypeDetails(genotype, variant, genesById)] + [genotypeDetails(genotype, genesById)] : + [, genotypeDetails(genotype, genesById)] )) const content = ( From b296883ee2174a516d378a37c14b240a01c9fae8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 May 2026 11:41:11 -0400 Subject: [PATCH 022/155] clean up --- clickhouse_search/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index bd3c1dd1eb..b43764cd73 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -154,7 +154,7 @@ }, 'transcripts': { 'ENSG00000177000': [ - {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'git config --global user.name': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, + {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, From 1a06aa76b95084a07cc90b5ddc41ac11a296f796 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 May 2026 11:48:55 -0400 Subject: [PATCH 023/155] fix linting --- ui/shared/components/panel/variants/Frequencies.jsx | 6 +++--- ui/shared/components/panel/variants/Pathogenicity.jsx | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/shared/components/panel/variants/Frequencies.jsx b/ui/shared/components/panel/variants/Frequencies.jsx index 1d17c1117a..19b6a862ac 100644 --- a/ui/shared/components/panel/variants/Frequencies.jsx +++ b/ui/shared/components/panel/variants/Frequencies.jsx @@ -190,14 +190,14 @@ gnomadLink.propTypes = { fieldTitle: PropTypes.string, } -const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN }) => { - return (ac && an < (maxAN / 2)) ? ( +const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN }) => ( + (ac && an < (maxAN / 2)) ? ( } content={`This variant is covered in fewer than 50% of individuals in ${fieldTitle}. This may indicate a low-quality site.`} /> ) : null -} +) const GNOMAD_URL_INFO = { urls: { [GENOME_VERSION_37]: 'gnomad.broadinstitute.org', [GENOME_VERSION_38]: 'gnomad.broadinstitute.org' }, diff --git a/ui/shared/components/panel/variants/Pathogenicity.jsx b/ui/shared/components/panel/variants/Pathogenicity.jsx index 6489cf26d4..fcf4b9fa9a 100644 --- a/ui/shared/components/panel/variants/Pathogenicity.jsx +++ b/ui/shared/components/panel/variants/Pathogenicity.jsx @@ -83,12 +83,12 @@ const clinvarLabel = (pathogenicity, assertions, conflictingPathogenicities) => return label } -const clinvarPopup = (clinvar) => ( - clinvar.conditions ? ( +const clinvarPopup = ({ conditions }) => ( + conditions ? (
Conditions: - {[...new Set(clinvar.conditions)].map(condition => ( + {[...new Set(conditions)].map(condition => ( {condition} ))} From e94aaf48edce924ba1b193cd7af68c7a636381b7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 11:28:17 -0400 Subject: [PATCH 024/155] caid test coverage --- .../management/tests/register_caids_tests.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index 9985799c50..d0c27f257f 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -50,6 +50,26 @@ }, ], }, + { + '@id': 'http://reg.genome.network/allele/CA16716504', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': 'C', + 'end': 10131, + 'referenceAllele': '', # ref allele is '' + 'start': 10131, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + 'externalRecords': { + 'gnomAD_4': [{'id': '1-91511686-91511734'}], # has invalid gnomad ID + }, + }, { '@id': 'http://reg.genome.network/allele/CA997563845', 'genomicAlleles': [ @@ -135,6 +155,21 @@ def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): call_command("register_caids", batch_size=5) mock_safe_post_to_slack.assert_not_called() + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=400, + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + responses.reset() mock_safe_post_to_slack.reset_mock() mock_logger.reset_mock() @@ -224,7 +259,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') self.assertEqual(dv_38.version, '10') mock_logger.info.assert_called_with( - "2 registered variant(s) cannot be mapped back to ours. \n" + "3 registered variant(s) cannot be mapped back to ours. \n" "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" ) mock_logger.warning.assert_called_with( From c99964db9a306bc4b9e0cb83dbf744057a749359 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 11:39:05 -0400 Subject: [PATCH 025/155] relaod clinvar type sepcific helper function --- .../commands/reload_clinvar_all_variants.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 8f14c1a289..bc23dca7b6 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -20,27 +20,19 @@ logger = logging.getLogger(__name__) -def replace_underscores_with_spaces(value: Union[str, list[str]]) -> Union[str, list[str]]: - if isinstance(value, str): - return value.replace('_', ' ') - elif isinstance(value, list): - return [s.replace('_', ' ') for s in value] - raise TypeError("Expected str or list[str]") +def replace_underscores_with_spaces(value: list[str]) -> list[str]: + return [s.replace('_', ' ') for s in value] -def replace_spaces_with_underscores(value: Union[str, list[str], list[tuple[str, int]]]) -> Union[str, list[str]]: - if isinstance(value, str): - return value.replace(' ', '_') - elif isinstance(value, list): - if len(value) > 0 and isinstance(value[0], tuple): - return [(t[0].replace(' ', '_'), t[1]) for t in value] - return [s.replace(' ', '_') for s in value] - raise TypeError("Expected str or list[str]") +def replace_spaces_with_underscores(value: Union[list[str], list[tuple[str, int]]]) -> list[str]: + if len(value) > 0 and isinstance(value[0], tuple): + return [(t[0].replace(' ', '_'), t[1]) for t in value] + return [s.replace(' ', '_') for s in value] BATCH_SIZE = 1000 CLINVAR_ASSERTIONS = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_ASSERTIONS) -CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY) +CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY])[0] CLINVAR_CONFLICTING_DATA_FROM_SUBMITTERS = 'conflicting data from submitters' -CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY) +CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY])[0] CLINVAR_PATHOGENICITIES = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_PATHOGENICITIES) CLINVAR_GOLD_STARS_LOOKUP = { 'no classification for the single variant': 0, @@ -228,7 +220,7 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, props = { 'version': new_version, 'allele_id': allele_id, - 'pathogenicity': replace_spaces_with_underscores(pathogenicity), + 'pathogenicity': replace_spaces_with_underscores([pathogenicity])[0], 'assertions': replace_spaces_with_underscores(assertions), 'conflicting_pathogenicities': replace_spaces_with_underscores(conflicting_pathogenicities), 'gold_stars': gold_stars, From 30e51d50f531515f6ac3863ac300d594844d8097 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 12:30:11 -0400 Subject: [PATCH 026/155] manager clean up --- clickhouse_search/managers.py | 39 +++++++------------------------ clickhouse_search/search_tests.py | 7 ++++-- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7924553107..6ada3818e1 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -303,11 +303,6 @@ def entry_model(self): def gt_stats_dict_rel(self): return getattr(self.entry_model, 'gt_stats', None) - @property - def genome_version(self): - return self.model.ANNOTATION_CONSTANTS['genomeVersion'] - - def subquery_join(self, subquery, join_key='key'): join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) @@ -406,13 +401,6 @@ def _comp_het_conditional_fields(self, query, prefix=''): if field in query.query.annotations } - @property - def populations(self): - return { - population: {subfield for subfield, _ in field.base_fields} - for population, field in self.model.POPULATION_FIELDS - } - def _filter_frequency(self, results, **kwargs): return results @@ -645,7 +633,7 @@ def _parse_annotation_filters(self, annotations, pathogenicity): filter_qs.append(Q(screen__in=value)) elif field == 'mitomap_pathogenic': if self.has_annotation('mitomapPathogenic'): - filter_qs.append(Q(mitomapPathogenic=value)) + filter_qs.append(Q(mitomapPathogenic=value)) # TODO should be covered? elif field == SPLICE_AI_FIELD: if value and SPLICE_AI_FIELD in self.entry_model.PREDICTIONS: filter_qs.append(Q(preds__0__gte=float(value))) @@ -872,16 +860,16 @@ def _parse_in_silico_qs(self, results, in_silico, require_score, in_silico_qs, i return super()._parse_in_silico_qs(results, in_silico, require_score, in_silico_qs, in_silico_missing_qs) - def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): + def _filter_frequency(self, results, freqs=None, **kwargs): for population, pop_filter in (freqs or {}).items(): - pop_subfields = self.populations.get(population) - if not pop_subfields: + pop_field = dict(self.model.POPULATION_FIELDS).get(population) + if not pop_field: continue if pop_filter.get('af') is not None and pop_filter['af'] < 1: results = results.filter(**{f'populations__{population}__af__lte': pop_filter['af']}) elif pop_filter.get('ac') is not None: - if 'ac' in pop_subfields: + if any(subfield == 'ac' for subfield, _ in pop_field.base_fields): ac_field = f'populations__{population}__ac' else: ac_field = f'ac_{population}' @@ -889,9 +877,6 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): results = results.filter(**{f'{ac_field}__lte': pop_filter['ac']}) - if pop_filter.get('hh') is not None: - results = results.filter(**{f'populations__{population}__hom__lte': pop_filter['hh']}) - return results @@ -1012,10 +997,6 @@ def genotype_fields(self): def gt_stats_dict_rel(self): return getattr(self.model, 'gt_stats', None) - @property - def genome_version(self): - return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] - @property def filtered_chrom(self): return self.annotations_model.ANNOTATION_CONSTANTS.get('chrom') @@ -1432,7 +1413,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=None, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): + def filter_locus(self, exclude_intervals=None, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): entries = self if variant_ids: @@ -1449,7 +1430,7 @@ def filter_locus(self, exclude_intervals=None, require_gene_filter=False, interv raise InvalidDatasetTypeException if genes or intervals: - entries = entries.filter(self._filter_locations_q(intervals, genes, require_gene_filter)) + entries = entries.filter(self._filter_locations_q(intervals, genes)) elif exclude_intervals: entries = entries.exclude(self._filter_locations_q(exclude_intervals)) @@ -1463,13 +1444,13 @@ def _parse_variant_ids(raw_variant_items): parsed_variant_ids[variant_id] = parse_variant_id(variant_id) return parsed_variant_ids - def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): + def _filter_locations_q(self, intervals, genes=None): locus_q = None if genes: should_filter_interval = self._can_filter_gene_interval(genes) if should_filter_interval: intervals = list((genes or {}).values()) + (intervals or []) - if require_gene_filter or (not should_filter_interval): + if not should_filter_interval: locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) if intervals: @@ -1478,8 +1459,6 @@ def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): interval_q |= self._interval_query(**interval) if locus_q is None: locus_q = interval_q - elif require_gene_filter: - locus_q &= interval_q else: locus_q |= interval_q diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 88f80e2dc6..ff4e44283c 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -520,7 +520,7 @@ def test_inheritance_filter(self): inheritance_mode = 'x_linked_recessive' self._assert_expected_search([], inheritance_mode=inheritance_mode) - # self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) + self._assert_expected_search([], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}) inheritance_mode = 'homozygous_recessive' self._assert_expected_search( @@ -1906,7 +1906,7 @@ def test_in_silico_filter(self): ) self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'requireScore': True}, + [VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'vest': None, 'requireScore': True}, ) self._assert_expected_search( @@ -1924,6 +1924,9 @@ def test_in_silico_filter(self): self._assert_expected_search( [SV_VARIANT4], in_silico=sv_in_silico, project_families=SV_PROJECT_FAMILIES, ) + self._assert_expected_search( + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3], in_silico={'strvctvre': 0.2}, project_families=SV_PROJECT_FAMILIES, + ) self._set_grch37_search() self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico, is_37=True) From 6eadb02d0d41a03c04508bd12111eeffa7acb073 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 12:49:16 -0400 Subject: [PATCH 027/155] caid error handling clean up --- clickhouse_search/management/commands/register_caids.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/commands/register_caids.py b/clickhouse_search/management/commands/register_caids.py index dd0455bc93..95cda66b7c 100644 --- a/clickhouse_search/management/commands/register_caids.py +++ b/clickhouse_search/management/commands/register_caids.py @@ -154,8 +154,10 @@ def handle_api_response( genome_version: Literal[GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38], response: requests.Response, ) -> dict[str, str]: + if not response.ok: + raise HTTPError(f"Unexpected AR response code: {response.status_code}") response_json = response.json() - if not response.ok or "errorType" in response_json: + if "errorType" in response_json: error = AlleleRegistryError.from_api_response(response_json) logger.error(error) raise HTTPError(error.message) From 0ccc1e886c73f1c8855356bd0a504e06403e92c8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 14:02:18 -0400 Subject: [PATCH 028/155] caid error handling clean up --- clickhouse_search/management/commands/register_caids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/management/commands/register_caids.py b/clickhouse_search/management/commands/register_caids.py index 95cda66b7c..72d13929e6 100644 --- a/clickhouse_search/management/commands/register_caids.py +++ b/clickhouse_search/management/commands/register_caids.py @@ -108,8 +108,8 @@ class AlleleRegistryError: def from_api_response(cls, response: dict): return cls( error_type=response["errorType"], - description=response["description"], - message=response["message"], + description=response.get("description"), + message=response.get("message"), input_line=response.get("inputLine"), ) From afe878a9cf4e97542b2835d8983370f4a158cbe6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 14:36:33 -0400 Subject: [PATCH 029/155] search coverage --- clickhouse_search/search.py | 27 +++++---------------------- clickhouse_search/search_tests.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2c45bec171..ebb5926e12 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -514,9 +514,6 @@ def format_clickhouse_export_results(results): def format_clickhouse_results(results): - if not results: - return [] - genome_version = (results[0] if isinstance(results[0], list) else results)[0]['genomeVersion'] keys_with_no_details = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant @@ -1020,14 +1017,6 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids): return lookup -def _main_transcript(selected_transcript_id, sorted_transcripts): - if not sorted_transcripts: - return {} - if selected_transcript_id: - return next((t for t in sorted_transcripts if t['transcriptId'] == selected_transcript_id), {}) - return sorted_transcripts[0] - - def delete_clickhouse_project(project, dataset_type, sample_type=None): if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES: dataset_type = 'GCNV' @@ -1061,22 +1050,16 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): GENOME_VERSION_GRCh38: ('hg19', 'hg38'), GENOME_VERSION_GRCh37: ('hg38', 'hg19'), } -def _get_liftover(genome_version): + +def _run_liftover(genome_version, chrom, pos): if not LIFTOVERS[genome_version]: try: LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) except Exception as e: logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) - return LIFTOVERS[genome_version] - + return None -def _run_liftover(genome_version, chrom, pos): - liftover = _get_liftover(genome_version) - if not liftover: - return None - lifted_coord = liftover.convert_coordinate( + lifted_coord = LIFTOVERS[genome_version].convert_coordinate( 'chr{}'.format(chrom.lstrip('chr')), int(pos) ) - if lifted_coord and lifted_coord[0]: - return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) - return None + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) if lifted_coord and lifted_coord[0] else None diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ff4e44283c..e4b41158d3 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -519,8 +519,11 @@ def test_inheritance_filter(self): ) inheritance_mode = 'x_linked_recessive' - self._assert_expected_search([], inheritance_mode=inheritance_mode) + self._assert_expected_search([], inheritance_mode=inheritance_mode, export_data=[EXPORT_DATA[0][:24]]) self._assert_expected_search([], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}) + self._assert_expected_search( + [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'}, + ) inheritance_mode = 'homozygous_recessive' self._assert_expected_search( @@ -1807,6 +1810,15 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) + self._assert_expected_search( + [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], + inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, + annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, [ + {'selectedGeneId': 'ENSG00000277258'}, + {'selectedGeneId': 'ENSG00000277258'}, + ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], + ) + selected_transcript_annotations = {'other': ['non_coding_transcript_exon_variant']} self._assert_expected_search( [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], From f30958367dc65b8c9548445a063da70502810ac5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 14:44:33 -0400 Subject: [PATCH 030/155] panel app cleanup --- panelapp/panelapp_utils.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/panelapp/panelapp_utils.py b/panelapp/panelapp_utils.py index d4b8112e53..7394e28d05 100644 --- a/panelapp/panelapp_utils.py +++ b/panelapp/panelapp_utils.py @@ -14,32 +14,17 @@ class TooManyRequestsError(Exception): pass -def _extract_ensembl_id_from_json(raw_gene_json): - ensembl_genes_json = raw_gene_json.get('gene_data', {}).get('ensembl_genes') - if ensembl_genes_json and isinstance(ensembl_genes_json, dict): - return ensembl_genes_json \ - .get('GRch38', {}) \ - .get('90', {}) \ - .get('ensembl_id') - else: - return None - - def get_valid_panel_genes(panel_app_id, panel, panels_api_url, genes_by_panel_id, gene_ids_to_gene): if len(genes_by_panel_id[panel_app_id]) != panel['stats']['number_of_genes']: panel_genes_url = f'{panels_api_url}/{panel_app_id}/genes' _get_all_genes(panel_app_id, panel_genes_url, genes_by_panel_id) - all_genes_for_panel = genes_by_panel_id[panel_app_id] - if not all_genes_for_panel: - return {} - panel_genes_by_id = { - _extract_ensembl_id_from_json(gene): gene for gene in all_genes_for_panel - if _extract_ensembl_id_from_json(gene) + gene['gene_data']['ensembl_genes'].get('GRch38', {}).get('90', {}).get('ensembl_id'): gene + for gene in genes_by_panel_id[panel_app_id] if isinstance(gene.get('gene_data', {}).get('ensembl_genes'), dict) } valid_panel_genes = { - gene_id: panel_gene for gene_id, panel_gene in panel_genes_by_id.items() if gene_id in gene_ids_to_gene + gene_id: panel_gene for gene_id, panel_gene in panel_genes_by_id.items() if gene_id and gene_id in gene_ids_to_gene } if len(panel_genes_by_id) > len(valid_panel_genes): invalid_items = sorted(set(panel_genes_by_id.keys()) - set(valid_panel_genes.keys())) From a42533c5a293c93d2e2b34784d771cd4ae098c83 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 14:47:25 -0400 Subject: [PATCH 031/155] debug --- clickhouse_search/search_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e4b41158d3..ccc0f39685 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1810,10 +1810,17 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) + # TODO debug + self._assert_expected_search_error('?', inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, + annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, {}, [ + {'selectedGeneId': 'ENSG00000277258'}, + {'selectedGeneId': 'ENSG00000277258'}, + ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}]) + self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, [ + annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, {}, [ {'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}, ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], From a69144bd290ae1cb8b15cd4f33f56a27303c5f2d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:05:35 -0400 Subject: [PATCH 032/155] clean up export --- clickhouse_search/search.py | 34 +++++++++++++-------------- seqr/views/apis/variant_search_api.py | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index ebb5926e12..b841b1f1d3 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -482,20 +482,12 @@ def get_sorted_search_results(results, sort, families): def format_clickhouse_export_results(results): - formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] - if not formatted_results: - return [] - - genome_version = formatted_results[0]['genomeVersion'] - keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} - detail_qs = get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) - details_by_key = { - detail['key']: detail for detail in detail_qs.values( - 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), - **detail_qs.split_variant_id_annotations(), - ) - } + details_by_key = _get_details_by_key(results, lambda detail_qs: detail_qs.values( + 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), + **detail_qs.split_variant_id_annotations(), + )) + formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] gene_ids = set() for result in formatted_results: if 'transcripts' in result: @@ -513,16 +505,24 @@ def format_clickhouse_export_results(results): return formatted_results -def format_clickhouse_results(results): +def _get_details_by_key(results, format_details): + if not results: + return {} + genome_version = (results[0] if isinstance(results[0], list) else results)[0]['genomeVersion'] keys_with_no_details = { - variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant + variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if + not 'transcripts' in variant } - details_by_key = { + return { detail['key']: detail for detail in - get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + format_details(get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details)) } + +def format_clickhouse_results(results): + details_by_key = _get_details_by_key(results, lambda detail_qs: detail_qs.result_values()) + formatted_results = [] for variant in results: if isinstance(variant, list): diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 61f1748532..19b888276f 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -352,8 +352,8 @@ def export_variants_handler(request, search_hash): variants = split_variants - max_families_per_variant = max([len(variant.get('familyGuids', [1])) for variant in variants]) - max_samples_per_variant = max([len(variant.get('genotypes', {})) for variant in variants]) + max_families_per_variant = max([len(variant.get('familyGuids', [1])) for variant in variants] or [0]) + max_samples_per_variant = max([len(variant.get('genotypes', {})) for variant in variants] or [0]) rows = [] for variant in variants: From 03475e8fe3057d31ecf53c73eeaaee883432d7a6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:22:53 -0400 Subject: [PATCH 033/155] fix test --- clickhouse_search/search_tests.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ccc0f39685..9b371c9bc8 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1810,16 +1810,10 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) - # TODO debug - self._assert_expected_search_error('?', inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, {}, [ - {'selectedGeneId': 'ENSG00000277258'}, - {'selectedGeneId': 'ENSG00000277258'}, - ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}]) - self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, + locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, {}, [ {'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}, From fe4c30b339492a58febc0aa4a37af87f45a246fd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:24:26 -0400 Subject: [PATCH 034/155] debug --- clickhouse_search/search_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9b371c9bc8..41bff159e2 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -166,6 +166,10 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) + # TODO debug + if response.status_code == 500: + self.maxDiff = None + self.assertDictEqual(response.json(), {}) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariantIds': [], From 85b68f94e47495b197fa9dab6fc8ef8e5ae8361d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:40:30 -0400 Subject: [PATCH 035/155] reference data prase gene record cleanup --- reference_data/models.py | 94 ++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/reference_data/models.py b/reference_data/models.py index 8aad05578c..4de80437aa 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -336,6 +336,8 @@ def update_records(cls, gencode_release=CURRENT_VERSION, existing_gene_ids=None, class GeneMetadataModel(LoadableModel): + RECORD_FIELDS = {} + gene = models.ForeignKey(GeneInfo, on_delete=models.CASCADE) class Meta: @@ -358,9 +360,12 @@ def parse_record(cls, record, skipped_genes=None, **kwargs): record = None yield record - @staticmethod - def parse_gene_record(record): - return record + @classmethod + def parse_gene_record(cls, record): + return { + field: format(record[record_field]) if format else record[record_field] + for field, (record_field, format) in cls.RECORD_FIELDS.items() + } @classmethod def update_records(cls, **kwargs): @@ -436,6 +441,13 @@ class GeneConstraint(GeneMetadataModel): CURRENT_VERSION = 'gnomad.v2.1.1.lof_metrics.by_gene' URL = f'http://storage.googleapis.com/seqr-reference-data/gene_constraint/{CURRENT_VERSION}.txt' + RECORD_FIELDS = { + 'gene_id': ('gene_id', lambda gene_id: gene_id.split(".")[0]), + 'gene_symbol': ('gene', None), + 'mis_z': ('mis_z', lambda value: float(value) if value != 'NaN' else -100), + 'pLI': ('pLI', lambda value: float(value) if value != 'NA' else 0), + 'louef': ('oe_lof_upper', lambda value: float(value) if value != 'NA' else 100), + } mis_z = models.FloatField() mis_z_rank = models.IntegerField() @@ -447,16 +459,6 @@ class GeneConstraint(GeneMetadataModel): class Meta: json_fields = ['mis_z', 'mis_z_rank', 'pLI', 'pLI_rank', 'louef', 'louef_rank'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_id': record['gene_id'].split(".")[0], - 'gene_symbol': record['gene'], - 'mis_z': float(record['mis_z']) if record['mis_z'] != 'NaN' else -100, - 'pLI': float(record['pLI']) if record['pLI'] != 'NA' else 0, - 'louef': float(record['oe_lof_upper']) if record['oe_lof'] != 'NA' else 100, - } - @classmethod def get_record_models(cls, records, **kwargs): # add _rank fields @@ -470,6 +472,11 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): CURRENT_VERSION = 'Collins_rCNV_2022' URL = f'https://zenodo.org/record/6347673/files/{CURRENT_VERSION}.dosage_sensitivity_scores.tsv.gz' + RECORD_FIELDS = { + 'gene_symbol': ('#gene', None), + 'pHI': ('pHaplo', float), + 'pTS': ('pTriplo', float), + } pHI = models.FloatField() pTS = models.FloatField() @@ -477,32 +484,21 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): class Meta: json_fields = ['pHI', 'pTS'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['#gene'], - 'pHI': float(record['pHaplo']), - 'pTS': float(record['pTriplo']), - } - class GeneShet(GeneMetadataModel): CURRENT_VERSION = '7939768' URL = f'https://zenodo.org/record/{CURRENT_VERSION}/files/s_het_estimates.genebayes.tsv' + RECORD_FIELDS = { + 'gene_id': ('ensg', None), + 'post_mean': ('post_mean', float), + } post_mean = models.FloatField() class Meta: json_fields = ['post_mean'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_id': record['ensg'], - 'post_mean': float(record['post_mean']), - } - class Omim(LoadableModel): @@ -682,8 +678,8 @@ class dbNSFPGene(GeneMetadataModel): class Meta: json_fields = ['function_desc', 'disease_desc', 'gene_names'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): parsed_record = {DBNSFP_FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') for k, v in record.items() if not k.startswith(DBNSFP_EXCLUDE_FIELDS)} parsed_record["function_desc"] = parsed_record["function_desc"].replace("FUNCTION: ", "") @@ -703,6 +699,11 @@ class PrimateAI(GeneMetadataModel): CURRENT_VERSION = 'cleaned_v0.2' URL = f'http://storage.googleapis.com/seqr-reference-data/primate_ai/Gene_metrics_clinvar_pcnt.{CURRENT_VERSION}.txt' + RECORD_FIELDS = { + 'gene_symbol': ('genesymbol', None), + 'percentile_25': ('pcnt25', float), + 'percentile_75': ('pcnt75', float), + } percentile_25 = models.FloatField() percentile_75 = models.FloatField() @@ -710,19 +711,12 @@ class PrimateAI(GeneMetadataModel): class Meta: json_fields = ['percentile_25', 'percentile_75'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['genesymbol'], - 'percentile_25': float(record['pcnt25']), - 'percentile_75': float(record['pcnt75']), - } - class MGI(GeneMetadataModel): CURRENT_VERSION = 'HMD_HumanPhenotype' URL = f'https://storage.googleapis.com/seqr-reference-data/mgi/{CURRENT_VERSION}.rpt.txt' + RECORD_FIELDS = {k: (k, lambda v: v.strip()) for k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} marker_id = models.CharField(max_length=15) @@ -734,10 +728,6 @@ class Meta: def get_file_header(f): return ['gene_symbol', 'entrez_gene_id', 'mouse_gene_symbol', 'marker_id', 'phenotype_ids'] - @staticmethod - def parse_gene_record(record): - return {k: v.strip() for k, v in record.items() if k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} - @classmethod def update_records(cls, **kwargs): entrez_id_to_gene = dict(dbNSFPGene.objects.values_list('entrez_gene_id', 'gene_id')) @@ -783,12 +773,12 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['gene_symbol'], 'hgnc_id': record['gene_curie'], - 'classifications': [{title: record[field] for field, title in GenCC.CLASSIFICATION_FIELDS.items()}] + 'classifications': [{title: record[field] for field, title in cls.CLASSIFICATION_FIELDS.items()}] } @classmethod @@ -807,6 +797,12 @@ def get_record_models(cls, records, **kwargs): class ClinGen(GeneMetadataModel): URL = 'https://search.clinicalgenome.org/kb/gene-dosage/download' + RECORD_FIELDS = { + 'gene_symbol': ('gene_symbol', None), + 'haploinsufficiency': ('haploinsufficiency', lambda value: value.replace(' for Haploinsufficiency', '')), + 'triplosensitivity': ('triplosensitivity', lambda value: value.replace(' for Triplosensitivity', '')), + 'href': ('online_report', None), + } haploinsufficiency = models.TextField() triplosensitivity = models.TextField() @@ -836,14 +832,6 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['gene_symbol'], - 'haploinsufficiency': record['haploinsufficiency'].replace(' for Haploinsufficiency', ''), - 'triplosensitivity': record['triplosensitivity'].replace(' for Triplosensitivity', ''), - 'href': record['online_report'], - } class DataVersions(models.Model): data_model_name = models.CharField(max_length=30, primary_key=True) From af191423ade3a37af17340b88d4288d5463a7cad Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:53:38 -0400 Subject: [PATCH 036/155] fix error --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index b841b1f1d3..bb0712e7b5 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -712,7 +712,7 @@ def _affected_male_families(sample_data, affected_male_family_guids): if len(affected_male_family_guids) == sample_data['num_families']: return sample_data sample_type_families = { - sample_type: families.intersection(affected_male_family_guids) + sample_type: set(families).intersection(affected_male_family_guids) for sample_type, families in sample_data['sample_type_families'].items() } return { From 7d30d309f22f8856fe5781cc6376d1a6a663b23f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:54:37 -0400 Subject: [PATCH 037/155] debug --- clickhouse_search/search_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 41bff159e2..0e0530069f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1814,6 +1814,8 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) + # TODO debug + self.maxDiff = None self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, From 68e0fe0625825aa5539cc364c9fadb25e761d07a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 16:06:33 -0400 Subject: [PATCH 038/155] fix multi dt bug --- clickhouse_search/search.py | 2 +- clickhouse_search/search_tests.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index bb0712e7b5..fc6d83df68 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -323,7 +323,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d exclude_key_pairs=exclude_key_pairs.get(f'{Dataset.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), ) dataset_results = _evaluate_results(result_q, is_comp_het=True) - if not sv_sample_data['samples']: + if not (sv_sample_data['samples'] and type_snv_indel_sample_data['samples']): _add_individual_guids(dataset_results) results += dataset_results searched_dataset_types.add(sv_dataset_type) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 0e0530069f..cfeaf71a79 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -166,10 +166,6 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) - # TODO debug - if response.status_code == 500: - self.maxDiff = None - self.assertDictEqual(response.json(), {}) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariantIds': [], From b9cc8a3576f7456478329b36371b970aef276ab3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 16:21:52 -0400 Subject: [PATCH 039/155] add coverage --- clickhouse_search/search_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index cfeaf71a79..b08209fb0d 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -521,9 +521,6 @@ def test_inheritance_filter(self): inheritance_mode = 'x_linked_recessive' self._assert_expected_search([], inheritance_mode=inheritance_mode, export_data=[EXPORT_DATA[0][:24]]) self._assert_expected_search([], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}) - self._assert_expected_search( - [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'}, - ) inheritance_mode = 'homozygous_recessive' self._assert_expected_search( @@ -592,6 +589,10 @@ def test_inheritance_filter(self): ], project_families=SV_PROJECT_FAMILIES, ) + self._assert_expected_search( + [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'}, + ) + inheritance_mode = 'recessive' self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, gene_counts={ From c1f6bff983d955c135fb2415f304921a795c7e99 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 16:26:24 -0400 Subject: [PATCH 040/155] clean up parent hpo mapping --- reference_data/models.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/reference_data/models.py b/reference_data/models.py index 4de80437aa..3c8dc65b01 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -215,16 +215,14 @@ def _get_category_id(parent_id_map, hpo_id): return None if hpo_id not in parent_id_map: - return None + raise ValueError('Strange id: %s' % hpo_id) + + parent_hpo_id = parent_id_map[hpo_id] + if parent_hpo_id == 'HP:0000118': + return hpo_id - while hpo_id and parent_id_map.get(hpo_id) != 'HP:0000118': - if hpo_id not in parent_id_map: - raise ValueError('Strange id: %s' % hpo_id) - hpo_id = parent_id_map[hpo_id] - if hpo_id == 'HP:0000001': - return None + return _get_category_id(parent_id_map, parent_hpo_id) - return hpo_id class GeneInfo(LoadableModel): ALL_GENCODE_VERSIONS = ['39', '31', '29', '28', '27', '19'] From df837d6f8e4453292502055f6effb109c96d6516 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:04:25 -0400 Subject: [PATCH 041/155] fix hpo update --- reference_data/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reference_data/models.py b/reference_data/models.py index 3c8dc65b01..5dab1969b7 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -209,8 +209,8 @@ def get_record_models(cls, records, **kwargs): model.category_id = cls._get_category_id(parent_id_map, model.hpo_id) return models - @staticmethod - def _get_category_id(parent_id_map, hpo_id): + @classmethod + def _get_category_id(cls, parent_id_map, hpo_id): if hpo_id == 'HP:0000001': return None @@ -221,7 +221,7 @@ def _get_category_id(parent_id_map, hpo_id): if parent_hpo_id == 'HP:0000118': return hpo_id - return _get_category_id(parent_id_map, parent_hpo_id) + return cls._get_category_id(parent_id_map, parent_hpo_id) class GeneInfo(LoadableModel): From c7c70b7f6e18b4a6058dce9e22e118717d692a5e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:08:35 -0400 Subject: [PATCH 042/155] debug --- clickhouse_search/search_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index b08209fb0d..9c3e2baef1 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -589,6 +589,8 @@ def test_inheritance_filter(self): ], project_families=SV_PROJECT_FAMILIES, ) + # TODO + self._assert_expected_search_error('', inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'},) self._assert_expected_search( [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'}, ) From 26ab1f9d642c50c2da75609720ebedc90e402821 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:19:14 -0400 Subject: [PATCH 043/155] fix test --- clickhouse_search/search_tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9c3e2baef1..2e1c392f4e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -589,10 +589,9 @@ def test_inheritance_filter(self): ], project_families=SV_PROJECT_FAMILIES, ) - # TODO - self._assert_expected_search_error('', inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'},) self._assert_expected_search( - [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, locus={'rawItems': 'chrX:1-100000000'}, + [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, + **COMP_HET_ALL_PASS_FILTERS, locus={'rawItems': 'chrX:1-100000000'}, ) inheritance_mode = 'recessive' @@ -1813,8 +1812,6 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) - # TODO debug - self.maxDiff = None self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, From 1627ea18bc4d3f70d2bebc6e5d2a604bd5fc0e0e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:23:16 -0400 Subject: [PATCH 044/155] sample cleanup --- seqr/views/utils/project_context_utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 4319549b77..c03ceeb79a 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -92,11 +92,6 @@ def add_families_context(response, family_models, project_guid, user, is_analyst def add_child_ids(response): - if 'samplesByGuid' in response: - sample_guids_by_individual = defaultdict(list) - for sample in response['samplesByGuid'].values(): - sample_guids_by_individual[sample['individualGuid']].append(sample['sampleGuid']) - if 'igvSamplesByGuid' in response: igv_sample_guids_by_individual = defaultdict(list) for sample in response['igvSamplesByGuid'].values(): @@ -104,8 +99,6 @@ def add_child_ids(response): individual_guids_by_family = defaultdict(list) for individual in response['individualsByGuid'].values(): - if 'samplesByGuid' in response: - individual['sampleGuids'] = sample_guids_by_individual[individual['individualGuid']] if 'igvSamplesByGuid' in response: individual['igvSampleGuids'] = igv_sample_guids_by_individual[individual['individualGuid']] individual_guids_by_family[individual['familyGuid']].append(individual['individualGuid']) From b722fb9e09c8fe34025c5e751e4f3cf365b8da25 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:35:29 -0400 Subject: [PATCH 045/155] debug --- clickhouse_search/search_tests.py | 3 +++ .../management/tests/update_hpo_tests.py | 20 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 2e1c392f4e..6f58f41843 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -166,6 +166,9 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) + # TODO + if response.status_code == 500: + self.assertDictEqual({}, response.json()) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariantIds': [], diff --git a/reference_data/management/tests/update_hpo_tests.py b/reference_data/management/tests/update_hpo_tests.py index 7e0ef8c12c..c7b107a134 100644 --- a/reference_data/management/tests/update_hpo_tests.py +++ b/reference_data/management/tests/update_hpo_tests.py @@ -136,10 +136,22 @@ def test_update_hpo_command(self): 'Location': 'https://github.com/obophenotype/human-phenotype-ontology/releases/download/2025-03-12/hp.obo', }, } - self._test_update_command( - 'HumanPhenotypeOntology','2025-03-12', - existing_records=12, created_records=5, skipped_records=0, head_response=head_response, - ) + # TODO + # self._test_update_command( + # 'HumanPhenotypeOntology','2025-03-12', + # existing_records=12, created_records=5, skipped_records=0, head_response=head_response, + # ) + with self.assertRaises(CommandError) as e: + self._test_update_command( + 'HumanPhenotypeOntology', '2025-03-12', + existing_records=12, created_records=5, skipped_records=0, head_response=head_response, + ) + self.assert_json_logs(user=None, offset=3, expected=[ + ('unable to update HumanPhenotypeOntology: Strange id: HP:0000003', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) records = {record.hpo_id: { 'is_category': record.is_category, From 5012fc9c7177da382a1d5bc466b3c556d18e0b51 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:51:45 -0400 Subject: [PATCH 046/155] remove deprecated family structure notation --- seqr/views/utils/individual_utils.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b1fcaf7209..a9b924ca62 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -124,17 +124,11 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated record['displayName'] = '' # Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID - if 'father' in record or 'mother' in record: + if record.get(JsonConstants.MATERNAL_ID_COLUMN) is not None or record.get(JsonConstants.PATERNAL_ID_COLUMN) is not None: parent_updates.append({ 'individual': individual, - 'mother': record.pop('mother', None), - 'father': record.pop('father', None), - }) - elif record.get('maternalId') is not None or record.get('paternalId') is not None: - parent_updates.append({ - 'individual': individual, - 'maternalId': record.pop('maternalId', None), - 'paternalId': record.pop('paternalId', None), + JsonConstants.MATERNAL_ID_COLUMN: record.pop(JsonConstants.MATERNAL_ID_COLUMN, None), + JsonConstants.PATERNAL_ID_COLUMN: record.pop(JsonConstants.PATERNAL_ID_COLUMN, None), }) family_notes = record.pop(JsonConstants.FAMILY_NOTES_COLUMN, None) From dadf38dd28deb8ea9e39d9a910406e911ad271cf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:54:45 -0400 Subject: [PATCH 047/155] fix hpo update --- .../management/tests/update_hpo_tests.py | 20 ++++--------------- reference_data/models.py | 2 +- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/reference_data/management/tests/update_hpo_tests.py b/reference_data/management/tests/update_hpo_tests.py index c7b107a134..7e0ef8c12c 100644 --- a/reference_data/management/tests/update_hpo_tests.py +++ b/reference_data/management/tests/update_hpo_tests.py @@ -136,22 +136,10 @@ def test_update_hpo_command(self): 'Location': 'https://github.com/obophenotype/human-phenotype-ontology/releases/download/2025-03-12/hp.obo', }, } - # TODO - # self._test_update_command( - # 'HumanPhenotypeOntology','2025-03-12', - # existing_records=12, created_records=5, skipped_records=0, head_response=head_response, - # ) - with self.assertRaises(CommandError) as e: - self._test_update_command( - 'HumanPhenotypeOntology', '2025-03-12', - existing_records=12, created_records=5, skipped_records=0, head_response=head_response, - ) - self.assert_json_logs(user=None, offset=3, expected=[ - ('unable to update HumanPhenotypeOntology: Strange id: HP:0000003', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }), - ]) + self._test_update_command( + 'HumanPhenotypeOntology','2025-03-12', + existing_records=12, created_records=5, skipped_records=0, head_response=head_response, + ) records = {record.hpo_id: { 'is_category': record.is_category, diff --git a/reference_data/models.py b/reference_data/models.py index 5dab1969b7..e64a578852 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -211,7 +211,7 @@ def get_record_models(cls, records, **kwargs): @classmethod def _get_category_id(cls, parent_id_map, hpo_id): - if hpo_id == 'HP:0000001': + if hpo_id == 'HP:0000001' or hpo_id is None: return None if hpo_id not in parent_id_map: From 4efd9733552031c240c4e50ad3a1ceb8ee3ec0f6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:56:51 -0400 Subject: [PATCH 048/155] fix search error --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fc6d83df68..904202adb4 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -696,7 +696,7 @@ def _add_missing_multi_type_samples(individuals, data): def _no_affected_male_families(sample_data, user): sample_type_families = { - sample_type: families - set(sample_data['affected_male_family_guids']) + sample_type: set(families) - set(sample_data['affected_male_family_guids']) for sample_type, families in sample_data['sample_type_families'].items() } num_families = len(set().union(*sample_type_families.values())) From 6d0aeeaaa9f19f19a7331946a0cac64993cfff9e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 12:12:25 -0400 Subject: [PATCH 049/155] test csv file iter --- seqr/views/apis/project_api_tests.py | 48 ++++++++++++++-------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 4155a9b96a..d5ad854535 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -58,14 +58,14 @@ 'parsed_file_data': RNA_OUTLIER_SAMPLE_DATA, 'required_columns': RNA_OUTLIER_REQUIRED_COLUMNS, 'rows': [ - 'sampleID\tgeneID\tFDR set\tpValue\tpadjust\tzScore', - 'NA19675_1\tENSG00000240361\tdetail1\t0.01\t0.13\t-3.1', - 'NA19675_1\tENSG00000240361\tdetail2\t0.01\t0.13\t-3.1', - 'NA19675_1\tENSG00000233750\tdetail1\t0.064\t0.0000057\t7.8', - 'NA21234\tENSG00000233750\tdetail1\t0.064\t0.0000057\t7.8', - 'HG00731\tENSG00000240361\t\t0.04\t0.112\t1.9', - 'NA21234\tNOT_A_GENE_ID1\tdetail1\t0.064\t0.0000057\t7.8', - 'NA21234\t\tdetail1\t0.064\t0.0000057\t7.8', + 'sampleID,geneID,FDR set,pValue,padjust,zScore', + 'NA19675_1,ENSG00000240361,detail1,0.01,0.13,-3.1', + 'NA19675_1,ENSG00000240361,detail2,0.01,0.13-3.1', + 'NA19675_1,ENSG00000233750,detail1,0.064,0.0000057,7.8', + 'NA21234,ENSG00000233750,detail1,0.064,0.0000057,7.8', + 'HG00731,ENSG00000240361,,0.04,0.112,1.9', + 'NA21234,NOT_A_GENE_ID1,detail1,0.064,0.0000057,7.8', + 'NA21234,,detail1,0.064,0.0000057,7.8', ], 'message_data_type': 'Expression Outlier', }, @@ -76,11 +76,11 @@ 'required_columns': RNA_TPM_REQUIRED_COLUMNS, 'mismatch_field': 'tpm', 'rows': [ - 'Name\tDescription\tNA19675_1', - 'ENSG00000240361\tsome gene of interest\t7.8', - 'ENSG00000233750\t\t0.0', - 'NOT_A_GENE_ID1\t\t0.064', - '\t\t0.064', + 'Name,Description,NA19675_1', + 'ENSG00000240361,some gene of interest,7.8', + 'ENSG00000233750,,0.0', + 'NOT_A_GENE_ID1,,0.064', + ',,0.064', ], 'message_data_type': 'Expression', }, @@ -95,13 +95,13 @@ 'row_id': 'ENSG00000233750-2-167254166-167258349-*-psi3', 'invalid_format_field': 'p_value', 'rows': [ - 'hgncSymbol\tseqnames\tstart\tend\tstrand\tsampleID\ttype\tpValue\tpadjust\tdeltaPsi\tcounts\tmeanCounts\ttotalCounts\tmeanTotalCounts\tnonsplitCounts', - 'ENSG00000233750;ENSG00000240361\tchr2\t167254166\t167258349\t*\tNA19675_1\tpsi3\t1.56E-25\t-4.9\t-0.46\t166\t16.6\t1660\t1.66\t1', - 'ENSG00000240361\tchr7\t132885746\t132975168\t*\tNA19675_1\tpsi5\t1.08E-56\t-6.53\t-0.85\t231\t0.231\t2313\t231.3\t1', - 'ENSG00000233750\tchr2\t167258096\t167258349\t*\tNA21234\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - '\tchr2\t167258096\t167258349\t*\tHG00731\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - 'NOT_A_GENE_ID1\tchr2\t167258096\t167258349\t*\tNA21234\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - '\tchr2\t167258096\t167258349\t*\tNA19675_1\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', + 'hgncSymbol,seqnames,start,end,strand,sampleID,type,pValue,padjust,deltaPsi,counts,meanCounts,totalCounts,meanTotalCounts,nonsplitCounts', + 'ENSG00000233750;ENSG00000240361,chr2,167254166,167258349,*,NA19675_1,psi3,1.56E-25,-4.9,-0.46,166,16.6,1660,1.66,1', + 'ENSG00000240361,chr7,132885746,132975168,*,NA19675_1,psi5,1.08E-56,-6.53,-0.85,231,0.231,2313,231.3,1', + 'ENSG00000233750,chr2,167258096,167258349,*,NA21234,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + ',chr2,167258096,167258349,*,HG00731,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + 'NOT_A_GENE_ID1,chr2,167258096,167258349,*,NA21234,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + ',chr2,167258096,167258349,*,NA19675_1,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', ], 'message_data_type': 'Splice Outlier', } @@ -730,7 +730,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex self.login_manager() # Test errors - file = f'{self.TEMP_DIR}/new_samples.tsv.gz' + file = f'{self.TEMP_DIR}/new_samples.csv.gz' body = {'dataType': data_type, 'file': file, 'tissue': tissue, 'sequencingType': sequencing_type} self._set_file_not_found(file, mock_subprocess, mock_does_file_exist, mock_open) self.reset_logs() @@ -740,7 +740,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex mock_subprocess.return_value.wait.return_value = 0 self._set_local_file_iter([], mock_does_file_exist, mock_open) - invalid_file_ext = file.replace('tsv.gz', 'xlsx') + invalid_file_ext = file.replace('csv.gz', 'xlsx') invalid_body = {**body, 'file': invalid_file_ext} response = self.client.post(url, content_type='application/json', data=json.dumps(invalid_body)) self.assertEqual(response.status_code, 400) @@ -786,7 +786,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test database models are correct self.assertEqual(model_cls.objects.count(), initial_model_count - initial_sample_model_count) rna_samples = RnaSample.objects.filter( - tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, sequencing_type=sequencing_type, + tissue_type=tissue, data_type=data_type, data_source='new_samples.csv.gz', is_active=False, sequencing_type=sequencing_type, ) self.assertEqual(rna_samples.count(), 1 if single_sample_file else 2) guid_map = {'NA19675_1': rna_samples.get(individual_id=1).guid} @@ -796,7 +796,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test notifications subprocess_logs = self._get_expected_read_file_subprocess_calls([ - f'gsutil cp gs://seqr-scratch-temp/new_samples.tsv.gz tmp/temp_uploads/{file_path}', + f'gsutil cp gs://seqr-scratch-temp/new_samples.csv.gz tmp/temp_uploads/{file_path}', f'gsutil mv tmp/temp_uploads/{file_path}/*.json.gz gs://seqr-scratch-temp/{file_path}', ], mock_subprocess, wait_command=True) self.assert_json_logs(self.manager_user, subprocess_logs[:1] + [ From a8b190b4bc1ad9f5d97d9bfbe55327643a98a92b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 12:12:50 -0400 Subject: [PATCH 050/155] clean up --- clickhouse_search/search_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6f58f41843..2e1c392f4e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -166,9 +166,6 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) - # TODO - if response.status_code == 500: - self.assertDictEqual({}, response.json()) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariantIds': [], From a09ec5808e4f96b945bee164ecb522c47f5f690a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 12:28:43 -0400 Subject: [PATCH 051/155] clean up usage --- matchmaker/matchmaker_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matchmaker/matchmaker_utils.py b/matchmaker/matchmaker_utils.py index 4ed0cce103..644b1dfa9b 100644 --- a/matchmaker/matchmaker_utils.py +++ b/matchmaker/matchmaker_utils.py @@ -126,7 +126,7 @@ def _parse_mme_gene_variants(result, gene_symbols_to_ids): def get_gene_ids_for_feature(gene_feature, gene_symbols_to_ids): - gene_id = gene_feature.get('gene', {}).get('id') + gene_id = gene_feature['gene']['id'] if not gene_id: return [] if not gene_id.startswith('ENSG'): From dc5e1ffd1678ac326f54a1928b0339ef7c69176b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 12:33:05 -0400 Subject: [PATCH 052/155] fix typo --- seqr/views/apis/project_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index d5ad854535..b3806cb304 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -60,7 +60,7 @@ 'rows': [ 'sampleID,geneID,FDR set,pValue,padjust,zScore', 'NA19675_1,ENSG00000240361,detail1,0.01,0.13,-3.1', - 'NA19675_1,ENSG00000240361,detail2,0.01,0.13-3.1', + 'NA19675_1,ENSG00000240361,detail2,0.01,0.13,3.1', 'NA19675_1,ENSG00000233750,detail1,0.064,0.0000057,7.8', 'NA21234,ENSG00000233750,detail1,0.064,0.0000057,7.8', 'HG00731,ENSG00000240361,,0.04,0.112,1.9', From 4189a60b1cabe66df94fb0ce5201cf107e2688e4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 14:16:04 -0400 Subject: [PATCH 053/155] fix typo --- seqr/views/apis/project_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index b3806cb304..c0899a25a2 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -60,7 +60,7 @@ 'rows': [ 'sampleID,geneID,FDR set,pValue,padjust,zScore', 'NA19675_1,ENSG00000240361,detail1,0.01,0.13,-3.1', - 'NA19675_1,ENSG00000240361,detail2,0.01,0.13,3.1', + 'NA19675_1,ENSG00000240361,detail2,0.01,0.13,-3.1', 'NA19675_1,ENSG00000233750,detail1,0.064,0.0000057,7.8', 'NA21234,ENSG00000233750,detail1,0.064,0.0000057,7.8', 'HG00731,ENSG00000240361,,0.04,0.112,1.9', From 6eff66e5c329d62c4c0fa0f30c454c0a9ca90ec6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:10:55 -0400 Subject: [PATCH 054/155] clean up --- matchmaker/matchmaker_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/matchmaker/matchmaker_utils.py b/matchmaker/matchmaker_utils.py index 644b1dfa9b..b9731baa9a 100644 --- a/matchmaker/matchmaker_utils.py +++ b/matchmaker/matchmaker_utils.py @@ -127,8 +127,6 @@ def _parse_mme_gene_variants(result, gene_symbols_to_ids): def get_gene_ids_for_feature(gene_feature, gene_symbols_to_ids): gene_id = gene_feature['gene']['id'] - if not gene_id: - return [] if not gene_id.startswith('ENSG'): gene_ids = gene_symbols_to_ids.get(gene_feature['gene']['id'], []) else: From 19872c88012ed2f4da6e21aa3992349d732ac8da Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:35:40 -0400 Subject: [PATCH 055/155] fix genotype sorting --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 904202adb4..5d200dbc0e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -469,7 +469,7 @@ def _set_individual_guids(result, sample_map): for genotype in genotypes: individual_guid = sample_map[(family_guid, genotype['sampleId'])] individual_genotypes[individual_guid].append({**genotype, 'individualGuid': individual_guid}) - result['genotypes'] = {k: v[0] if len(v) == 1 else v for k, v in individual_genotypes.items()} + result['genotypes'] = {k: v[0] if len(v) == 1 else sorted(v, key=lambda g: g.get('sampleType')) for k, v in individual_genotypes.items()} def get_sorted_search_results(results, sort, families): From 115de3233706d55815586b200ae2334789d5ca64 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 12:49:24 -0400 Subject: [PATCH 056/155] test split pdo warning --- .../check_for_new_samples_from_pipeline_tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 7ae407db96..1d98257103 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -209,6 +209,7 @@ }, 'sex_check': { 'F000001_1': {'reasons': ['Sample NA19679 has pedigree sex F but imputed sex M']}, + 'F000011_11': {'reasons': ['Sample NA20870 has pedigree sex M but imputed sex F']}, 'F000014_14': {'reasons': ['Sample NA21987 has pedigree sex M but imputed sex F']}, }, 'missing_samples': { @@ -488,7 +489,7 @@ def _test_success_call(self, anvil_email_calls, next_dataset_id=155): }) # Test notifications - self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) + self.assertEqual(self.mock_send_slack.call_count, 8 + len(self.ADDITIONAL_SLACK_CALLS)) self.mock_send_slack.assert_has_calls([ mock.call( 'seqr-data-loading', @@ -508,6 +509,13 @@ def _test_success_call(self, anvil_email_calls, next_dataset_id=155): The following 2 families failed missing samples: - 2: Missing samples: {'HG00732', 'HG00733'} - 3: Missing samples: {'NA20870'}""", + ), + mock.call( + 'seqr_loading_notifications', + f"""Encountered the following errors loading Test Reprocessed Project: + +The following 1 families failed sex check: +- 11: Sample NA20870 has pedigree sex M but imputed sex F{self.SKIPPED_PDO_MESSAGE}""", ), mock.call( 'seqr_loading_notifications', @@ -767,6 +775,7 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): *Run ID:* manual__2025-01-14 *Validation Errors:* ['Missing the following expected contigs:chr17']""") SLACK_VALIDATION_MESSAGE = '' + SKIPPED_PDO_MESSAGE = '' def setUp(self): patcher = mock.patch('seqr.views.utils.export_utils.os.makedirs') @@ -878,6 +887,7 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe - Missing the following expected contigs:chr17 The following users have been notified: test_user_manager@test.com""") SLACK_VALIDATION_MESSAGE = '\nSee more at https://storage.cloud.google.com/seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json' + SKIPPED_PDO_MESSAGE = '\n\nSkipped samples in this project have been moved to PDO-1234_sr' def setUp(self): patcher = mock.patch('seqr.utils.file_utils.subprocess.Popen') From 7dfe8a27661c017968e87f2c63f1949108889994 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 12:52:57 -0400 Subject: [PATCH 057/155] update qc coverage --- .../management/tests/update_individuals_sample_qc_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/seqr/management/tests/update_individuals_sample_qc_tests.py b/seqr/management/tests/update_individuals_sample_qc_tests.py index 389bf402f1..4c6e0e0562 100644 --- a/seqr/management/tests/update_individuals_sample_qc_tests.py +++ b/seqr/management/tests/update_individuals_sample_qc_tests.py @@ -13,9 +13,12 @@ 'family_samples': { 'F000011_11': ['NA20885'], 'F000012_12': ['NA20888', 'NA20889'], - 'F000014_14': ['NA21234'], }, - 'failed_family_samples': {}, + 'failed_family_samples': { + 'relatedness_check': { + 'F000014_14': {'reasons': ['NA21234 failed']}, + }, + }, 'relatedness_check_file_path': '', 'sample_qc': { 'NA20885': { From 8348e3e2e2c03cc6d729eaa65fd3ac8a7df4bf89 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 12:58:36 -0400 Subject: [PATCH 058/155] clean up --- seqr/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/seqr/models.py b/seqr/models.py index 149dd30655..a21a0278ed 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -1150,9 +1150,7 @@ def bulk_create(cls, user, new_models, **kwargs): return models @classmethod - def bulk_delete(cls, user, queryset=None, **filter_kwargs): - if queryset is None: - queryset = cls.objects.filter(**filter_kwargs) + def bulk_delete(cls, user, queryset): cls.log_model_no_guid_bulk_update(queryset, user, 'delete') return queryset.delete() From 6dc86d97d9d472cff6dfd0872eb9efb3fc28473c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 13:24:24 -0400 Subject: [PATCH 059/155] test error handling for failed email send --- seqr/views/apis/data_manager_api_tests.py | 52 ++++++++++++++--------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index b40b2a37d6..63261b2805 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -898,6 +898,7 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema # Test uploading new data self.reset_logs() mock_send_email.reset_mock() + mock_send_email.side_effect = Exception('Email server down') mock_subprocess.return_value.stdout = self._join_data(PHENOTYPE_PRIORITIZATION_HEADER + UPDATE_LIRICAL_DATA) mock_random.randint.side_effect = [177442291, 215071655] response = self.client.post(url, content_type='application/json', data=json.dumps(request_body)) @@ -919,33 +920,44 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema 'entityIds': ['PP177442291_na19678ensg0000010', 'PP215071655_na19678ensg0000010'], }}), ]) + email_body = 'data for 2 Lirical sample(s)' + self.assert_json_logs(user=None, offset=4, expected=[ + ('Error sending project email for R0001_1kg: Email server down', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + 'detail': { + 'to': ['test_user_manager@test.com'], + 'subject': 'New Lirical data available in seqr', + 'email_body': self._expected_email_body(email_body=email_body), + }, + }), + ]) saved_data = _get_json_for_models(PhenotypePrioritization.objects.filter(tool='lirical').order_by('id'), nested_fields=[{'fields': ('individual', 'guid'), 'key': 'individualGuid'}]) self.assertListEqual(saved_data, EXPECTED_UPDATED_LIRICAL_DATA) self._assert_expected_notifications(mock_send_email, [ - {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': 'data for 2 Lirical sample(s)'}, + {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': email_body}, ]) @staticmethod - def _assert_expected_notifications(mock_send_email, expected_notifs: list[dict]): - calls = [] - for notif_dict in expected_notifs: - project_guid = notif_dict.get('project_guid', PROJECT_GUID) - project_name = notif_dict.get('project_name', '1kg project nåme with uniçøde') - url = f'https://test-seqr.org/project/{project_guid}/project_page' - project_link = f'{project_name}' - expected_email_body = ( - f'Dear seqr user,\n\nThis is to notify you that {notif_dict["email_body"]} ' - f'has been loaded in seqr project {project_link}\n\nAll the best,\nThe seqr team' - ) - calls.append( - mock.call( - email_body=expected_email_body, - subject=f'New {notif_dict["data_type"]} data available in seqr', - to=['test_user_manager@test.com'], - process_message=_set_bulk_notification_stream, - ) - ) + def _expected_email_body(email_body='', project_guid=PROJECT_GUID, project_name='1kg project nåme with uniçøde', **kwargs): + url = f'https://test-seqr.org/project/{project_guid}/project_page' + project_link = f'{project_name}' + return ( + f'Dear seqr user,\n\nThis is to notify you that {email_body} ' + f'has been loaded in seqr project {project_link}\n\nAll the best,\nThe seqr team' + ) + + @classmethod + def _assert_expected_notifications(cls, mock_send_email, expected_notifs: list[dict]): + calls = [ + mock.call( + email_body=cls._expected_email_body(**notif_dict), + subject=f'New {notif_dict["data_type"]} data available in seqr', + to=['test_user_manager@test.com'], + process_message=_set_bulk_notification_stream, + ) for notif_dict in expected_notifs + ] mock_send_email.assert_has_calls(calls) def test_loading_vcfs(self): From 2102647b552f696bc0449153a1bf8005e2309400 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 13:51:39 -0400 Subject: [PATCH 060/155] dont over mock social auth logging --- seqr/utils/social_auth_pipeline_tests.py | 56 ++++++++++++------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/seqr/utils/social_auth_pipeline_tests.py b/seqr/utils/social_auth_pipeline_tests.py index 9208004d85..185e709148 100644 --- a/seqr/utils/social_auth_pipeline_tests.py +++ b/seqr/utils/social_auth_pipeline_tests.py @@ -1,67 +1,65 @@ import mock import responses -from unittest import TestCase from social_core.backends.google import GoogleOAuth2 from seqr.utils.social_auth_pipeline import validate_anvil_registration, validate_user_exist, log_signed_in -from seqr.views.utils.test_utils import TEST_TERRA_API_ROOT_URL, REGISTER_RESPONSE +from seqr.views.utils.test_utils import AuthenticationTestCase, TEST_TERRA_API_ROOT_URL, REGISTER_RESPONSE @mock.patch('seqr.views.utils.terra_api_utils.TERRA_API_ROOT_URL', TEST_TERRA_API_ROOT_URL) -class SocialAuthPipelineTest(TestCase): +class SocialAuthPipelineTest(AuthenticationTestCase): + + fixtures = ['users'] @responses.activate - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_validate_anvil_registration(self, mock_logger): + def test_validate_anvil_registration(self): url = TEST_TERRA_API_ROOT_URL + 'register' responses.add(responses.GET, url, status=404) r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) # nosec - mock_logger.warning.assert_called_with( - 'User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', - extra={'user_email': 'test@seqr.org'} - ) + self.assert_json_logs(None, [ + ('User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', { + 'severity': 'WARNING', 'user': 'test@seqr.org', + }) + ]) self.assertEqual(r.url, '/login/error/anvil_registration') - self.assertEqual(len(mock_logger.method_calls), 1) backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) # nosec self.assertEqual(r.url, '/login/error/anvil_registration?next=%2Ffoo%2Fbar') - mock_logger.reset_mock() + self.reset_logs() responses.replace(responses.GET, url, status=200, body=REGISTER_RESPONSE) r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) - mock_logger.warning.assert_not_called() + self.assert_json_logs(None, [('GET https://terra.api/register 200 127', None)]) self.assertIsNone(r) - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_validate_user_exist(self, mock_logger): + def test_validate_user_exist(self): validate_user_exist(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, user='test') - self.assertEqual(len(mock_logger.method_calls), 0) + self.assert_no_logs() r = validate_user_exist(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}) - mock_logger.warning.assert_called_with( - 'Google user test_user_manager@test.com is trying to login without an existing seqr account (google-oauth2).', - extra={'user_email': 'test_user_manager@test.com'}) + self.assert_json_logs(None, [ + ('Google user test_user_manager@test.com is trying to login without an existing seqr account (google-oauth2).', { + 'severity': 'WARNING', 'user': 'test_user_manager@test.com', + }) + ]) self.assertEqual(r.url, '/login/error/no_account') - self.assertEqual(len(mock_logger.method_calls), 1) backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') r = validate_user_exist(backend, {'email': 'test_user_manager@test.com'}) self.assertEqual(r.url, '/login/error/no_account?next=%2Ffoo%2Fbar') - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_log_signed_in(self, mock_logger): + def test_log_signed_in(self): log_signed_in(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, user='test') - mock_logger.info.assert_called_with('Logged in test_user_manager@test.com (google-oauth2)', - extra={'user_email': 'test_user_manager@test.com'}) - self.assertEqual(len(mock_logger.method_calls), 1) + self.assert_json_logs(None, [ + ('Logged in test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}) + ]) - mock_logger.reset_mock() + self.reset_logs() log_signed_in(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, is_new=True, user='test') - mock_logger.info.assert_has_calls([ - mock.call('Logged in test_user_manager@test.com (google-oauth2)', extra={'user_email': 'test_user_manager@test.com'}), - mock.call('Created user test_user_manager@test.com (google-oauth2)', extra={'user_email': 'test_user_manager@test.com'}), + self.assert_json_logs(None, [ + ('Logged in test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}), + ('Created user test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}) ]) - self.assertEqual(len(mock_logger.method_calls), 2) From ed7cf133ad2439596f81ec61680cf5f77cdb3218 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 13:52:35 -0400 Subject: [PATCH 061/155] better mocking --- seqr/utils/social_auth_pipeline_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/utils/social_auth_pipeline_tests.py b/seqr/utils/social_auth_pipeline_tests.py index 185e709148..fba3e39be0 100644 --- a/seqr/utils/social_auth_pipeline_tests.py +++ b/seqr/utils/social_auth_pipeline_tests.py @@ -31,7 +31,7 @@ def test_validate_anvil_registration(self): self.reset_logs() responses.replace(responses.GET, url, status=200, body=REGISTER_RESPONSE) r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) - self.assert_json_logs(None, [('GET https://terra.api/register 200 127', None)]) + self.assert_json_logs(None, [(f'GET {url} 200 127', None)]) self.assertIsNone(r) def test_validate_user_exist(self): From 1ec822712cae41894939ae621083062d34395a12 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 14:14:51 -0400 Subject: [PATCH 062/155] test success validating other data types --- seqr/views/apis/data_manager_api_tests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 63261b2805..599de32256 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1048,6 +1048,12 @@ def test_validate_callset(self): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) + self._add_file_iter([]) + response = self.client.post(url, content_type='application/json', data=json.dumps({ + **self.REQUEST_BODY, 'filePath': f'{self.CALLSET_DIR}/mito_calls.mt', 'datasetType': 'MITO', + })) + self._assert_expected_validate_mito_response(response) + @mock.patch('seqr.views.utils.permissions_utils.INTERNAL_NAMESPACES', ['my-seqr-billing', 'ext-data']) @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://seqr.broadinstitute.org/') @responses.activate @@ -1440,6 +1446,13 @@ def _test_update_rna_seq(self, data_type, *args, **kwargs): self.assertEqual(response.status_code, 500) self.assertDictEqual(response.json(), {'error': 'Airtable is not configured'}) + def _assert_expected_validate_mito_response(self, response): + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), { + 'errors': ['Invalid VCF file format - file path must end with .vcf or .vcf.gz or .vcf.bgz'], + 'warnings': None, + }) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') class AnvilDataManagerAPITest(AnvilAuthenticationTestCase, DataManagerAPITest): @@ -1753,3 +1766,7 @@ def _assert_expected_airtable_errors(self, url): self.assertDictEqual(response.json(), { 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', }) + + def _assert_expected_validate_mito_response(self, response): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'vcfSamples': None}) From 7af90dab1f9ed1b8764d5ab5fc3a681c0629db25 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 14:37:41 -0400 Subject: [PATCH 063/155] test successful wait for service account grant --- seqr/views/apis/anvil_workspace_api_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 70266db8a4..2c444a7c6f 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -256,6 +256,10 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service mock_has_service_account.return_value = False response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) self.assertEqual(response.status_code, 400) + mock_logger.info.assert_called_with( + f'Added service account for {TEST_WORKSPACE_NAMESPACE}/{TEST_NO_PROJECT_WORKSPACE_NAME}, waiting for access to grant', + self.manager_user, + ) self.assertEqual(response.json()['error'], 'Failed to grant seqr service account access to the workspace') mock_has_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) @@ -264,6 +268,7 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service # Test valid operation mock_time.reset_mock() + mock_logger.reset_mock() mock_has_service_account.reset_mock() mock_add_service_account.return_value = False response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) @@ -273,6 +278,17 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service TEST_NO_PROJECT_WORKSPACE_NAME) mock_has_service_account.assert_not_called() mock_time.sleep.assert_not_called() + mock_logger.assert_not_called() + + mock_time.reset_mock() + mock_add_service_account.return_value = True + mock_has_service_account.return_value = True + response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'success': True}) + mock_add_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) + mock_has_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) + mock_time.sleep.assert_called_with(3) mock_logger.info.assert_called_with( f'Added service account for {TEST_WORKSPACE_NAMESPACE}/{TEST_NO_PROJECT_WORKSPACE_NAME}, waiting for access to grant', self.manager_user, From 67d91ade0765cc924da3e3e479d64c32623f3f50 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 15:53:24 -0400 Subject: [PATCH 064/155] add edge case test --- seqr/views/apis/data_manager_api_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 599de32256..e5b397a934 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1651,18 +1651,22 @@ def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_m required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",SeqrIDWithMismatch='NA21987'", ) + responses.replace(responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200) responses.calls.reset() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), { 'warnings': None, 'errors': [ + 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', 'The following families have previously loaded samples absent from airtable\nFamily fam14: NA21234, NA21654', + 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', 'The following samples are included in airtable but are missing from the VCF: NA21987', ], }) self.assertEqual(len(responses.calls), 2) self._assert_expected_airtable_call(required_sample_field='SV_CallsetPath', project_guid='R0004_non_analyst_project') + responses.replace(responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', json=AIRTABLE_SAMPLE_RECORDS, status=200) def _test_load_single_project(self, mock_open, mock_gzip_open, mock_mkdir, response, *args, url=None, body=None, **kwargs): super()._test_load_single_project(mock_open, mock_gzip_open, mock_mkdir, response, url, body) From 2d7911abeca894ebe188333be8f670de14fdc810 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 16:02:12 -0400 Subject: [PATCH 065/155] update fixture --- seqr/fixtures/1kg_project.json | 1 + 1 file changed, 1 insertion(+) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 4761736bdf..1f3cf3c552 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1472,6 +1472,7 @@ "last_modified_date": "2017-03-13T09:07:49.744Z", "individual": 7, "sample_type": "alignment", + "index_file_path": "gs://readviz/NA20870.cram.crai", "file_path": "gs://readviz/NA20870.cram" } }, From 975e2300e28b5c03c29929a98cb532f3e27a51bd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 16:21:41 -0400 Subject: [PATCH 066/155] gregor paternal id mapping --- seqr/views/apis/individual_api_tests.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index dc0e8f248b..8edd779db7 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1072,8 +1072,12 @@ def test_individuals_metadata_hpo_term_number_table_handler(self): def _set_metadata_file_iter(self, genetic_findings_table): self.gs_files.update({ f'{file_name}.tsv': iter(['\t'.join(row).encode() for row in file]) for file_name, file in [ - ('experiment_dna_short_read', EXPERIMENT_TABLE), - ('experiment', EXPERIMENT_LOOKUP_TABLE), + ('experiment_dna_short_read', EXPERIMENT_TABLE + [[ + 'Broad_exome_HG00732', 'Broad_SM-JDBTM', 'Broad_HG00732_1', 'Kapa HyperPrep', '151', 'exome', '', '', '2022-08-15', '385', 'NovaSeq', '', + ]]), + ('experiment', EXPERIMENT_LOOKUP_TABLE + [[ + 'experiment_dna_short_read.Broad_exome_HG00732', 'experiment_dna_short_read', 'Broad_exome_HG00732', 'Broad_HG00732', + ]]), ('participant', LOAD_PARTICIPANT_TABLE), ('phenotype', PHENOTYPE_TABLE), ('genetic_findings', genetic_findings_table), @@ -1171,8 +1175,8 @@ def test_import_gregor_metadata(self): 'mother__individual_id', 'father__individual_id', 'features', 'absent_features', 'case_review_status', ) self.assertDictEqual(individual_db_data[0], { - 'individual_id': 'Broad_HG00732', - 'display_name': '', + 'individual_id': 'Broad_HG00732_1', + 'display_name': 'Broad_HG00732', 'family__guid': new_family_guid, 'affected': 'N', 'sex': 'M', @@ -1220,7 +1224,7 @@ def test_import_gregor_metadata(self): 'sex': 'F', 'proband_relationship': 'S', 'mother__individual_id': None, - 'father__individual_id': 'Broad_HG00732', + 'father__individual_id': 'Broad_HG00732_1', 'population': 'AMR', 'features': [{'id': 'HP:0011675'}], 'absent_features': [{'id': 'HP:0002017'}], From e91de03c7d20f5267cb0aa8dd1bb87f13bec12d9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 16:41:36 -0400 Subject: [PATCH 067/155] remove phenotips json upload support --- seqr/views/apis/individual_api.py | 171 ++++++++---------------- seqr/views/apis/individual_api_tests.py | 13 +- 2 files changed, 62 insertions(+), 122 deletions(-) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 75846a1bb1..40654e768f 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -359,8 +359,6 @@ def _update_and_parse_individuals_and_families(project, individual_records, user AR_DSPERM_COL = 'ar_donorsperm' def _bool_value(val): - if isinstance(val, bool): - return val if val.lower() == 'true': return True elif val.lower() == 'false': @@ -368,8 +366,6 @@ def _bool_value(val): raise ValueError def _array_value(val): - if isinstance(val, list): - return val return [o.strip() for o in val.split(',')] def _gene_value(val): @@ -380,8 +376,6 @@ def _gene_value(val): return gene def _gene_list_value(val): - if isinstance(val, list): - return val seperator_escaped_val = ''.join(m.replace(',', ';') if not m.startswith('(') else m for m in re.split('(\([^)]+\))', val)) return [_gene_value(o) for o in seperator_escaped_val.split(';')] @@ -410,60 +404,6 @@ def _gene_list_value(val): CANDIDATE_GENES_COL: _gene_list_value, } -def _get_year(val): - return datetime.strptime(val, '%Y-%m-%d').year - -def _nested_val(nested_key): - return lambda val: val.get(nested_key) - -def _get_phenotips_features(observed): - def get_observed_features(features): - return [{'id': feature['id']} for feature in features if feature['observed'] == observed] - return get_observed_features - -PHENOTIPS_JSON_FIELD_MAP = { - 'family_id': [(FAMILY_ID_COL, None)], - 'external_id': [(INDIVIDUAL_ID_COL, None)], - 'features': [ - (FEATURES_COL, _get_phenotips_features('yes')), - (ABSENT_FEATURES_COL, _get_phenotips_features('no')), - ], - 'date_of_birth': [(BIRTH_COL, _get_year)], - 'date_of_death': [(DEATH_COL, _get_year)], - 'global_age_of_onset': [(ONSET_AGE_COL, lambda val: val[0]['label'])], - 'family_history': [ - (CONSANGUINITY_COL, _nested_val('consanguinity')), - (AFFECTED_REL_COL, _nested_val('affectedRelatives')), - ], - 'global_mode_of_inheritance': [(EXP_INHERITANCE_COL, lambda val: [o['label'] for o in val])], - 'prenatal_perinatal_history': [ - (AR_FM_COL, _nested_val('assistedReproduction_fertilityMeds')), - (AR_IUI_COL, _nested_val('assistedReproduction_iui')), - (AR_IVF_COL, _nested_val('ivf')), - (AR_ICSI_COL, _nested_val('icsi')), - (AR_SURROGACY_COL, _nested_val('assistedReproduction_surrogacy')), - (AR_DEGG_COL, _nested_val('assistedReproduction_donoregg')), - (AR_DSPERM_COL, _nested_val('assistedReproduction_donorsperm')), - ], - 'ethnicity': [ - (MAT_ETHNICITY_COL, _nested_val('maternal_ethnicity')), - (PAT_ETHNICITY_COL, _nested_val('paternal_ethnicity')), - ], - 'disorders': [(DISORDERS_COL, lambda val: [int(d['id'].lstrip('MIM:')) for d in val])], - 'genes': [(CANDIDATE_GENES_COL, None)], - 'rejectedGenes': [(REJECTED_GENES_COL, None)], -} - -def _parse_phenotips_record(row): - record = {} - for k, formatters in PHENOTIPS_JSON_FIELD_MAP.items(): - val = row.get(k) - if val: - for col, formatter in formatters: - field_val = formatter(val) if formatter else val - if field_val is not None: - record[col] = field_val - return record @login_and_policies_required def receive_individuals_metadata_handler(request, project_guid): @@ -500,64 +440,61 @@ def process_records(json_records, filename=''): def _process_hpo_records(records, filename, project, user): - if filename.endswith('.json'): - row_dicts = [_parse_phenotips_record(record) for record in records] - else: - column_map = {} - for i, field in enumerate(records[0]): - key = field.lower() - if re.match("hpo.*present", key): - column_map[FEATURES_COL] = i - elif re.match("hpo.*absent", key): - column_map[ABSENT_FEATURES_COL] = i - elif re.match("hp.*number*", key): - if not HPO_TERM_NUMBER_COL in column_map: - column_map[HPO_TERM_NUMBER_COL] = [] - column_map[HPO_TERM_NUMBER_COL].append(i) - elif 'family' in key or 'pedigree' in key: - column_map[FAMILY_ID_COL] = i + column_map = {} + for i, field in enumerate(records[0]): + key = field.lower() + if re.match("hpo.*present", key): + column_map[FEATURES_COL] = i + elif re.match("hpo.*absent", key): + column_map[ABSENT_FEATURES_COL] = i + elif re.match("hp.*number*", key): + if not HPO_TERM_NUMBER_COL in column_map: + column_map[HPO_TERM_NUMBER_COL] = [] + column_map[HPO_TERM_NUMBER_COL].append(i) + elif 'family' in key or 'pedigree' in key: + column_map[FAMILY_ID_COL] = i + else: + col_key = next((col for col, text in [ + (NOTES_COL, 'notes'), (INDIVIDUAL_ID_COL, 'individual'), (AFFECTED_REL_COL, 'affected relative'), + (AFFECTED_FEATURE_COL, 'affected'), (BIRTH_COL, 'birth'), (DEATH_COL, 'death'), + (ONSET_AGE_COL, 'onset'), (AR_ICSI_COL, 'relative'), + (CONSANGUINITY_COL, 'consanguinity'), (EXP_INHERITANCE_COL, 'inheritance'), (AR_FM_COL, 'fertility'), + (AR_IUI_COL, 'intrauterine'), (AR_IVF_COL, 'in vitro'), (AR_ICSI_COL, 'cytoplasmic'), + (AR_SURROGACY_COL, 'surrogacy'), (AR_DEGG_COL, 'donor egg'), (AR_DSPERM_COL, 'donor sperm'), + (MAT_ETHNICITY_COL, 'maternal ancestry'), (PAT_ETHNICITY_COL, 'paternal ancestry'), + (DISORDERS_COL, 'disorders'), (REJECTED_GENES_COL, 'tested genes'), + (CANDIDATE_GENES_COL, 'candidate genes'), (ASSIGNED_ANALYST_COL, 'assigned analyst'), + ] if text in key), None) + if col_key: + column_map[col_key] = i + + if INDIVIDUAL_ID_COL not in column_map: + raise ValueError('Invalid header, missing individual id column') + + row_dicts = [{column: row[index] if isinstance(index, int) else next((row[i] for i in index if row[i]), None) + for column, index in column_map.items()} for row in records[1:]] + + if FEATURES_COL in column_map or ABSENT_FEATURES_COL in column_map: + for row in row_dicts: + row[FEATURES_COL] = parse_hpo_terms(row.get(FEATURES_COL)) + row[ABSENT_FEATURES_COL] = parse_hpo_terms(row.get(ABSENT_FEATURES_COL)) + + elif HPO_TERM_NUMBER_COL in column_map: + aggregate_rows = defaultdict(lambda: {FEATURES_COL: set(), ABSENT_FEATURES_COL: set()}) + for row in row_dicts: + column = ABSENT_FEATURES_COL if row.pop(AFFECTED_FEATURE_COL) == 'no' else FEATURES_COL + aggregate_entry = aggregate_rows[(row.get(FAMILY_ID_COL), row.get(INDIVIDUAL_ID_COL))] + term = row.pop(HPO_TERM_NUMBER_COL, None) + if term: + aggregate_entry[column].add(term.strip()) else: - col_key = next((col for col, text in [ - (NOTES_COL, 'notes'), (INDIVIDUAL_ID_COL, 'individual'), (AFFECTED_REL_COL, 'affected relative'), - (AFFECTED_FEATURE_COL, 'affected'), (BIRTH_COL, 'birth'), (DEATH_COL, 'death'), - (ONSET_AGE_COL, 'onset'), (AR_ICSI_COL, 'relative'), - (CONSANGUINITY_COL, 'consanguinity'), (EXP_INHERITANCE_COL, 'inheritance'), (AR_FM_COL, 'fertility'), - (AR_IUI_COL, 'intrauterine'), (AR_IVF_COL, 'in vitro'), (AR_ICSI_COL, 'cytoplasmic'), - (AR_SURROGACY_COL, 'surrogacy'), (AR_DEGG_COL, 'donor egg'), (AR_DSPERM_COL, 'donor sperm'), - (MAT_ETHNICITY_COL, 'maternal ancestry'), (PAT_ETHNICITY_COL, 'paternal ancestry'), - (DISORDERS_COL, 'disorders'), (REJECTED_GENES_COL, 'tested genes'), - (CANDIDATE_GENES_COL, 'candidate genes'), (ASSIGNED_ANALYST_COL, 'assigned analyst'), - ] if text in key), None) - if col_key: - column_map[col_key] = i - - if INDIVIDUAL_ID_COL not in column_map: - raise ValueError('Invalid header, missing individual id column') - - row_dicts = [{column: row[index] if isinstance(index, int) else next((row[i] for i in index if row[i]), None) - for column, index in column_map.items()} for row in records[1:]] - - if FEATURES_COL in column_map or ABSENT_FEATURES_COL in column_map: - for row in row_dicts: - row[FEATURES_COL] = parse_hpo_terms(row.get(FEATURES_COL)) - row[ABSENT_FEATURES_COL] = parse_hpo_terms(row.get(ABSENT_FEATURES_COL)) - - elif HPO_TERM_NUMBER_COL in column_map: - aggregate_rows = defaultdict(lambda: {FEATURES_COL: set(), ABSENT_FEATURES_COL: set()}) - for row in row_dicts: - column = ABSENT_FEATURES_COL if row.pop(AFFECTED_FEATURE_COL) == 'no' else FEATURES_COL - aggregate_entry = aggregate_rows[(row.get(FAMILY_ID_COL), row.get(INDIVIDUAL_ID_COL))] - term = row.pop(HPO_TERM_NUMBER_COL, None) - if term: - aggregate_entry[column].add(term.strip()) - else: - aggregate_entry[column] = set() - aggregate_entry.update({k: v for k, v in row.items() if v}) - - row_dicts = [ - {**entry, **{col: [{'id': feature} for feature in entry[col]] for col in [FEATURES_COL, ABSENT_FEATURES_COL]}} - for entry in aggregate_rows.values() - ] + aggregate_entry[column] = set() + aggregate_entry.update({k: v for k, v in row.items() if v}) + + row_dicts = [ + {**entry, **{col: [{'id': feature} for feature in entry[col]] for col in [FEATURES_COL, ABSENT_FEATURES_COL]}} + for entry in aggregate_rows.values() + ] return _parse_individual_hpo_terms(row_dicts, project, user) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 8edd779db7..a3841adde0 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1019,11 +1019,11 @@ def test_individuals_metadata_table_handler(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid header, missing individual id column'], 'warnings': []}) - header = 'family_id,individual_id,hpo_term_present,hpo_term_absent,sex,birth year,other affected relatives,onset age,expected inheritance,maternal ancestry,candidate genes,assigned analyst' + header = 'individual_id,hpo_term_present,hpo_term_absent,sex,birth year,other affected relatives,onset age,expected inheritance,maternal ancestry,candidate genes,assigned analyst' rows = [ - '1,NA19678,,,,,no,infant,recessive,,,not_an_email', - '1,NA19679,HP:0100258 (Preaxial polydactyly),,,,,,,,,test_user_no_access@test.com', - '1,HG00731,HP:0002017,HP:0012469 (Infantile spasms);HP:0011675 (Arrhythmia);HP:0011675 (Arrhythmia),,,,,,,,,', + 'NA19678,,,,,no,infant,recessive,,,not_an_email', + 'NA19679,HP:0100258 (Preaxial polydactyly),,,,,,,,,test_user_no_access@test.com', + 'HG007311,HP:0002017,HP:0012469 (Infantile spasms);HP:0011675 (Arrhythmia);HP:0011675 (Arrhythmia),,,,,,,,,', ] f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) @@ -1038,12 +1038,15 @@ def test_individuals_metadata_table_handler(self): 'The following invalid values for "onset_age" will not be added: infant (NA19678)', 'The following invalid values for "expected_inheritance" will not be added: recessive (NA19678)', 'The following invalid values for "assigned_analyst" will not be added: not_an_email (NA19678); test_user_no_access@test.com (NA19679)', - 'Unable to find matching ids for 1 individuals. The following entries will not be updated: HG00731', + 'Unable to find matching ids for 1 individuals. The following entries will not be updated: HG007311', 'No changes detected for 2 individuals. The following entries will not be updated: NA19678, NA19679', ]}) # send valid request + header = f'family_id,{header}' rows[0] = '1,NA19678,,,,,false,,,,,' + rows[1] = f'1,{rows[1]}' + rows[2] = rows[2].replace('HG007311', '1,HG00731') rows.append('1,NA19675_1,HP:0002017,"HP:0012469 (Infantile spasms);HP:0004322 (Short stature, severe)",F,2000,True,Juvenile onset,"Autosomal dominant inheritance, Sporadic","Finnish, Irish","IKBKAP -- (multiple panels, no confirm), EHBP1L1",test_user_collaborator@test.com') f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) From 23a86eb45c8bd6c80f61e58a22f3c22d6f396f13 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 16:47:33 -0400 Subject: [PATCH 068/155] fix project test mock --- seqr/views/apis/project_api_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 0b183aad3f..bc13166699 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -1068,6 +1068,7 @@ def _check_created_project_groups(self, project): self.assertSetEqual(set(project.can_edit_group.user_set.all()), {self.pm_user}) self.assertSetEqual(set(project.can_view_group.user_set.all()), {self.pm_user}) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') def test_update_project_workspace(self): url = reverse(update_project_workspace, args=[PROJECT_GUID]) # For non-AnVIL seqr, updating workspace should always fail From 6bc9269897cb703b9a26e118a910ea466b5a9eb1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 May 2026 16:55:11 -0400 Subject: [PATCH 069/155] test edge case --- seqr/views/apis/summary_data_api_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 8677bda356..d3a1b909c4 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -616,6 +616,11 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc {'4': {'name': 'De-Novo', 'date': '2023-12-05'}, 'support': {'name': 'High in Silico Scores', 'date': '2023-12-05'}}, ) + # Test reloading skips unchanged tags + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'info': ['Loaded 0 new and 1 updated AIP tags for 2 families (skipped 2 unchanged tags)']}) + self.check_no_analyst_no_access(url) def _assert_expected_new_saved_variant(self, new_saved_variant): From 5aa8754b6205396a426aeee726c1ea1017d48965 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 11:26:06 -0400 Subject: [PATCH 070/155] report edge case test --- seqr/views/apis/report_api_tests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 60d214a668..032a662405 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -824,7 +824,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo match=[responses.matchers.query_param_matcher({'fields[]': 'SMID'}, strict_match=False)] ) responses.add( - responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json=AIRTABLE_GREGOR_RECORDS, + responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json={'records': AIRTABLE_GREGOR_RECORDS['records'][:2]+AIRTABLE_GREGOR_RECORDS['records'][3:]}, status=200) responses.add(responses.GET, MOCK_DATA_MODEL_URL, status=404) @@ -856,7 +856,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo recommended_warnings = [ 'The following entries are missing RNA airtable data: NA19675', - 'The following entries are missing WES airtable data: NA19675, NA19679', + 'The following entries are missing WES airtable data: NA19675', 'The following entries have WGS airtable data but do not have equivalent loaded data in seqr, so airtable data is omitted: NA19675, NA20888, VCGS_FAM203_621', 'The following entries are missing recommended "recontactable" in the "participant" table: Broad_HG00731, Broad_HG00732, Broad_HG00733, Broad_NA19678, Broad_NA20870, Broad_NA20872, Broad_NA20874, Broad_NA20875, Broad_NA20876, Broad_NA20881', 'The following entries are missing recommended "reported_race" in the "participant" table: Broad_HG00733, Broad_NA19678, Broad_NA19679, Broad_NA20870, Broad_NA20872, Broad_NA20874, Broad_NA20875, Broad_NA20876, Broad_NA20881, Broad_NA20888', @@ -875,6 +875,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo f'No data model found for "{file}" table' for file in reversed(EXPECTED_GREGOR_FILES) if file not in INVALID_MODEL_TABLES ] + [ missing_participant_error, + 'The following entries are missing airtable metadata: NA19679', 'The following tables are required in the data model but absent from the reports: subject, dna_read_data_set', ] + [ 'The following entries are missing required "prior_testing" in the "participant" table: Broad_HG00731, Broad_HG00732', @@ -885,17 +886,23 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo 'The following entries are missing required "mean_coverage" (from Airtable) in the "aligned_dna_short_read" table: Broad_exome_VCGS_FAM203_621_D2_1', 'The following entries have non-unique values for "alignment_software" (from Airtable) in the "aligned_dna_short_read" table: BWA-MEM-2.3 (Broad_exome_NA20888_1, Broad_exome_VCGS_FAM203_621_D2_1)', 'The following entries have invalid values for "analysis_details" (from Airtable) in the "aligned_dna_short_read" table. Allowed values are a google bucket path starting with gs://. Invalid values: Broad_exome_VCGS_FAM203_621_D2_1 (DOI:10.5281/zenodo.4469317)', - 'The following entries have invalid values for "date_data_generation" (from Airtable) in the "experiment_rna_short_read" table. Allowed values have data type float. Invalid values: NA19679 (2023-02-11)', 'The following entries are missing required "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_NA19675_1_21_3343353', 'The following entries have non-unique values for "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_exome_VCGS_FAM203_621_D2 (Broad_HG00731_19_1912632, Broad_HG00731_1_248367227)', ] self.assertListEqual(response.json()['errors'], validation_errors) + responses.replace( + responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json=AIRTABLE_GREGOR_RECORDS, + ) mock_open.reset_mock() response = self.client.post( url, content_type='application/json', data=json.dumps({**body, 'overrideValidation': True}) ) self.assertEqual(response.status_code, 200) + validation_errors.pop(8) + validation_errors.insert(17, 'The following entries have invalid values for "date_data_generation" (from Airtable) in the "experiment_rna_short_read" table. Allowed values have data type float. Invalid values: NA19679 (2023-02-11)') + recommended_warnings[1] += ', NA19679' + validation_warnings[4] = recommended_warnings[1] expected_response = { 'info': ['Successfully validated and uploaded Gregor Report for 9 families'], 'warnings': validation_errors + validation_warnings, From 2ddb7a20fcc69f281c2166d0ac9de6fb035f1e3a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 11:38:28 -0400 Subject: [PATCH 071/155] cleaner logic --- .../management/commands/reload_clinvar_all_variants.py | 5 ++++- .../management/tests/reload_clinvar_all_variants_tests.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 50c2ac478e..a449e87ded 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -261,7 +261,7 @@ def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenu if existing_version_obj: if existing_version_obj.version == new_version: logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') - return new_version + return None logger.info( f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') # Drop any currently existing variants in the table that may exist due to a # previously failed partial run. Note that we validate that the Postgresql existing version @@ -306,6 +306,9 @@ def handle(self, *args, **options): with gzip.open(tmpfile.name, 'rb') as gzipped_file: new_version = parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) + if not new_version: + return + for model, batch in model_to_batch.items(): if batch: model.objects.using('clickhouse_write').bulk_create(batch) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index c22bff5763..b7a18267d5 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -103,6 +103,8 @@ def test_new_version_already_exists(self, mock_logger, mock_safe_post_to_slack): responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) call_command('reload_clinvar_all_variants') mock_logger.assert_called_with('Clinvar ClickHouse tables already successfully updated to 2025-06-30, gracefully exiting.') + mock_safe_post_to_slack.assert_not_called() + self.assertEqual(DataVersions.objects.get(data_model_name='Clinvar').version, '2025-06-30') @responses.activate def test_parse_variants_all_types(self, mock_logger, mock_safe_post_to_slack): From 20a1dee4db44bca9e85f953b89d008c7e0fb1fe4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 12:58:34 -0400 Subject: [PATCH 072/155] clean up --- seqr/views/utils/airtable_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py index 3d57e5d5f3..4baebea0a2 100644 --- a/seqr/views/utils/airtable_utils.py +++ b/seqr/views/utils/airtable_utils.py @@ -75,8 +75,10 @@ def safe_patch_records(self, record_type, record_or_filters, record_and_filters, and_filters=record_and_filters, page_size=max_records + 1, ) - if not records or len(records) > max_records: - error = f'''Unable to identify Airtable "{record_type}" record to update + except Exception: + records = [] + if not records or len(records) > max_records: + error = f'''Unable to identify Airtable "{record_type}" record to update Record lookup criteria: ``` @@ -88,12 +90,10 @@ def safe_patch_records(self, record_type, record_or_filters, record_and_filters, ``` {json.dumps(update)} ```''' - safe_post_to_slack(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, error) - return + safe_post_to_slack(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, error) + return - self.safe_patch_records_by_id(record_type, list(records.keys()), update, error_detail=error_detail) - except Exception as e: - logger.error(f'Airtable patch "{record_type}" error: {e}', self._user, detail=error_detail) + self.safe_patch_records_by_id(record_type, list(records.keys()), update, error_detail=error_detail) def safe_patch_records_by_id(self, record_type, record_ids, update, error_detail=None): self._safe_bulk_update_records( From 9c98219e6462eeb86e5886921ea09d231b3967a9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 14:26:21 -0400 Subject: [PATCH 073/155] test alerting when fetch for patch fails --- ...eck_for_new_samples_from_pipeline_tests.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 1d98257103..f147e8556d 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -457,17 +457,20 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) self.assertEqual(len(responses.calls), num_calls) - def _test_success_call(self, anvil_email_calls, next_dataset_id=155): + def _test_success_call(self, anvil_email_calls, next_dataset_id=155, fetched_tracking=True): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) self.maxDiff = None + tracking_logs = [ + ('Fetched 2 AnVIL Seqr Loading Requests Tracking records from airtable', None), + ] if fetched_tracking and self.AIRTABLE_LOGS else [] self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), (f'create Dataset D0000{next_dataset_id}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), (f'create Dataset D0000{next_dataset_id+1}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), - ] + self.AIRTABLE_LOGS + [ + ] + self.AIRTABLE_LOGS + tracking_logs + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, 'GRCh38/MITO': [ @@ -739,9 +742,9 @@ def test_command(self): # Test reloading shared annotations is skipped if too many saved variants snv_indel_datasets.delete() - airtable_logs = self.AIRTABLE_LOGS[:-1] + airtable_logs = [] if self.AIRTABLE_LOGS: - airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) + airtable_logs = self.AIRTABLE_LOGS + [('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)] self._test_call(num_runs=2, run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), @@ -850,7 +853,6 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe ('Fetching PDO records 0-1 from airtable', None), ('Fetched 1 PDO records from airtable', None), ('Fetching AnVIL Seqr Loading Requests Tracking records 0-2 from airtable', None), - ('Fetched 2 AnVIL Seqr Loading Requests Tracking records from airtable', None), ] VALIDATION_LOGS = [ '==> gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', @@ -901,14 +903,6 @@ def setUp(self): super().setUp() def _add_responses(self): - responses.add( - responses.GET, - self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), - json={'records': [{'id': 'rec12345', 'fields': {}}, {'id': 'rec67890', 'fields': {}}]}) - responses.add( - responses.GET, - self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), - json={'records': [{'id': 'rec12345', 'fields': {}}]}) responses.add( responses.GET, self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format('R0002_empty'), @@ -933,6 +927,14 @@ def _add_responses(self): @responses.activate def test_command(self, *args, **kwargs): + responses.add( + responses.GET, + self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), + json={'records': [{'id': 'rec12345', 'fields': {}}, {'id': 'rec67890', 'fields': {}}]}) + responses.add( + responses.GET, + self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), + json={'records': [{'id': 'rec12345', 'fields': {}}]}) self._add_responses() super().test_command(*args, **kwargs) @@ -1052,4 +1054,4 @@ def test_loading_delay_command(self): self._test_success_call(self._anvil_email_calls( email_text=ANVIL_ERROR_TEXT_EMAIL_TEMPLATE.format(error='\n'+ANVIL_ERROR_DELAY), email_html=ANVIL_ERROR_HTML_EMAIL_TEMPLATE.format(error='
'+ANVIL_ERROR_DELAY), - ), next_dataset_id=161) \ No newline at end of file + ), next_dataset_id=161, fetched_tracking=False) \ No newline at end of file From be9a14b073717158f6b792b86e74aec05a9e75df Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 15:04:35 -0400 Subject: [PATCH 074/155] pedigee util cleanup --- seqr/views/apis/individual_api_tests.py | 4 +++- seqr/views/utils/pedigree_info_utils.py | 24 +++++++----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index a3841adde0..ae8864c028 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -93,6 +93,7 @@ LOAD_PARTICIPANT_TABLE = deepcopy(PARTICIPANT_TABLE) for row in LOAD_PARTICIPANT_TABLE[4:]: row[7] = row[7].replace('Broad_', '') +LOAD_PARTICIPANT_TABLE[5][11] = 'Niece' LOAD_PARTICIPANT_TABLE[6][15] += '|Asian' LOAD_PARTICIPANT_TABLE[6][17] = '' @@ -1128,6 +1129,7 @@ def test_import_gregor_metadata(self): }) warnings = [ 'Broad_HG00733 is the mother of VCGS_FAM203_621_D2 but is not included', + 'Skipped Invalid proband relationship "Niece" for NA20888 with given gender Male', 'Skipped the following unrecognized HPO terms: HP:0001509', ] self.assertDictEqual(response_json['importStats'], {'gregorMetadata': { @@ -1211,7 +1213,7 @@ def test_import_gregor_metadata(self): 'family__guid': 'F000012_12', 'affected': 'A', 'sex': 'M', - 'proband_relationship': '', + 'proband_relationship': None, 'mother__individual_id': None, 'father__individual_id': None, 'population': 'SAS', diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index 6fd9c94f36..5e1030a3a6 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -173,9 +173,7 @@ def _parse_header_columns(header, allow_id_update, update_features): column = None full_key = key key = key.lower() - if full_key in JsonConstants.JSON_COLUMNS: - column = full_key - elif key == JsonConstants.FAMILY_NOTES_COLUMN.lower(): + if key == JsonConstants.FAMILY_NOTES_COLUMN.lower(): column = JsonConstants.FAMILY_NOTES_COLUMN elif key.startswith("notes"): column = JsonConstants.NOTES_COLUMN @@ -474,8 +472,6 @@ def _set_proband_relationship(json_records): ) if affected_children: affected = affected_children - if not affected: - continue affected = affected[0] relationships = { @@ -770,30 +766,24 @@ def _get_rgp_dsm_family_notes(row): def _get_rgp_dsm_proband_fields(row): DC = DSMConstants - try: + if row[DC.AGE_COLUMN]: age = int(row[DC.AGE_COLUMN]) birth_year = date.today().year - age - except ValueError: - birth_year = None death_year = None if row[DC.DECEASED_COLUMN] == DC.YES: - try: - age = int(row[DC.DECEASED_AGE_COLUMN]) - death_year = birth_year + age - except (ValueError, TypeError): - death_year = 0 + age = int(row[DC.DECEASED_AGE_COLUMN]) + death_year = birth_year + age - try: + onset_age = None + if row[DC.AGE_OF_ONSET_COLUMN]: onset_age_val = int(row[DC.AGE_OF_ONSET_COLUMN]) onset_age = next(age for cutoff, age in [ (2, 'I'), # Infantile onset (13, 'C'), # Childhood onset (20, 'J'), # Juvenile onset - (200, 'A')# Adult onset + (200, 'A') # Adult onset ] if onset_age_val < cutoff) - except (ValueError, TypeError): - onset_age = None affected_relatives = any( row['{}_{}'.format(parent, DC.AFFECTED_KEY)] == DC.YES for parent in [DC.MOTHER, DC.FATHER] From 723c5796eb4ebb8e72266b445bcc5bcf4bb89830 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 16:51:29 -0400 Subject: [PATCH 075/155] first pass new postgres dicts --- ...ariantdict_excludedvariantdict_and_more.py | 71 +++++++++++++++++++ clickhouse_search/models/postgres_dicts.py | 38 ++++++++++ 2 files changed, 109 insertions(+) create mode 100644 clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py new file mode 100644 index 0000000000..3a429dd57c --- /dev/null +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -0,0 +1,71 @@ +# Generated by Django 4.2.27 on 2026-05-14 20:43 + +import clickhouse_backend.models +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0042_delete_annotationsdiskgcnv_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='DiscoveryVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_discovery_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = 'CMG Discovery Tags')", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='ExcludedVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_excluded_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = 'Excluded')", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='IndividualMetadataDict', + fields=[ + ('family_guid', clickhouse_backend.models.StringField(primary_key=True, serialize=False)), + ('sampleId', clickhouse_backend.models.StringField()), + ('restrict_sharing', clickhouse_backend.models.BoolField()), + ('features', clickhouse_backend.models.StringField()), + ('omim_id', clickhouse_backend.models.UInt32Field()), + ('mondo_id', clickhouse_backend.models.StringField()), + ('is_solved', clickhouse_backend.models.BoolField()), + ], + options={ + 'db_table': 'seqrdb_individual_metadata_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key=('family_guid', 'sampleId')), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "select f.guid as family_guid, i.individual_id as sample_id, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index defeeb231d..1181a2736b 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -37,3 +37,41 @@ class Meta: layout = 'HASHED()' postgres_db = 'reference_data' postgres_query = 'SELECT gene_id, id FROM reference_data_geneinfo' + + +class IndividualMetadataDict(Dictionary): + family_guid = models.StringField(primary_key=True) + sampleId = models.StringField() + restrict_sharing = models.BoolField() + features = models.StringField() + omim_id = models.UInt32Field() + mondo_id = models.StringField() + is_solved = models.BoolField() + + class Meta: + db_table = 'seqrdb_individual_metadata_dict' + engine = models.MergeTree(primary_key=('family_guid', 'sampleId')) + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "select f.guid as family_guid, i.individual_id as sample_id, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" + + +class DiscoveryVariantDict(Dictionary): + key = models.UInt32Field(primary_key=True) + family_guids = models.ArrayField(models.StringField()) + + class Meta: + db_table = 'seqrdb_discovery_variant_dict' + engine = models.MergeTree(primary_key='key') + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = 'CMG Discovery Tags')" + + +class ExcludedVariantDict(Dictionary): + key = models.UInt32Field(primary_key=True) + family_guids = models.ArrayField(models.StringField()) + + class Meta: + db_table = 'seqrdb_excluded_variant_dict' + engine = models.MergeTree(primary_key='key') + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = 'Excluded')" From 5581accb456162e86d0a0b6ecf5b2205fb02fcb4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 17:09:52 -0400 Subject: [PATCH 076/155] allow additional genotype fields --- clickhouse_search/all_search_tests.py | 7 +- clickhouse_search/managers.py | 11 +-- ...ariantdict_excludedvariantdict_and_more.py | 71 ------------------- clickhouse_search/models/postgres_dicts.py | 4 +- 4 files changed, 13 insertions(+), 80 deletions(-) delete mode 100644 clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index fc11948cdc..c8d7ea2672 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -12,7 +12,8 @@ from clickhouse_search.models.gt_stats_models import ProjectGtStatsSnvIndel, \ ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, ProjectsToGtStatsMito, ProjectsToGtStatsSv, \ GtStatsDictGRCh37SnvIndel, GtStatsDictSnvIndel, GtStatsDictMito, GtStatsDictSv -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict, DiscoveryVariantDict, \ + ExcludedVariantDict from clickhouse_search.models.reference_data_models import ClinvarMvSnvIndel, ClinvarSearchMvSnvIndel, ClinvarMvMito, \ ClinvarSearchMvMito, ClinvarMvGRCh37SnvIndel, ClinvarSearchMvGRCh37SnvIndel, HgmdMv, HgmdSearchMv, \ DbnsfpSnvIndelMv, DbnsfpSnvIndelDict, EigenMv, EigenDict, SpliceAiMv, SpliceAiDict, GnomadNonCodingConstraintDict, \ @@ -81,8 +82,8 @@ def tearDownClass(cls): @classmethod def setUpTestData(cls): - AffectedDict.reload() - SexDict.reload() + for postgres_dict in [AffectedDict, SexDict, ExcludedVariantDict]: + postgres_dict.reload() for view in [ ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, ProjectsToGtStatsMito, ProjectsToGtStatsSv, ClinvarMvSnvIndel, ClinvarSearchMvSnvIndel, ClinvarMvMito, ClinvarSearchMvMito, ClinvarMvGRCh37SnvIndel, diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index b19f8f2250..ddc16337c8 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1055,7 +1055,7 @@ def _join_annotations(self, entries): def result_values(self, *args, sample_data=None, **kwargs): entries = self._join_annotations(self) - return self._search_call_data(entries, sample_data) + return self._search_call_data(entries, sample_data, **kwargs) def _filter_project_families(self, entries, sample_data): project_guids = sample_data['project_guids'] @@ -1312,7 +1312,7 @@ def annotation_fields(cls, entries): fields.append('seqrPop') return fields - def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, **kwargs): + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, additional_expressions=None, **kwargs): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -1323,7 +1323,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu if skip_entry_fields: entries = entries.annotate(numFamilies=Count('family_guid')) else: - gt_field, gt_expression = self.genotype_expression(sample_data) + gt_field, gt_expression = self.genotype_expression(sample_data, additional_expressions) entries = entries.annotate( familyGuids=ArraySort(ArrayDistinct(GroupArray('family_guid'))), **{gt_field: GroupArrayArray(gt_expression)}, @@ -1363,7 +1363,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu return entries - def genotype_expression(self, sample_data=None): + def genotype_expression(self, sample_data=None, additional_expressions=None): family_samples = defaultdict(list) samples = (sample_data or {}).get('samples') or [] for s in samples: @@ -1378,6 +1378,9 @@ def genotype_expression(self, sample_data=None): genotype_expressions.insert(0, f"map({', '.join(sample_map)})[family_guid][x.sampleId]") output_base_fields.insert(0, ('individualGuid', models.StringField())) output_field_kwargs = {'group_by_key': 'individualGuid', 'flatten_groups': True} + if additional_expressions: + genotype_expressions += list(additional_expressions.keys()) + output_base_fields += list(additional_expressions.values()) return 'genotypes' if samples else 'familyGenotypes', ArrayFilter( ArrayMap( 'calls', diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py deleted file mode 100644 index 3a429dd57c..0000000000 --- a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py +++ /dev/null @@ -1,71 +0,0 @@ -# Generated by Django 4.2.27 on 2026-05-14 20:43 - -import clickhouse_backend.models -from django.db import migrations -import django.db.models.manager - - -class Migration(migrations.Migration): - - dependencies = [ - ('clickhouse_search', '0042_delete_annotationsdiskgcnv_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='DiscoveryVariantDict', - fields=[ - ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), - ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), - ], - options={ - 'db_table': 'seqrdb_discovery_variant_dict', - 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), - 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = 'CMG Discovery Tags')", - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='ExcludedVariantDict', - fields=[ - ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), - ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), - ], - options={ - 'db_table': 'seqrdb_excluded_variant_dict', - 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), - 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = 'Excluded')", - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='IndividualMetadataDict', - fields=[ - ('family_guid', clickhouse_backend.models.StringField(primary_key=True, serialize=False)), - ('sampleId', clickhouse_backend.models.StringField()), - ('restrict_sharing', clickhouse_backend.models.BoolField()), - ('features', clickhouse_backend.models.StringField()), - ('omim_id', clickhouse_backend.models.UInt32Field()), - ('mondo_id', clickhouse_backend.models.StringField()), - ('is_solved', clickhouse_backend.models.BoolField()), - ], - options={ - 'db_table': 'seqrdb_individual_metadata_dict', - 'engine': clickhouse_backend.models.MergeTree(primary_key=('family_guid', 'sampleId')), - 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "select f.guid as family_guid, i.individual_id as sample_id, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - ] diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index 1181a2736b..bcb86d89b0 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -41,7 +41,7 @@ class Meta: class IndividualMetadataDict(Dictionary): family_guid = models.StringField(primary_key=True) - sampleId = models.StringField() + sampled = models.StringField() restrict_sharing = models.BoolField() features = models.StringField() omim_id = models.UInt32Field() @@ -52,7 +52,7 @@ class Meta: db_table = 'seqrdb_individual_metadata_dict' engine = models.MergeTree(primary_key=('family_guid', 'sampleId')) layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "select f.guid as family_guid, i.individual_id as sample_id, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" + postgres_query = "select f.guid as family_guid, i.individual_id as sampled, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" class DiscoveryVariantDict(Dictionary): From 4ff7eff48ac3511c70f8f0c719f50dc72fc0ace0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 17:48:45 -0400 Subject: [PATCH 077/155] fix dictionary defs --- clickhouse_search/all_search_tests.py | 2 +- ...ariantdict_excludedvariantdict_and_more.py | 71 +++++++++++++++++++ clickhouse_search/models/postgres_dicts.py | 8 +-- clickhouse_search/search.py | 6 +- 4 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index c8d7ea2672..b3c74e7649 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -82,7 +82,7 @@ def tearDownClass(cls): @classmethod def setUpTestData(cls): - for postgres_dict in [AffectedDict, SexDict, ExcludedVariantDict]: + for postgres_dict in [AffectedDict, SexDict, IndividualMetadataDict, DiscoveryVariantDict, ExcludedVariantDict]: postgres_dict.reload() for view in [ ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, ProjectsToGtStatsMito, ProjectsToGtStatsSv, diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py new file mode 100644 index 0000000000..8acbf9d10e --- /dev/null +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -0,0 +1,71 @@ +# Generated by Django 4.2.27 on 2026-05-14 21:48 + +import clickhouse_backend.models +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0042_delete_annotationsdiskgcnv_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='DiscoveryVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_discovery_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='ExcludedVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_excluded_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='IndividualMetadataDict', + fields=[ + ('family_guid', clickhouse_backend.models.StringField(primary_key=True, serialize=False)), + ('sampleId', clickhouse_backend.models.StringField()), + ('restrict_sharing', clickhouse_backend.models.BoolField()), + ('features', clickhouse_backend.models.StringField()), + ('omim_id', clickhouse_backend.models.UInt32Field()), + ('mondo_id', clickhouse_backend.models.StringField()), + ('is_solved', clickhouse_backend.models.BoolField()), + ], + options={ + 'db_table': 'seqrdb_individual_metadata_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key=('family_guid', 'sampleId')), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index bcb86d89b0..b20ec604c4 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -41,7 +41,7 @@ class Meta: class IndividualMetadataDict(Dictionary): family_guid = models.StringField(primary_key=True) - sampled = models.StringField() + sampleId = models.StringField() restrict_sharing = models.BoolField() features = models.StringField() omim_id = models.UInt32Field() @@ -52,7 +52,7 @@ class Meta: db_table = 'seqrdb_individual_metadata_dict' engine = models.MergeTree(primary_key=('family_guid', 'sampleId')) layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "select f.guid as family_guid, i.individual_id as sampled, p.restrict_sharing as restrict_sharing, i.features as features, i.post_discovery_omim_numbers[1] as omim_id, i.post_discovery_mondo_id as mondo_id, f.analysis_status in ('S', 'S_kgfp', 'S_kgdp', 'S_ng', 'ES') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" + postgres_query = "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" class DiscoveryVariantDict(Dictionary): @@ -63,7 +63,7 @@ class Meta: db_table = 'seqrdb_discovery_variant_dict' engine = models.MergeTree(primary_key='key') layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = 'CMG Discovery Tags')" + postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key" class ExcludedVariantDict(Dictionary): @@ -74,4 +74,4 @@ class Meta: db_table = 'seqrdb_excluded_variant_dict' engine = models.MergeTree(primary_key='key') layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_savedvariant vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = 'Excluded')" + postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key" diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2c45bec171..b81e4aad1b 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -14,6 +14,7 @@ ArrayMap, Modulo from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP +from clickhouse_search.models.postgres_dicts import SexDict, AffectedDict, IndividualMetadataDict from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP @@ -881,7 +882,10 @@ def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] entries = _filter_lookup_entries(entries, affected_only, hom_only) - entries = entries.result_values() + entries = entries.result_values(additional_expressions=OrderedDict({ + SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U'): ('sex', models.StringField()), + AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U'): ('affected', models.StringField()), + })) results = variants_cls.objects.subquery_join(entries) if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) From 1de880cc9abb7a7f8d98c7cfb8ab833ce927dc82 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 18:03:22 -0400 Subject: [PATCH 078/155] load individual metadata from clickhouse --- clickhouse_search/search.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index b81e4aad1b..dbba078ed9 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -1,4 +1,4 @@ -from clickhouse_backend.models import ArrayField, StringField +from clickhouse_backend.models import ArrayField, BoolField, StringField from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist @@ -882,10 +882,7 @@ def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] entries = _filter_lookup_entries(entries, affected_only, hom_only) - entries = entries.result_values(additional_expressions=OrderedDict({ - SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U'): ('sex', models.StringField()), - AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U'): ('affected', models.StringField()), - })) + entries = entries.result_values(additional_expressions=_lookup_genotype_expressions()) results = variants_cls.objects.subquery_join(entries) if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) @@ -902,6 +899,21 @@ def _filter_lookup_entries(entries, affected_only, hom_only): entries = entries.filter(calls__array_exists={'gt': (2,)}) return entries +def _lookup_genotype_expressions(): + affected_expr = AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U') + sex_expr = SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U') + metadata_expr = IndividualMetadataDict.dict_get_sql( + key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'features'], + ) + return { + f'tupleConcat(({affected_expr}, {sex_expr}), {metadata_expr})': ('metadata', NamedTupleField([ + ('affected', StringField()), + ('sex', StringField()), + ('restrict_sharing', BoolField()), + ('features', StringField()), + ])), + } + def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only): data_type = entry_qs = None for dataset_type, entry_cls in sorted(ENTRY_CLASS_MAP[genome_version].items()): From 75e740cf30db5334cd85de1afb316c41cd032544 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 18:08:33 -0400 Subject: [PATCH 079/155] add contact info to metadata --- .../0043_discoveryvariantdict_excludedvariantdict_and_more.py | 3 ++- clickhouse_search/models/postgres_dicts.py | 3 ++- clickhouse_search/search.py | 3 ++- seqr/views/apis/variant_search_api.py | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py index 8acbf9d10e..3711767b1f 100644 --- a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -56,12 +56,13 @@ class Migration(migrations.Migration): ('omim_id', clickhouse_backend.models.UInt32Field()), ('mondo_id', clickhouse_backend.models.StringField()), ('is_solved', clickhouse_backend.models.BoolField()), + ('vlm_contact_email', clickhouse_backend.models.StringField()), ], options={ 'db_table': 'seqrdb_individual_metadata_dict', 'engine': clickhouse_backend.models.MergeTree(primary_key=('family_guid', 'sampleId')), 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", + 'postgres_query': "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved, p.vlm_contact_email as vlm_contact_email FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index b20ec604c4..a0ca45306c 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -47,12 +47,13 @@ class IndividualMetadataDict(Dictionary): omim_id = models.UInt32Field() mondo_id = models.StringField() is_solved = models.BoolField() + vlm_contact_email = models.StringField() class Meta: db_table = 'seqrdb_individual_metadata_dict' engine = models.MergeTree(primary_key=('family_guid', 'sampleId')) layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" + postgres_query = "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved, p.vlm_contact_email as vlm_contact_email FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" class DiscoveryVariantDict(Dictionary): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index dbba078ed9..800c225ab8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -903,7 +903,7 @@ def _lookup_genotype_expressions(): affected_expr = AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U') sex_expr = SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U') metadata_expr = IndividualMetadataDict.dict_get_sql( - key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'features'], + key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'features', 'vlm_contact_email'], ) return { f'tupleConcat(({affected_expr}, {sex_expr}), {metadata_expr})': ('metadata', NamedTupleField([ @@ -911,6 +911,7 @@ def _lookup_genotype_expressions(): ('sex', StringField()), ('restrict_sharing', BoolField()), ('features', StringField()), + ('vlm_contact_email', StringField()), ])), } diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 85875748b6..6ae800878d 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -647,6 +647,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): + del genotype['metadata'] individual_guid = individual_guid_map.get((family_guid, genotype['sampleId'])) if not individual_guid: logger.error( From 51f033b4d234309336f88b9f8a37648441e6eaa3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 18:43:46 -0400 Subject: [PATCH 080/155] use clickhosue metadata instead of postgres --- clickhouse_search/all_search_tests.py | 31 ++++++++++++-- clickhouse_search/search.py | 4 +- seqr/views/apis/variant_search_api.py | 60 ++++++++++----------------- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index b3c74e7649..bb185f34d4 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1004,6 +1004,27 @@ def test_variant_lookup(self, mock_liftover): self.check_require_login(reverse(variant_lookup_handler)) + individual_metadata = { + 'I000006_hg00733': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000005_hg00732': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000004_hg00731': { + 'affected': 'A', 'features': '[{"id": "HP:0002011"}, {"id": "HP:0011675"}]', 'restrict_sharing': False, + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000015_na20885': { + 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, + 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, + 'I000018_na21234': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + } lookup_variant = { **VARIANT1, 'liftedFamilyGuids': ['F000014_14'], @@ -1077,7 +1098,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-10439-AC-A', expected_variant, cache_key, cached_variants=[lookup_variant], skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', - }, expected_individuals=expected_individuals, locusListsByGuid={}, + }, expected_individuals=expected_individuals, individual_metadata=individual_metadata, locusListsByGuid={}, ) expected_gcnv_variant = { @@ -1155,6 +1176,7 @@ def test_variant_lookup(self, mock_liftover): additional_variant=expected_gcnv_variant, cached_variants=[SV_VARIANT4, GCNV_VARIANT4], skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById=sv_genes, + individual_metadata=individual_metadata, ) self.login_manager() @@ -1357,7 +1379,7 @@ def test_variant_lookup(self, mock_liftover): ) self.assert_json_logs(self.no_access_user, unmapped_sample_logs) - def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, individual_metadata=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' if hom_only: url += '&homOnly=true' @@ -1399,7 +1421,10 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version gts = [gts] for gt in gts: family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] - family_genotypes[family_guid].append({k: v for k, v in gt.items() if k != 'individualGuid'}) + family_genotypes[family_guid].append({ + **{k: v for k, v in gt.items() if k != 'individualGuid'}, + 'metadata': (individual_metadata or {}).get(gt['individualGuid']), + }) parsed_cached_variants.append({ **{k: v for k, v in v.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': { diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 800c225ab8..d467621658 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -911,7 +911,7 @@ def _lookup_genotype_expressions(): ('sex', StringField()), ('restrict_sharing', BoolField()), ('features', StringField()), - ('vlm_contact_email', StringField()), + ('vlmContactEmail', StringField()), ])), } @@ -986,7 +986,7 @@ def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_id = f"{variant['liftedOverChrom']}-{variant['liftedOverPos']}-{variant['ref']}-{variant['alt']}" lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) - gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() + gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression(additional_expressions=_lookup_genotype_expressions()) lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) if lifted_entry_data: variant['familyGenotypes'].update(lifted_entry_data[0]['familyGenotypes']) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 6ae800878d..9e487ae5ac 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -16,8 +16,8 @@ from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset +from reference_data.models import HumanPhenotypeOntology, GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP +from seqr.models import Project, Family, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger @@ -27,7 +27,7 @@ from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ - get_json_for_saved_searches, add_individual_hpo_details, FAMILY_ADDITIONAL_VALUES + get_json_for_saved_searches, FAMILY_ADDITIONAL_VALUES from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities @@ -37,17 +37,6 @@ logger = SeqrLogger(__name__) -GENOTYPE_AC_LOOKUP = { - 'ref_ref': [0, 0], - 'has_ref': [0, 1], - 'ref_alt': [1, 1], - 'has_alt': [1, 2], - 'alt_alt': [2, 2], -} -AFFECTED = Individual.AFFECTED_STATUS_AFFECTED -UNAFFECTED = Individual.AFFECTED_STATUS_UNAFFECTED - - @login_and_policies_required def query_variants_handler(request, search_hash): page = int(request.GET.get('page') or 1) @@ -631,17 +620,6 @@ def _get_lookup_cache_key(user, variant_id, sample_type, genome_version, affecte def _update_lookup_variant(variant, response, individual_guid_map, user): - no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) - individual_summary_map = { - (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) - for i in Individual.objects.filter(family__guid__in=no_access_families).values( - 'family__guid', 'individual_id', 'affected', 'sex', 'features', 'guid', - vlmContactEmail=F('family__project__vlm_contact_email'), - restrict_sharing=F('family__project__restrict_sharing'), - ) - } - add_individual_hpo_details([i for _, i in individual_summary_map.values()]) - variant['genotypes'] = {} variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] @@ -665,23 +643,17 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): variant['liftedFamilyGuids'][variant['liftedFamilyGuids'].index(unmapped_family_guid)] = family_guid - individual_guid_map = {} + individual_key_map = {} for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) - if individual_key not in individual_summary_map: - logger.error( - f'Unable to map sample {individual_key[1]} in family {individual_key[0]} to an individual for variant {variant["variantId"]}', - user, - ) - continue - unmapped_individual_guid, individual = individual_summary_map[individual_key] - if unmapped_individual_guid in individual_guid_map: - individual_guid = individual_guid_map[unmapped_individual_guid] + individual = genotype.pop('metadata') + if individual_key in individual_key_map: + individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] continue individual_guid = f'I{j}_{family_guid}' - individual_guid_map[unmapped_individual_guid] = individual_guid - features = individual['features'] or [] + individual_key_map[individual_key] = individual_guid + features = _add_hpo_details(individual['features']) if individual.pop('restrict_sharing'): feature_category_count = defaultdict(int) for feature in features: @@ -699,6 +671,20 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['genotypes'][individual_guid] = genotype +def _add_hpo_details(raw_features): + if not raw_features: + return [] + features = json.loads(raw_features) + hpo_terms_by_id = { + hpo.pop('hpo_id'): hpo for hpo in HumanPhenotypeOntology.objects.filter( + hpo_id__in=[feature['id'] for feature in features], + ).values('hpo_id', category=F('category_id'), label=F('name')) + } + for feature in features: + feature.update(hpo_terms_by_id.get(feature['id'], {})) + return features + + @login_and_policies_required def vlm_lookup_handler(request): kwargs = {_to_snake_case(k): v for k, v in request.GET.items()} From cb518900750dcc00d026a7f27e61251dac128243 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 May 2026 19:03:20 -0400 Subject: [PATCH 081/155] fix metadata for other sv lookup --- clickhouse_search/all_search_tests.py | 10 +++++++++- clickhouse_search/search.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index bb185f34d4..f8e64f2d9e 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1024,6 +1024,12 @@ def test_variant_lookup(self, mock_liftover): 'I000018_na21234': { 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', }, + 'I000019_na21987': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I000021_na21654': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', + }, } lookup_variant = { **VARIANT1, @@ -1199,7 +1205,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], individual_metadata=individual_metadata, ) hom_only_lookup_variant = { @@ -1217,6 +1223,7 @@ def test_variant_lookup(self, mock_liftover): '1-10439-AC-A', hom_only_lookup_variant, f'{cache_key}__hom', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], + individual_metadata=individual_metadata, ) url = self._assert_expected_lookup( @@ -1234,6 +1241,7 @@ def test_variant_lookup(self, mock_liftover): 'VFD0000023_1248367227_r0390_10': mock.ANY, 'VFD0000024_1248367227_r0390_10': mock.ANY, 'VFD0000025_1248367227_r0390_10': mock.ANY, 'VFD0000026_1248367227_r0390_10': mock.ANY, }, + individual_metadata=individual_metadata, ) response = self.client.get(url + '&affectedOnly=true') diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index d467621658..33624e8734 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -968,7 +968,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff padding = int((variant['end'] - variant['pos']) * 0.2) entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) - entries = _filter_lookup_entries(entries, affected_only, hom_only) + entries = _filter_lookup_entries(entries, affected_only, hom_only).result_values(additional_expressions=_lookup_genotype_expressions()) results = other_variants_cls.objects.subquery_join(entries).search( padded_interval_end=(variant['end'], padding), **other_variants_cls.objects.get_parsed_annotations_filters( annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, From ca6c4252a0866a2b0a3506b8a769e1d54b35495d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 10:46:58 -0400 Subject: [PATCH 082/155] shared helper for cached variant formatting --- clickhouse_search/all_search_tests.py | 119 ++++++++++++++------------ 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index f8e64f2d9e..4bd5bcb279 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1004,33 +1004,6 @@ def test_variant_lookup(self, mock_liftover): self.check_require_login(reverse(variant_lookup_handler)) - individual_metadata = { - 'I000006_hg00733': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I000005_hg00732': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I000004_hg00731': { - 'affected': 'A', 'features': '[{"id": "HP:0002011"}, {"id": "HP:0011675"}]', 'restrict_sharing': False, - 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I000015_na20885': { - 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, - 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', - }, - 'I000018_na21234': { - 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I000019_na21987': { - 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I000021_na21654': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - } lookup_variant = { **VARIANT1, 'liftedFamilyGuids': ['F000014_14'], @@ -1104,7 +1077,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-10439-AC-A', expected_variant, cache_key, cached_variants=[lookup_variant], skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', - }, expected_individuals=expected_individuals, individual_metadata=individual_metadata, locusListsByGuid={}, + }, expected_individuals=expected_individuals, locusListsByGuid={}, ) expected_gcnv_variant = { @@ -1182,7 +1155,6 @@ def test_variant_lookup(self, mock_liftover): additional_variant=expected_gcnv_variant, cached_variants=[SV_VARIANT4, GCNV_VARIANT4], skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById=sv_genes, - individual_metadata=individual_metadata, ) self.login_manager() @@ -1205,7 +1177,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], individual_metadata=individual_metadata, + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], ) hom_only_lookup_variant = { @@ -1223,7 +1195,6 @@ def test_variant_lookup(self, mock_liftover): '1-10439-AC-A', hom_only_lookup_variant, f'{cache_key}__hom', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], - individual_metadata=individual_metadata, ) url = self._assert_expected_lookup( @@ -1241,7 +1212,6 @@ def test_variant_lookup(self, mock_liftover): 'VFD0000023_1248367227_r0390_10': mock.ANY, 'VFD0000024_1248367227_r0390_10': mock.ANY, 'VFD0000025_1248367227_r0390_10': mock.ANY, 'VFD0000026_1248367227_r0390_10': mock.ANY, }, - individual_metadata=individual_metadata, ) response = self.client.get(url + '&affectedOnly=true') @@ -1253,10 +1223,9 @@ def test_variant_lookup(self, mock_liftover): self.assertEqual(response.status_code, 404) self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) - self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [{ - **VARIANT1, - 'familyGenotypes': {'F000002_2': list(VARIANT1['genotypes'].values())}, - }]) + self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [ + self._cached_lookup_variant(VARIANT1), + ]) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json()['variantsById'], {'1-10439-AC-A': { @@ -1387,7 +1356,42 @@ def test_variant_lookup(self, mock_liftover): ) self.assert_json_logs(self.no_access_user, unmapped_sample_logs) - def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, individual_metadata=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): + INDIVIDUAL_METADATA = { + 'I000006_hg00733': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000005_hg00732': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000004_hg00731': { + 'affected': 'A', 'features': '[{"id": "HP:0002011"}, {"id": "HP:0011675"}]', 'restrict_sharing': False, + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I000015_na20885': { + 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, + 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, + 'I000018_na21234': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I000019_na21987': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I000021_na21654': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I000002_na19678': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' if hom_only: url += '&homOnly=true' @@ -1421,28 +1425,31 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version } self.assertDictEqual(response.json(), expected_body) - parsed_cached_variants = [] - for v in (cached_variants or variants): - family_genotypes = defaultdict(list) - for individual_guid, gts in v['genotypes'].items(): - if not isinstance(gts, list): - gts = [gts] - for gt in gts: - family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] - family_genotypes[family_guid].append({ - **{k: v for k, v in gt.items() if k != 'individualGuid'}, - 'metadata': (individual_metadata or {}).get(gt['individualGuid']), - }) - parsed_cached_variants.append({ - **{k: v for k, v in v.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': { - family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) - for family_guid, gts in family_genotypes.items() - }, - }) + parsed_cached_variants = [ + self._cached_lookup_variant(v, expected_individuals) for v in (cached_variants or variants) + ] self.assert_cached_results(parsed_cached_variants, cache_key) return url + def _cached_lookup_variant(self, variant, expected_individuals=None): + family_genotypes = defaultdict(list) + for individual_guid, gts in variant['genotypes'].items(): + if not isinstance(gts, list): + gts = [gts] + for gt in gts: + family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] + family_genotypes[family_guid].append({ + **{k: v for k, v in gt.items() if k != 'individualGuid'}, + 'metadata': self.INDIVIDUAL_METADATA.get(gt['individualGuid']), + }) + return { + **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, + 'familyGenotypes': { + family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) + for family_guid, gts in family_genotypes.items() + }, + } + def test_get_single_variant(self): url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') url = url_template.format('21-3343353-GAGA-G', 'F000001_1') From 9e6254dfe297c3be51847f9ff295c32db07c9dad Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 11:05:59 -0400 Subject: [PATCH 083/155] no access missing indiv --- clickhouse_search/all_search_tests.py | 26 ++++++++++++++++++-------- clickhouse_search/managers.py | 4 ++-- clickhouse_search/search.py | 6 ++++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 4bd5bcb279..8b67157d76 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1327,34 +1327,44 @@ def test_variant_lookup(self, mock_liftover): ] self.assert_json_logs(self.manager_user, unmapped_sample_logs) + # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() self.reset_logs() - expected_individuals = {'I1_F0_7-143270172-A-G': { + expected_individuals = {'I0_F0_7-143270172-A-G': { + 'affected': 'N', + 'familyGuid': 'F0_7-143270172-A-G', + 'features': [], + 'individualGuid': 'I0_F0_7-143270172-A-G', + 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, 'I1_F0_7-143270172-A-G': { 'affected': 'A', 'familyGuid': 'F0_7-143270172-A-G', 'features': [ {'category': 'HP:0000707', 'id': 'HP:0002011', 'label': 'Morphological abnormality of the central nervous system'}, - {'category': 'HP:0001626', 'id': 'HP:0011675', 'label': 'Arrhythmia'}, + {'category': 'HP:0001626', 'id': 'HP:0011675', 'label': 'Arrhythmia'}, ], 'individualGuid': 'I1_F0_7-143270172-A-G', 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }} - no_access_missing_gt_variant = { + no_access_variant = { **GRCH37_VARIANT, 'familyGuids': ['F0_7-143270172-A-G'], - 'genotypes': {'I1_F0_7-143270172-A-G': { - k: v for k, v in GRCH37_VARIANT['genotypes']['I000004_hg00731'].items() + 'genotypes': {mapped_guid: { + k: v for k, v in GRCH37_VARIANT['genotypes'][guid].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }} + } for guid, mapped_guid in { + 'I000004_hg00731': 'I1_F0_7-143270172-A-G', 'I000006_hg00733': 'I0_F0_7-143270172-A-G', + }.items()} } self._assert_expected_lookup( - '7-143270172-A-G', no_access_missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], + '7-143270172-A-G', no_access_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', expected_individuals=expected_individuals, locusListsByGuid={}, skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', }, ) - self.assert_json_logs(self.no_access_user, unmapped_sample_logs) + self.assert_json_logs(self.no_access_user, unmapped_sample_logs[:1]) INDIVIDUAL_METADATA = { 'I000006_hg00733': { diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index ddc16337c8..0be54a6fcf 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1494,9 +1494,9 @@ def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): def _can_filter_gene_interval(self, genes): return (not hasattr(self.model, 'geneId_ids')) or len(genes) < self.MAX_XPOS_FILTER_INTERVALS or self.filtered_chrom - def search_padded_interval(self, chrom, pos, padding): + def search_padded_interval(self, chrom, pos, padding, **kwargs): interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) - return self.filter(interval_q).result_values() + return self.filter(interval_q).result_values(**kwargs) @staticmethod def _interval_query(chrom, start, end, **kwargs): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 33624e8734..f4d5ad42b8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -967,8 +967,10 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] padding = int((variant['end'] - variant['pos']) * 0.2) - entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) - entries = _filter_lookup_entries(entries, affected_only, hom_only).result_values(additional_expressions=_lookup_genotype_expressions()) + entries = other_entry_class.objects.search_padded_interval( + variant['chrom'], variant['pos'], padding, additional_expressions=_lookup_genotype_expressions(), + ) + entries = _filter_lookup_entries(entries, affected_only, hom_only) results = other_variants_cls.objects.subquery_join(entries).search( padded_interval_end=(variant['end'], padding), **other_variants_cls.objects.get_parsed_annotations_filters( annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, From 183b818acf436949620c9734c9f22d2003f2a3a6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 11:11:39 -0400 Subject: [PATCH 084/155] add reload to migration --- .../0043_discoveryvariantdict_excludedvariantdict_and_more.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py index 3711767b1f..f9e4eeca0a 100644 --- a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -69,4 +69,7 @@ class Migration(migrations.Migration): ('_overwrite_base_manager', django.db.models.manager.Manager()), ], ), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_discovery_variant_dict"'), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_excluded_variant_dict"'), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_individual_metadata_dict"'), ] From 2c1ebd3c1874f82ca030122f53f3bd66ff8fb29b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 11:48:30 -0400 Subject: [PATCH 085/155] pr feedback --- seqr/views/apis/variant_search_api.py | 43 ++++++++++++++------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 9e487ae5ac..fb3bc6d53e 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -625,7 +625,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): - del genotype['metadata'] + genotype.pop('metadata', None) individual_guid = individual_guid_map.get((family_guid, genotype['sampleId'])) if not individual_guid: logger.error( @@ -638,6 +638,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype + all_feature_ids = set() for i, (unmapped_family_guid, genotypes) in enumerate(sorted(variant.pop('familyGenotypes').items())): family_guid = f'F{i}_{variant["variantId"]}' variant['lookupFamilyGuids'].append(family_guid) @@ -646,22 +647,15 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): individual_key_map = {} for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) - individual = genotype.pop('metadata') + individual = genotype.pop('metadata', {}) if individual_key in individual_key_map: individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] continue individual_guid = f'I{j}_{family_guid}' individual_key_map[individual_key] = individual_guid - features = _add_hpo_details(individual['features']) - if individual.pop('restrict_sharing'): - feature_category_count = defaultdict(int) - for feature in features: - feature_category_count[feature.get('category', 'Other')] += 1 - features = [ - {'category': category, 'label': f'{count} terms'} - for category, count in feature_category_count.items() - ] + features = json.loads(individual['features']) if individual.get('features') else [] + all_feature_ids.update([feature['id'] for feature in features]) response['individualsByGuid'][individual_guid] = { **individual, 'familyGuid': family_guid, @@ -670,19 +664,26 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): } variant['genotypes'][individual_guid] = genotype + _parse_hpo_terms(all_feature_ids, response['individualsByGuid'].values()) -def _add_hpo_details(raw_features): - if not raw_features: - return [] - features = json.loads(raw_features) + +def _parse_hpo_terms(all_feature_ids, individuals): hpo_terms_by_id = { - hpo.pop('hpo_id'): hpo for hpo in HumanPhenotypeOntology.objects.filter( - hpo_id__in=[feature['id'] for feature in features], - ).values('hpo_id', category=F('category_id'), label=F('name')) + hpo.pop('hpo_id'): hpo for hpo in HumanPhenotypeOntology.objects.filter(hpo_id__in=all_feature_ids).values( + 'hpo_id', category=F('category_id'), label=F('name'), + ) } - for feature in features: - feature.update(hpo_terms_by_id.get(feature['id'], {})) - return features + for individual in individuals: + for feature in individual['features'] or []: + feature.update(hpo_terms_by_id.get(feature['id'], {})) + if individual.pop('restrict_sharing', False) and individual['features']: + feature_category_count = defaultdict(int) + for feature in individual['features']: + feature_category_count[feature.get('category', 'Other')] += 1 + individual['features'] = [ + {'category': category, 'label': f'{count} terms'} + for category, count in feature_category_count.items() + ] @login_and_policies_required From b64895e91011cbfdb850fae8faae1fd162b78516 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 13:00:10 -0400 Subject: [PATCH 086/155] add discovery tags to snv indel variants --- clickhouse_search/managers.py | 3 +- seqr/views/utils/orm_to_json_utils.py | 47 +------------------- seqr/views/utils/variant_utils.py | 64 ++++++++++++++++++--------- 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 0be54a6fcf..1464a48e62 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -11,7 +11,7 @@ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ GroupArrayIntersect, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat, Untuple, \ IntDiv, Modulo, SplitByString, ArrayIndex, Multiply, IndexOf -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, DiscoveryVariantDict from clickhouse_search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ @@ -921,6 +921,7 @@ def annotation_values(self): annotations = { **super().annotation_values, **self.split_variant_id_annotations(), + 'discoveryFamilies': DiscoveryVariantDict.dict_get_expression('key'), } if self.has_annotation('hgmd_join'): annotations.update({ diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 34d3c2616a..7f5b98c938 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -8,13 +8,13 @@ from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology -from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ +from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, CAN_VIEW, CAN_EDIT, \ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier, VariantSearchResults from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ - is_internal_anvil_project, get_project_guids_user_can_view, get_anvil_analyst_user_emails + is_internal_anvil_project, get_anvil_analyst_user_emails from seqr.views.utils.terra_api_utils import is_anvil_authenticated, anvil_enabled from settings import ANALYST_USER_GROUP, SERVICE_ACCOUNT_FOR_ANVIL, MEDIA_URL @@ -441,49 +441,6 @@ def get_json_for_saved_variants_with_tags(saved_variants, additional_model_field return response -def get_json_for_discovery_tags(variants, user): - from seqr.views.utils.variant_utils import get_variant_key - response = {} - discovery_tags = defaultdict(list) - - saved_variants = SavedVariant.objects.filter( - variant_id__in={variant['variantId'] for variant in variants}, - family__project__guid__in=get_project_guids_user_can_view(user), - ).only('id', 'guid', 'ref', 'alt', 'xpos', 'family_id').prefetch_related('family', 'family__project') - saved_variants_by_guid = {sv.guid: sv for sv in saved_variants} - saved_variant_id_map = {sv.id: guid for guid, sv in saved_variants_by_guid.items()} - - discovery_tag_json, _ = get_json_for_saved_variants_child_entities( - VariantTag, saved_variant_id_map, tag_filter={'variant_tag_type__category': 'CMG Discovery Tags'}) - if discovery_tag_json: - existing_families = set() - for variant in variants: - existing_families.update(variant['familyGuids']) - - family_ids = set() - for tag in discovery_tag_json: - for variant_guid in tag.pop('variantGuids'): - variant = saved_variants_by_guid[variant_guid] - if variant.family.guid not in existing_families: - family_ids.add(variant.family_id) - tag_json = {'savedVariant': { - 'variantGuid': variant.guid, - 'familyGuid': variant.family.guid, - 'projectGuid': variant.family.project.guid, - }} - tag_json.update(tag) - variant_key = get_variant_key( - genomeVersion=variant.family.project.genome_version, - xpos=variant.xpos, ref=variant.ref, alt=variant.alt, - ) - discovery_tags[variant_key].append(tag_json) - - response['familiesByGuid'] = { - f['familyGuid']: f for f in _get_json_for_families(Family.objects.filter(id__in=family_ids)) - } - return discovery_tags, response - - def get_json_for_variant_note(note): return _get_json_for_model(note, guid_key='noteGuid') diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 19088364e6..3c7d264996 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -16,10 +16,10 @@ from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_to_orm_utils import create_model_from_json -from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ - get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ +from seqr.views.utils.orm_to_json_utils import get_json_for_locus_lists, get_json_for_saved_variants_child_entities, \ + get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, _get_json_for_families, \ get_json_for_matchmaker_submissions -from seqr.views.utils.permissions_utils import has_case_review_permissions, user_is_analyst +from seqr.views.utils.permissions_utils import has_case_review_permissions, get_project_guids_user_can_view from seqr.views.utils.project_context_utils import add_project_tag_types, add_families_context from settings import REDIS_SERVICE_HOSTNAME, REDIS_SERVICE_PORT @@ -233,10 +233,6 @@ def reset_cached_search_results(project, reset_index_metadata=False): logger.error("Unable to reset cached search results: {}".format(e)) -def get_variant_key(xpos=None, ref=None, alt=None, genomeVersion=None, **kwargs): - return '{}-{}-{}_{}'.format(xpos, ref, alt, genomeVersion) - - def _requires_transcript_metadata(variant): if isinstance(variant, list): return _requires_transcript_metadata(variant[0]) @@ -355,13 +351,44 @@ def _get_family_has_rna_tpm(family_genes, gene_ids, sample_family_map): return family_tpms -def _add_discovery_tags(variants, discovery_tags): - for variant in variants: - tags = discovery_tags.get(get_variant_key(**variant)) - if tags: - if not variant.get('discoveryTags'): - variant['discoveryTags'] = [] - variant['discoveryTags'] += [tag for tag in tags if tag['savedVariant']['familyGuid'] not in variant['familyGuids']] +def _parse_discovery_tags(variants_by_id, family_guids, user): + discovery_family_guids = set() + for variant_id, variant in variants_by_id.items(): + discovery_families = set(variant.pop('discoveryFamilies')) - set(variant['familyGuids']) + if discovery_families: + discovery_family_guids.update(discovery_families) + variant['discoveryTags'] = [] + variant['noAccessDiscoveryFamilies'] = len(discovery_families) + + discovery_families_by_guid = { + f['familyGuid']: f for f in _get_json_for_families(Family.objects.filter( + guid__in=discovery_family_guids, project__guid__in=get_project_guids_user_can_view(user), + ).exclude(guid__in=family_guids)) + } + if not discovery_families_by_guid: + return {} + + saved_variants_by_guid = { + sv['variantGuid']: sv for sv in SavedVariant.objects.filter( + key__in={variant['key'] for variant in variants}, + family__guid__in=discovery_families_by_guid, + ).values('id', 'variant_id', variantGuid=F('guid'), familyGuid=F('family__guid'), projectGuid=F('family__project__guid')) + } + + saved_variant_id_map = {sv.pop('id'): guid for guid, sv in saved_variants_by_guid.items()} + discovery_tag_json, _ = get_json_for_saved_variants_child_entities( + VariantTag, saved_variant_id_map, tag_filter={'variant_tag_type__category': 'CMG Discovery Tags'}) + + for tag in discovery_tag_json: + for variant_guid in tag.pop('variantGuids'): + variant = saved_variants_by_guid[variant_guid] + variant_id = variant.pop('variant_id') + tag_json = {'savedVariant': variant} + tag_json.update(tag) + variants_by_id[variant_id]['discoveryTags'].append(tag_json) + variants_by_id[variant_id]['noAccessDiscoveryFamilies'] -= 1 + + return {'familiesByGuid': discovery_families_by_guid} def _add_pa_detail(locus_list_gene, locus_list_guid, gene_json): @@ -406,18 +433,11 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a variants, genome_versions={genome_version} if genome_version else {p.genome_version for p in projects}, get_family_genes=include_individual_gene_scores, )) - discovery_tags = None - is_analyst = user_is_analyst(request.user) - if is_analyst: - discovery_tags, discovery_response = get_json_for_discovery_tags(variants, request.user) - response.update(discovery_response) + response.update(_parse_discovery_tags(variants_by_id, family_guids, request.user)) response['locusListsByGuid'] = _add_locus_lists( projects, response['genesById'], add_list_detail=add_locus_list_detail, user=request.user) - if discovery_tags: - _add_discovery_tags(variants, discovery_tags) - response['mmeSubmissionsByGuid'] = _mme_response_context(response['savedVariantsByGuid']) rna_tpm = _set_response_gene_scores(response, response.pop('family_genes'), response['genesById'].keys()) if include_individual_gene_scores else None From b65fb97ce5d02b2e3400988f7212ae99016ee371 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 13:05:33 -0400 Subject: [PATCH 087/155] fix variant tables --- ...ryvariantdict_excludedvariantdict_and_more.py | 16 ++++++++++------ clickhouse_search/models/postgres_dicts.py | 10 ++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py index f9e4eeca0a..66f666513a 100644 --- a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -16,13 +16,15 @@ class Migration(migrations.Migration): name='DiscoveryVariantDict', fields=[ ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), - ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ('dataset_type', clickhouse_backend.models.StringField()), + ('family_guids', + clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), ], options={ 'db_table': 'seqrdb_discovery_variant_dict', - 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'engine': clickhouse_backend.models.MergeTree(primary_key=('key', 'dataset_type')), 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key", + 'postgres_query': "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key, sv.dataset_type", }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -33,13 +35,15 @@ class Migration(migrations.Migration): name='ExcludedVariantDict', fields=[ ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), - ('family_guids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ('dataset_type', clickhouse_backend.models.StringField()), + ('family_guids', + clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), ], options={ 'db_table': 'seqrdb_excluded_variant_dict', - 'engine': clickhouse_backend.models.MergeTree(primary_key='key'), + 'engine': clickhouse_backend.models.MergeTree(primary_key=('key', 'dataset_type')), 'layout': 'COMPLEX_KEY_HASHED()', - 'postgres_query': "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key", + 'postgres_query': "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key, sv.dataset_type", }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index a0ca45306c..ac4098cf11 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -58,21 +58,23 @@ class Meta: class DiscoveryVariantDict(Dictionary): key = models.UInt32Field(primary_key=True) + dataset_type = models.StringField() family_guids = models.ArrayField(models.StringField()) class Meta: db_table = 'seqrdb_discovery_variant_dict' - engine = models.MergeTree(primary_key='key') + engine = models.MergeTree(primary_key=('key', 'dataset_type')) layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key" + postgres_query = "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key, sv.dataset_type" class ExcludedVariantDict(Dictionary): key = models.UInt32Field(primary_key=True) + dataset_type = models.StringField() family_guids = models.ArrayField(models.StringField()) class Meta: db_table = 'seqrdb_excluded_variant_dict' - engine = models.MergeTree(primary_key='key') + engine = models.MergeTree(primary_key=('key', 'dataset_type')) layout = 'COMPLEX_KEY_HASHED()' - postgres_query = "SELECT sv.key as key, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key" + postgres_query = "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key, sv.dataset_type" From 57924a4bf1e9ef9d2a21289ed87839099e9394fa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 14:50:16 -0400 Subject: [PATCH 088/155] use dataset tpe for varant dict get key --- clickhouse_search/backend/table_models.py | 4 ++-- clickhouse_search/managers.py | 2 +- clickhouse_search/models/postgres_dicts.py | 19 ++++++++++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/backend/table_models.py b/clickhouse_search/backend/table_models.py index c8cb45a684..819ca96972 100644 --- a/clickhouse_search/backend/table_models.py +++ b/clickhouse_search/backend/table_models.py @@ -81,13 +81,13 @@ def base_fields(cls): return [(field.db_column or field.name, field) for field in cls._meta.local_fields if field.name != 'key'] @classmethod - def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False, **kwargs): + def dict_get_expression(cls, *expressions, key_template='%(expressions)s', field_names=None, force_tuple=False, **kwargs): base_fields = cls.base_fields() if field_names: base_fields = [f for f in base_fields if f[0] in field_names] output_field = base_fields[0][1] if len(base_fields) == 1 and not force_tuple else NamedTupleField(base_fields) dict_get_func = Func(*expressions, output_field=output_field) - dict_get_func.template = cls.dict_get_sql('%(expressions)s', [field_name for field_name, _ in base_fields], **kwargs) + dict_get_func.template = cls.dict_get_sql(key_template, [field_name for field_name, _ in base_fields], **kwargs) if force_tuple and len(base_fields) == 1: dict_get_func.template = f'tuple({dict_get_func.template})' return dict_get_func diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1464a48e62..6e40644219 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -921,7 +921,7 @@ def annotation_values(self): annotations = { **super().annotation_values, **self.split_variant_id_annotations(), - 'discoveryFamilies': DiscoveryVariantDict.dict_get_expression('key'), + 'discoveryFamilies': DiscoveryVariantDict.dict_get_expression('key', dataset_type='SNV_INDEL'), } if self.has_annotation('hgmd_join'): annotations.update({ diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index ac4098cf11..6887b2b8d3 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -56,11 +56,23 @@ class Meta: postgres_query = "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved, p.vlm_contact_email as vlm_contact_email FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" -class DiscoveryVariantDict(Dictionary): +class VariantFamiliesDict(Dictionary): key = models.UInt32Field(primary_key=True) dataset_type = models.StringField() family_guids = models.ArrayField(models.StringField()) + class Meta: + abstract = True + + @classmethod + def dict_get_expression(cls, *expressions, dataset_type=None, **kwargs): + return super().dict_get_expression( + *expressions, key_template=f"(%(expressions)s, '{dataset_type}')", field_names=['family_guids'], **kwargs, + ) + + +class DiscoveryVariantDict(VariantFamiliesDict): + class Meta: db_table = 'seqrdb_discovery_variant_dict' engine = models.MergeTree(primary_key=('key', 'dataset_type')) @@ -68,10 +80,7 @@ class Meta: postgres_query = "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key, sv.dataset_type" -class ExcludedVariantDict(Dictionary): - key = models.UInt32Field(primary_key=True) - dataset_type = models.StringField() - family_guids = models.ArrayField(models.StringField()) +class ExcludedVariantDict(VariantFamiliesDict): class Meta: db_table = 'seqrdb_excluded_variant_dict' From 0892033cfc7ad77521dfeda41f715b86fbb465ea Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:06:41 -0400 Subject: [PATCH 089/155] add sv discovery families --- clickhouse_search/managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 6eac69f0fb..0ea377322d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -752,6 +752,7 @@ class SvVariantsQuerySet(BaseVariantsQuerySet): def annotation_values(self): annotations = super().annotation_values annotations['transcripts'] = annotations.pop(self.model.sorted_gene_consequences.field.db_column) + annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type=f'SV_{self.entry_model.SAMPLE_TYPE}'), return annotations @staticmethod From f8a1815e6ea9f2afed9cd78d1819a45dccc9dd18 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:13:32 -0400 Subject: [PATCH 090/155] handle other data types --- clickhouse_search/managers.py | 5 ++++- clickhouse_search/test_utils.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 0ea377322d..27a503a476 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -537,6 +537,9 @@ def annotation_values(self): if self.has_annotation('mitomapPathogenic'): annotations['mitomapPathogenic'] = F('mitomapPathogenic') + if not self.variant_detail_field: + annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type='MITO') + return annotations @staticmethod @@ -752,7 +755,7 @@ class SvVariantsQuerySet(BaseVariantsQuerySet): def annotation_values(self): annotations = super().annotation_values annotations['transcripts'] = annotations.pop(self.model.sorted_gene_consequences.field.db_column) - annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type=f'SV_{self.entry_model.SAMPLE_TYPE}'), + annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type=f'SV_{self.entry_model.SAMPLE_TYPE}') return annotations @staticmethod diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index b43764cd73..249782d113 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1360,7 +1360,7 @@ def format_cached_variant(variant): if variant['key'] not in CACHED_CONSEQUENCES_BY_KEY: - return variant + return {**variant, 'discoveryFamilies': []} return { **{k: v for k, v in variant.items() if k not in [ 'mainTranscriptId', 'selectedMainTranscriptId', 'transcripts', 'CAID', 'chrom', 'pos', 'ref', 'alt', From 7147e6d36a25674d0092bf75bf457fe2647682f5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:21:09 -0400 Subject: [PATCH 091/155] add replace missing import --- seqr/views/utils/variant_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index e080edee5b..041e6cafdf 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -16,10 +16,10 @@ from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.xpos_utils import parse_variant_id, get_chrom_pos from seqr.views.utils.json_to_orm_utils import create_model_from_json -from seqr.views.utils.orm_to_json_utils import get_json_for_locus_lists, get_json_for_saved_variants_child_entities, \ +from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_child_entities, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, _get_json_for_families, \ get_json_for_matchmaker_submissions -from seqr.views.utils.permissions_utils import has_case_review_permissions, get_project_guids_user_can_view +from seqr.views.utils.permissions_utils import has_case_review_permissions, user_is_analyst, get_project_guids_user_can_view from seqr.views.utils.project_context_utils import add_project_tag_types, add_families_context from settings import REDIS_SERVICE_HOSTNAME, REDIS_SERVICE_PORT @@ -453,7 +453,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a if add_all_context or request.GET.get(LOAD_FAMILY_CONTEXT_PARAM) == 'true': families = Family.objects.filter(guid__in=family_guids) add_families_context( - response, families, project_guid=project.guid if project else None, user=request.user, is_analyst=is_analyst, + response, families, project_guid=project.guid if project else None, user=request.user, is_analyst=user_is_analyst(request.user), has_case_review_perm=bool(project) and has_case_review_permissions(project, request.user), include_igv=include_igv, ) From f974aaf7031b4d3e45c2e3e8478419d7f3d5e492 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:41:46 -0400 Subject: [PATCH 092/155] update discovery tests --- seqr/views/apis/saved_variant_api_tests.py | 75 ++++++++-------------- seqr/views/utils/variant_utils.py | 7 +- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 79c3a2ac19..52778af81d 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -185,6 +185,26 @@ def test_saved_variant_data(self): self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) + discovery_tags = [{ + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0003_tes', + 'familyGuid': 'F000012_12', + 'projectGuid': 'R0003_test', + }, + 'tagGuid': 'VT1726961_2103343353_r0003_tes', + 'name': 'Tier 1 - Novel gene and phenotype', + 'category': 'CMG Discovery Tags', + 'color': '#03441E', + 'searchHash': None, + 'searchName': None, + 'metadata': None, + 'lastModifiedDate': '2018-05-29T16:32:51.449Z', + 'createdBy': None, + }] + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) + self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 1) + self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) + tag = response_json['variantTagsByGuid']['VT1708633_2103343353_r0390_100'] self.assertSetEqual(set(tag.keys()), TAG_FIELDS) self.assertDictEqual(tag, { @@ -228,7 +248,9 @@ def test_saved_variant_data(self): 'spliceOutliers': {}, }}) - self.assertDictEqual(response_json['familiesByGuid'], {'F000001_1': {'tpmGenes': ['ENSG00000135953']}}) + self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000012_12'}) + self.assertSetEqual(set(response_json['familiesByGuid']['F000012_12'].keys()), FAMILY_FIELDS) + self.assertDictEqual(response_json['familiesByGuid']['F000001_1'], {'tpmGenes': ['ENSG00000135953']}) self.assertDictEqual(response_json['omimIntervals'], {}) @@ -266,7 +288,7 @@ def test_saved_variant_data(self): family_context_response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), family_context_response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) - self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2'}) + self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2', 'F000012_12'}) family_fields = {'individualGuids', 'tpmGenes'} family_fields.update(FAMILY_FIELDS) self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) @@ -317,7 +339,7 @@ def test_saved_variant_data(self): response = self.client.get(url.replace(PROJECT_GUID, 'R0003_test')) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), no_families_response_keys) + self.assertSetEqual(set(response_json.keys()), {*no_families_response_keys, 'familiesByGuid'}) self.assertSetEqual( set(response_json['savedVariantsByGuid'].keys()), @@ -333,51 +355,6 @@ def test_saved_variant_data(self): }}) self.assertDictEqual(response_json['rnaSeqData'], {}) - # Test cross-project discovery for analyst users - self.login_analyst_user() - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) - self.assertSetEqual( - set(response_json['savedVariantsByGuid'].keys()), - {'SV0000002_1248367227_r0390_100', VARIANT_GUID} - ) - variants = response_json['variantsById'] - self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) - discovery_tags = [{ - 'savedVariant': { - 'variantGuid': 'SV0000006_1248367227_r0003_tes', - 'familyGuid': 'F000012_12', - 'projectGuid': 'R0003_test', - }, - 'tagGuid': 'VT1726961_2103343353_r0003_tes', - 'name': 'Tier 1 - Novel gene and phenotype', - 'category': 'CMG Discovery Tags', - 'color': '#03441E', - 'searchHash': None, - 'searchName': None, - 'metadata': None, - 'lastModifiedDate': '2018-05-29T16:32:51.449Z', - 'createdBy': None, - }] - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) - self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000012_12'}) - self.assertSetEqual(set(response_json['familiesByGuid']['F000012_12'].keys()), FAMILY_FIELDS) - self.assertDictEqual(response_json['familiesByGuid']['F000001_1'], {'tpmGenes': ['ENSG00000135953']}) - - # Test discovery tags with family context - response = self.client.get(load_family_context_url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), family_context_response_keys) - variants = response_json['variantsById'] - self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) - self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2', 'F000012_12'}) - # Test empty project empty_project_url = url.replace(PROJECT_GUID, 'R0002_empty') response = self.client.get(empty_project_url) @@ -397,7 +374,7 @@ def test_saved_variant_data(self): self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals', 'transcriptsById'}) self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), fields) self.assertSetEqual(set(response_json['variantsById']['1-248367227-TC-T'].keys()), { - *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', + *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'noAccessDiscoveryFamilies', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 041e6cafdf..a170f008d3 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -351,10 +351,14 @@ def _get_family_has_rna_tpm(family_genes, gene_ids, sample_family_map): def _parse_discovery_tags(variants_by_id, family_guids, user): + keys = set() discovery_family_guids = set() for variant_id, variant in variants_by_id.items(): + if not variant['key']: + continue discovery_families = set(variant.pop('discoveryFamilies')) - set(variant['familyGuids']) if discovery_families: + keys.add(variant['key']) discovery_family_guids.update(discovery_families) variant['discoveryTags'] = [] variant['noAccessDiscoveryFamilies'] = len(discovery_families) @@ -369,8 +373,7 @@ def _parse_discovery_tags(variants_by_id, family_guids, user): saved_variants_by_guid = { sv['variantGuid']: sv for sv in SavedVariant.objects.filter( - key__in={variant['key'] for variant in variants}, - family__guid__in=discovery_families_by_guid, + key__in=keys, family__guid__in=discovery_families_by_guid, ).values('id', 'variant_id', variantGuid=F('guid'), familyGuid=F('family__guid'), projectGuid=F('family__project__guid')) } From 85d6e34903223de4dd7fe9f4282be12a6e4a30dd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:52:40 -0400 Subject: [PATCH 093/155] fix dataset type key conflicts --- seqr/views/utils/variant_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a170f008d3..a4bc22e30e 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -351,14 +351,14 @@ def _get_family_has_rna_tpm(family_genes, gene_ids, sample_family_map): def _parse_discovery_tags(variants_by_id, family_guids, user): - keys = set() + discovery_variant_ids = set() discovery_family_guids = set() for variant_id, variant in variants_by_id.items(): if not variant['key']: continue discovery_families = set(variant.pop('discoveryFamilies')) - set(variant['familyGuids']) if discovery_families: - keys.add(variant['key']) + discovery_variant_ids.add(variant_id) discovery_family_guids.update(discovery_families) variant['discoveryTags'] = [] variant['noAccessDiscoveryFamilies'] = len(discovery_families) @@ -373,7 +373,7 @@ def _parse_discovery_tags(variants_by_id, family_guids, user): saved_variants_by_guid = { sv['variantGuid']: sv for sv in SavedVariant.objects.filter( - key__in=keys, family__guid__in=discovery_families_by_guid, + variant_id__in=discovery_variant_ids, family__guid__in=discovery_families_by_guid, ).values('id', 'variant_id', variantGuid=F('guid'), familyGuid=F('family__guid'), projectGuid=F('family__project__guid')) } From 0208ccaf5d4d321e18b61ef618eb8391d0870606 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 15:55:55 -0400 Subject: [PATCH 094/155] fix anvil call test --- seqr/views/apis/saved_variant_api_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 52778af81d..7b4ff9f13e 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -356,6 +356,7 @@ def test_saved_variant_data(self): self.assertDictEqual(response_json['rnaSeqData'], {}) # Test empty project + self.login_analyst_user() empty_project_url = url.replace(PROJECT_GUID, 'R0002_empty') response = self.client.get(empty_project_url) self.assertEqual(response.status_code, 200) @@ -1159,9 +1160,9 @@ def test_saved_variant_data(self, *args): mock.ANY, 'ext-data', 'empty') self.mock_get_ws_access_level.assert_called_with( mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, 18) - self.mock_get_groups.assert_has_calls([mock.call(self.collaborator_user), mock.call(self.analyst_user)]) - self.assertEqual(self.mock_get_groups.call_count, 12) + self.assertEqual(self.mock_get_ws_access_level.call_count, 15) + self.mock_get_groups.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_get_groups.call_count, 1) self.mock_get_ws_acl.assert_not_called() self.mock_get_group_members.assert_not_called() From 6711ed3ac235e13cb60472862b65045f9975942e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 16:00:10 -0400 Subject: [PATCH 095/155] test fix --- clickhouse_search/all_search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 805dbc7afe..cc6f34d642 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1459,6 +1459,7 @@ def _cached_lookup_variant(self, variant, expected_individuals=None): }) return { **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, + 'discoveryFamilies': [], 'familyGenotypes': { family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) for family_guid, gts in family_genotypes.items() @@ -1507,7 +1508,6 @@ def test_get_single_variant(self): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json()['variantsById'], {'7-143270172-A-G': GRCH37_VARIANT}) - self.mock_redis.get.assert_not_called() self.mock_redis.set.assert_not_called() def test_frequency_filter(self): From c6e8609b2f0e0c1e421ec25bdae706ad36979aa4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 May 2026 17:32:21 -0400 Subject: [PATCH 096/155] todo --- seqr/views/apis/variant_search_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index f9e70e5b26..d952f642b1 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -596,6 +596,7 @@ def variant_lookup_handler(request): variant['familyGuids'] = list(families.values_list('guid', flat=True)) saved_variants = _get_saved_variant_models(variants) if families else None + # TODO move discovery family parsing up so not removed or add flag for tracking response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, From 661661cb51dbfc60c289bb9f582b0957893cc532 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 13:25:31 -0400 Subject: [PATCH 097/155] fix tests --- clickhouse_search/all_search_tests.py | 3 ++- seqr/views/apis/summary_data_api_tests.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index cc6f34d642..fdf54286e1 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1508,7 +1508,8 @@ def test_get_single_variant(self): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json()['variantsById'], {'7-143270172-A-G': GRCH37_VARIANT}) - self.mock_redis.set.assert_not_called() + self.assertTrue(all(call.args[0].startswith('projects__') for call in self.mock_redis.get.mock_calls)) + self.assertTrue(all(call.args[0].startswith('projects__') for call in self.mock_redis.set.mock_calls)) def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index d3a1b909c4..30588fb038 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -841,7 +841,7 @@ def test_mme_details(self, *args): def test_saved_variants_page(self): super(AnvilSummaryDataAPITest, self).test_saved_variants_page() assert_has_expected_calls(self, [ - self.no_access_user, self.manager_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user + self.no_access_user, self.manager_user, self.manager_user, self.manager_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') From dbed3240b0610d7433e3ce6e3b9a8e18080ea224 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 13:29:02 -0400 Subject: [PATCH 098/155] fix no access project bug --- seqr/views/utils/variant_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a4bc22e30e..202b764e0a 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -356,7 +356,7 @@ def _parse_discovery_tags(variants_by_id, family_guids, user): for variant_id, variant in variants_by_id.items(): if not variant['key']: continue - discovery_families = set(variant.pop('discoveryFamilies')) - set(variant['familyGuids']) + discovery_families = set(variant.pop('discoveryFamilies')) - set(variant.get('familyGuids', [])) if discovery_families: discovery_variant_ids.add(variant_id) discovery_family_guids.update(discovery_families) From 5901b35cd06b8358e0bea9817c29534d205b1e17 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 13:34:32 -0400 Subject: [PATCH 099/155] test search context --- clickhouse_search/all_search_tests.py | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index fdf54286e1..46791c473c 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -2482,14 +2482,7 @@ def test_search_context(self): self.assertEqual(response.status_code, 200) variants = response.json()['variantsById'] self.assertEqual(len(variants), 1) - self.assertFalse('discoveryTags' in variants['1-248367227-TC-T']) - - self.login_analyst_user() - response, _, _ = self._execute_search(search_hash=9876) - self.assertEqual(response.status_code, 200) - variants = response.json()['variantsById'] - self.assertEqual(len(variants), 1) - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{ + discovery_tag = { 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', 'familyGuid': 'F000012_12', @@ -2504,7 +2497,26 @@ def test_search_context(self): 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, - }]) + } + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{ + **discovery_tag, + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0004_non', + 'familyGuid': 'F000014_14', + 'projectGuid': 'R0004_non_analyst_project', + }, + 'tagGuid': 'VT1726961_2103343353_r0005_tes', + }, discovery_tag]) + self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 0) + self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY, 'F000014_14': mock.ANY}) + + self.login_analyst_user() + response, _, _ = self._execute_search(search_hash=9876) + self.assertEqual(response.status_code, 200) + variants = response.json()['variantsById'] + self.assertEqual(len(variants), 1) + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [discovery_tag]) + self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 1) self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) def test_cached_query_variants(self): From 557efd78d70b5a622460c799f1848907c763360e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 13:55:36 -0400 Subject: [PATCH 100/155] return discovery family mapping in lookup --- clickhouse_search/all_search_tests.py | 47 +++++++++++++-------------- seqr/views/apis/variant_search_api.py | 13 +++++--- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 46791c473c..0a9287dd2d 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -440,7 +440,6 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, @@ -1031,23 +1030,23 @@ def test_variant_lookup(self, mock_liftover): 'liftedFamilyGuids': ['F2_1-10439-AC-A'], 'genotypes': { 'I0_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, ], 'I1_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, ], 'I2_F0_1-10439-AC-A': [ - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, ], 'I0_F1_1-10439-AC-A': [ - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, ], 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], - 'sampleType': 'WGS'}, + 'sampleType': 'WGS', 'hasDiscoveryTag': False}, }, } expected_individuals = { @@ -1089,36 +1088,36 @@ def test_variant_lookup(self, mock_liftover): **GCNV_VARIANT4, 'familyGuids': ['F0_suffix_140608_DUP'], 'genotypes': { - 'I0_F0_suffix_140608_DUP': { + 'I0_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_suffix_140608_DUP': { + }}, + 'I1_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_suffix_140608_DUP': { + }}, + 'I2_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, + }}, }, } expected_sv_variant = { **SV_VARIANT4, 'familyGuids': ['F0_phase2_DEL_chr14_4640'], 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': { + 'I0_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_phase2_DEL_chr14_4640': { + }}, + 'I1_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_phase2_DEL_chr14_4640': { + }}, + 'I2_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, + }}, }, } expected_sv_individuals = { @@ -1356,10 +1355,10 @@ def test_variant_lookup(self, mock_liftover): no_access_variant = { **GRCH37_VARIANT, 'familyGuids': ['F0_7-143270172-A-G'], - 'genotypes': {mapped_guid: { + 'genotypes': {mapped_guid: {'hasDiscoveryTag': False, **{ k: v for k, v in GRCH37_VARIANT['genotypes'][guid].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - } for guid, mapped_guid in { + }} for guid, mapped_guid in { 'I000004_hg00731': 'I1_F0_7-143270172-A-G', 'I000006_hg00733': 'I0_F0_7-143270172-A-G', }.items()} } diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index d952f642b1..d106fc7aee 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -587,16 +587,17 @@ def variant_lookup_handler(request): family_guids = set() for variant in variants: family_guids.update(variant['familyGenotypes'].keys()) + family_guids.update(variant['discoveryFamilies']) - families = Family.objects.filter( + family_guids = set(Family.objects.filter( guid__in=family_guids, project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), - ) + ).values_list('guid', flat=True)) for variant in variants: - variant['familyGuids'] = list(families.values_list('guid', flat=True)) + variant['familyGuids'] = family_guids.intersection(variant['familyGenotypes'].keys()) + variant['lookupDiscoveryFamilies'] = set(variant['discoveryFamilies']) - family_guids - saved_variants = _get_saved_variant_models(variants) if families else None - # TODO move discovery family parsing up so not removed or add flag for tracking + saved_variants = _get_saved_variant_models(variants) if family_guids else None response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, @@ -624,6 +625,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['genotypes'] = {} variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] + discovery_families = variant.pop('lookupDiscoveryFamilies') for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): genotype.pop('metadata', None) @@ -649,6 +651,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) individual = genotype.pop('metadata', {}) + genotype = {**genotype, 'hasDiscoveryTag': unmapped_family_guid in discovery_families} if individual_key in individual_key_map: individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] From 895bda5f26ac10efd9680fe0a18427e9622e0164 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 13:58:14 -0400 Subject: [PATCH 101/155] fix flappy test --- clickhouse_search/all_search_tests.py | 4 ++-- seqr/views/utils/orm_to_json_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 0a9287dd2d..0704db448d 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -2497,7 +2497,7 @@ def test_search_context(self): 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, } - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{ + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [discovery_tag, { **discovery_tag, 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0004_non', @@ -2505,7 +2505,7 @@ def test_search_context(self): 'projectGuid': 'R0004_non_analyst_project', }, 'tagGuid': 'VT1726961_2103343353_r0005_tes', - }, discovery_tag]) + }]) self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 0) self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY, 'F000014_14': mock.ANY}) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 7f5b98c938..c1e24ec699 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -371,9 +371,9 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta for savedvariant_id, tag_id in tag_cls.saved_variants.through.objects.filter( savedvariant_id__in=saved_variant_id_map.keys()).values_list( 'savedvariant_id', f'{tag_cls.__name__.lower()}_id', - ): + ).order_by(): variant_tag_id_map[tag_id].append(savedvariant_id) - tag_models = tag_cls.objects.filter(id__in=variant_tag_id_map.keys()) + tag_models = tag_cls.objects.filter(id__in=variant_tag_id_map.keys()).order_by('id') if tag_filter: tag_models = tag_models.filter(**tag_filter) From 8a3fe70232a93f0e3ef1433eabc61c6cf3228d5e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 15:12:21 -0400 Subject: [PATCH 102/155] clean up --- clickhouse_search/all_search_tests.py | 1 + seqr/views/utils/orm_to_json_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 0704db448d..022ce31c09 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -440,6 +440,7 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) + self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index c1e24ec699..579006024c 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -371,7 +371,7 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta for savedvariant_id, tag_id in tag_cls.saved_variants.through.objects.filter( savedvariant_id__in=saved_variant_id_map.keys()).values_list( 'savedvariant_id', f'{tag_cls.__name__.lower()}_id', - ).order_by(): + ): variant_tag_id_map[tag_id].append(savedvariant_id) tag_models = tag_cls.objects.filter(id__in=variant_tag_id_map.keys()).order_by('id') if tag_filter: From df659871fdd258dc57cc3541c5c7b3a8c0b85176 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 16:47:40 -0400 Subject: [PATCH 103/155] test discovery fixtures --- clickhouse_search/all_search_tests.py | 2 +- .../fixtures/clickhouse_saved_variants.json | 12 +++++----- .../fixtures/clickhouse_search.json | 24 ------------------- seqr/fixtures/report_variants.json | 2 +- 4 files changed, 8 insertions(+), 32 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 022ce31c09..8d62a1c709 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -109,7 +109,7 @@ def setUpTestData(cls): class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' - fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_saved_variants', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): self.MOCK_CACHE = {} diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 69197c9a56..0cd5005d98 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -59,7 +59,7 @@ "pathogenicity": "Uncertain_significance" } }, { - "model": "clickhouse_search.entriesgrch37snvindel", + "model": "clickhouse_search.entriessnvindel", "pk": 100, "fields": { "key": 100, @@ -119,7 +119,7 @@ "transcripts": [] } }, { - "model": "clickhouse_search.entriesgrch37snvindel", + "model": "clickhouse_search.entriessnvindel", "pk": 101, "fields": { "key": 101, @@ -305,16 +305,16 @@ } }, { "model": "clickhouse_search.keylookupmito", - "pk": 100, + "pk": 8, "fields": { - "key": 100, + "key": 8, "variant_id": "M-14783-T-C" } }, { "model": "clickhouse_search.variantsmito", - "pk": 100, + "pk": 8, "fields": { - "key": 100, + "key": 8, "variant_id": "M-14783-T-C", "rsid": "rs193302982", "lifted_over_pos": 14783, diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 82508b5e00..207645a64e 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -1054,24 +1054,6 @@ ] ] } -}, { - "model": "clickhouse_search.variantsmito", - "pk": 8, - "fields": { - "key": 8, - "variant_id": "M-14783-T-C", - "rsid": "rs193302982", - "lifted_over_pos": 14783, - "common_low_heteroplasmy": true, - "mitotip": null, - "haplogroup_defining": true, - "sorted_transcript_consequences": [ - [ - "L", "protein_coding", 1, "Tta/Cta", ["synonymous_variant"], "ENSG00000198727", "ENST00000361789.2:c.37T>C", - "ENSP00000354554.2:p.Leu13=", [null, []], "synonymous_variant", "ENST00000361789", 0 - ] - ] - } }, { "model": "clickhouse_search.keylookupmito", "pk": "M-4429-G-A", @@ -1084,12 +1066,6 @@ "fields": { "key": 7 } -}, { - "model": "clickhouse_search.keylookupmito", - "pk": "M-14783-T-C", - "fields": { - "key": 8 - } }, { "model": "clickhouse_search.clinvarallvariantsmito", "pk": "M-4429-G-A", diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index b35a39fce8..4b65b096ce 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -77,7 +77,7 @@ "ref": "T", "alt": "C", "variant_id": "M-14783-T-C", - "key": 100, + "key": 8, "dataset_type": "MITO", "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} From 02be7cbc2bab475896712caa627bd63e14425919 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 May 2026 17:49:04 -0400 Subject: [PATCH 104/155] update lookup discovery fixtures --- clickhouse_search/all_search_tests.py | 72 +++++++++++++++++++ .../fixtures/clickhouse_saved_variants.json | 30 +++++++- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 8d62a1c709..4c5f15ac86 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1332,6 +1332,78 @@ def test_variant_lookup(self, mock_liftover): ] self.assert_json_logs(self.manager_user, unmapped_sample_logs) + self.login_analyst_user() + discovery_variant = { + 'key': 100, + 'variantId': '1-248367227-TC-T', + 'chrom': '1', + 'pos': 248367227, + 'ref': 'TC', + 'alt': 'T', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'xpos': 1248367227, + 'rsid': None, + 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 10, 'gq': 99, 'ab': 0.5, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', + 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', + 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 60, 'gq': 20, 'ab': 0.0, 'filters': [], + }, + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 7, 'hom': 2, 'ac_wes': 0, 'ac_wgs': 7, 'hom_wes': 0, 'hom_wgs': 2}, + 'seqr_affected': {'ac': 4, 'hom': 2}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + }, + 'predictions': { + 'cadd': None, + 'eigen': None, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + 'absplice': None, + 'pext': None, + 'promoter_ai': None, + }, + 'transcripts': {}, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'mainTranscriptId': None, + 'selectedMainTranscriptId': None, + 'CAID': None, + 'noAccessDiscoveryFamilies': 1, + } + self.maxDiff = None + self._assert_expected_lookup('1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38') + # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() self.reset_logs() diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 0cd5005d98..abb26d633b 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -180,6 +180,15 @@ "variant_id": "1-248367227-TC-T" } }, { + "model": "clickhouse_search.variantssnvindel", + "pk": 100, + "fields": { + "key": 100, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "sorted_transcript_consequences": [] + } +},{ "model": "clickhouse_search.variantdetailssnvindel", "pk": 100, "fields": { @@ -195,7 +204,7 @@ } }, { "model": "clickhouse_search.entriessnvindel", - "pk": 100, + "pk": 1000, "fields": { "key": 100, "project_guid": "R0004_non_analyst_project", @@ -211,6 +220,25 @@ ["NA21234", 2, 99, 0, 49] ] } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 1001, + "fields": { + "key": 100, + "project_guid": "R0003_test", + "family_guid": "F000012_12", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], + "filters": [], + "sign": 1, + "calls": [ + ["NA19675", 1, 99, 0.5555556, 9], + ["NA20885", 1, 99, 0.0, 71] + ] + } }, { "model": "clickhouse_search.keylookupsv", "pk": 101, From 6d128f071188447fa7ece73df6050f75176df5b2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 09:35:35 -0400 Subject: [PATCH 105/155] discovery lookup test fixture --- clickhouse_search/all_search_tests.py | 44 +++++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 4c5f15ac86..01f8fa85b4 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1349,19 +1349,13 @@ def test_variant_lookup(self, mock_liftover): 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], 'genotypes': { 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', + 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 10, 'gq': 99, 'ab': 0.5, 'filters': [], + 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', - 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', - 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 60, 'gq': 20, 'ab': 0.0, 'filters': [], + 'I0_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], + 'hasDiscoveryTag': True, }, }, 'clinvar': None, @@ -1399,10 +1393,34 @@ def test_variant_lookup(self, mock_liftover): 'mainTranscriptId': None, 'selectedMainTranscriptId': None, 'CAID': None, + 'discoveryTags': [], 'noAccessDiscoveryFamilies': 1, } + expected_individuals = { + **{guid: mock.ANY for guid in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', 'I000020_na20870', + ]}, + 'I0_F0_1-248367227-TC-T': { + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I0_F0_1-248367227-TC-T', + 'affected': 'A', + 'sex': 'F', + 'features': [], + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + } self.maxDiff = None - self._assert_expected_lookup('1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38') + self._assert_expected_lookup( + '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000012_12'], expected_individuals=expected_individuals, + mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, + savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0003_tes': mock.ANY}, + variantNotesByGuid={'VN0714935_2103343353_r0390_100': mock.ANY, 'VN0714937_2103343353_r0390_100': mock.ANY}, + variantTagsByGuid={ + 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0003_tes': mock.ANY, + 'VT1726970_2103343353_r0004_tes': mock.ANY, 'VT1726985_2103343353_r0390_100': mock.ANY, + }, + ) # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() @@ -1527,7 +1545,7 @@ def _cached_lookup_variant(self, variant, expected_individuals=None): family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] family_genotypes[family_guid].append({ **{k: v for k, v in gt.items() if k != 'individualGuid'}, - 'metadata': self.INDIVIDUAL_METADATA.get(gt['individualGuid']), + 'metadata': self.INDIVIDUAL_METADATA.get(gt.get('individualGuid')), }) return { **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, From 3c3cc8c164faabbfe55a09f57ec6ab77bc9d21b8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 10:04:55 -0400 Subject: [PATCH 106/155] cached variant formatting --- clickhouse_search/all_search_tests.py | 79 +++++++++++++------ .../fixtures/clickhouse_saved_variants.json | 4 +- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 01f8fa85b4..8b14d7c6bd 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1333,7 +1333,7 @@ def test_variant_lookup(self, mock_liftover): self.assert_json_logs(self.manager_user, unmapped_sample_logs) self.login_analyst_user() - discovery_variant = { + base_discovery_variant = { 'key': 100, 'variantId': '1-248367227-TC-T', 'chrom': '1', @@ -1349,13 +1349,25 @@ def test_variant_lookup(self, mock_liftover): 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], 'genotypes': { 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', - 'familyGuid': 'F000002_2', + 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], }, - 'I0_F0_1-248367227-TC-T': { - 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], - 'hasDiscoveryTag': True, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], + }, + 'I000017_na20889': { + 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', 'individualGuid': 'I000017_na20889', + 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], + }, + 'I000016_na20888': { + 'sampleId': 'NA20888', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000016_na20888', + 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], + }, + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', 'individualGuid': 'I000018_na21234', + 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], }, }, 'clinvar': None, @@ -1363,7 +1375,7 @@ def test_variant_lookup(self, mock_liftover): 'screenRegionType': None, 'populations': { 'seqr': {'ac': 7, 'hom': 2, 'ac_wes': 0, 'ac_wgs': 7, 'hom_wes': 0, 'hom_wgs': 2}, - 'seqr_affected': {'ac': 4, 'hom': 2}, + 'seqr_affected': {'ac': 6, 'hom': 2}, 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, @@ -1393,26 +1405,41 @@ def test_variant_lookup(self, mock_liftover): 'mainTranscriptId': None, 'selectedMainTranscriptId': None, 'CAID': None, + } + discovery_variant = { + **base_discovery_variant, 'discoveryTags': [], 'noAccessDiscoveryFamilies': 1, - } - expected_individuals = { - **{guid: mock.ANY for guid in [ - 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', 'I000020_na20870', - ]}, - 'I0_F0_1-248367227-TC-T': { - 'familyGuid': 'F0_1-248367227-TC-T', - 'individualGuid': 'I0_F0_1-248367227-TC-T', - 'affected': 'A', - 'sex': 'F', - 'features': [], - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'genotypes': { + # TODO why are some genotypes missing?? + **{guid: gt for guid, gt in base_discovery_variant['genotypes'].items() if guid not in {'I000006_hg00733', 'I000018_na21234'}}, + 'I0_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], + 'hasDiscoveryTag': True, + }, }, } + cached_discovery_variant = { + **base_discovery_variant, + 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], + } self.maxDiff = None self._assert_expected_lookup( '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', - project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000012_12'], expected_individuals=expected_individuals, + cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], + family_guids=['F000002_2', 'F000012_12'], expected_individuals={ + **{guid: mock.ANY for guid in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', 'I000020_na20870', + ]}, + 'I0_F0_1-248367227-TC-T': { + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I0_F0_1-248367227-TC-T', + 'affected': 'A', + 'sex': 'F', + 'features': [], + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + }, mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0003_tes': mock.ANY}, variantNotesByGuid={'VN0714935_2103343353_r0390_100': mock.ANY, 'VN0714937_2103343353_r0390_100': mock.ANY}, @@ -1478,6 +1505,14 @@ def test_variant_lookup(self, mock_liftover): 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', }, + 'I000016_na20888': { + 'affected': 'A', 'features': '', 'restrict_sharing': True, 'sex': 'M', + 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, + 'I000017_na20889': { + 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, + 'sex': 'F', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, 'I000018_na21234': { 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', @@ -1545,11 +1580,11 @@ def _cached_lookup_variant(self, variant, expected_individuals=None): family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] family_genotypes[family_guid].append({ **{k: v for k, v in gt.items() if k != 'individualGuid'}, - 'metadata': self.INDIVIDUAL_METADATA.get(gt.get('individualGuid')), + 'metadata': self.INDIVIDUAL_METADATA.get(gt['individualGuid']), }) return { - **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, 'discoveryFamilies': [], + **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': { family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) for family_guid, gts in family_genotypes.items() diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index abb26d633b..ce860bc9c7 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -235,8 +235,8 @@ "filters": [], "sign": 1, "calls": [ - ["NA19675", 1, 99, 0.5555556, 9], - ["NA20885", 1, 99, 0.0, 71] + ["NA20889", 1, 99, 0.0, 71], + ["NA20888", 1, 99, 0.5555556, 9] ] } }, { From 128b738f850965d7add50b1391962fd9779c4260 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 10:06:41 -0400 Subject: [PATCH 107/155] cached variant formatting --- clickhouse_search/all_search_tests.py | 61 +++++++++++++++------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 8b14d7c6bd..61e7d3b0cf 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1312,26 +1312,6 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) - # Test error handling when the ClickHouse sampleId cannot be mapped to any Postgres Individual - self.reset_logs() - Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') - missing_gt_variant = { - **GRCH37_VARIANT, 'genotypes': {k: v for k, v in GRCH37_VARIANT['genotypes'].items() if k != 'I000006_hg00733'}, - } - self._assert_expected_lookup( - '7-143270172-A-G', missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', - project_guids=['R0001_1kg'], family_guids=['F000002_2'], - individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] - ) - unmapped_sample_logs = [ - ('Looking up variant 7-143270172-A-G with data type SNV_INDEL', None), - ('Unable to map sample HG00733 in family F000002_2 to an individual for variant 7-143270172-A-G', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }), - ] - self.assert_json_logs(self.manager_user, unmapped_sample_logs) - self.login_analyst_user() base_discovery_variant = { 'key': 100, @@ -1349,15 +1329,18 @@ def test_variant_lookup(self, mock_liftover): 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], 'genotypes': { 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], }, 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], }, 'I000017_na20889': { - 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', 'individualGuid': 'I000017_na20889', + 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000017_na20889', 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], }, 'I000016_na20888': { @@ -1366,7 +1349,8 @@ def test_variant_lookup(self, mock_liftover): 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], }, 'I000018_na21234': { - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', 'individualGuid': 'I000018_na21234', + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', + 'individualGuid': 'I000018_na21234', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], }, }, @@ -1412,7 +1396,8 @@ def test_variant_lookup(self, mock_liftover): 'noAccessDiscoveryFamilies': 1, 'genotypes': { # TODO why are some genotypes missing?? - **{guid: gt for guid, gt in base_discovery_variant['genotypes'].items() if guid not in {'I000006_hg00733', 'I000018_na21234'}}, + **{guid: gt for guid, gt in base_discovery_variant['genotypes'].items() if + guid not in {'I000006_hg00733', 'I000018_na21234'}}, 'I0_F0_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], 'hasDiscoveryTag': True, @@ -1429,7 +1414,8 @@ def test_variant_lookup(self, mock_liftover): cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000012_12'], expected_individuals={ **{guid: mock.ANY for guid in [ - 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', 'I000020_na20870', + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', + 'I000020_na20870', ]}, 'I0_F0_1-248367227-TC-T': { 'familyGuid': 'F0_1-248367227-TC-T', @@ -1441,7 +1427,8 @@ def test_variant_lookup(self, mock_liftover): }, }, mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, - savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0003_tes': mock.ANY}, + savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, + 'SV0000006_1248367227_r0003_tes': mock.ANY}, variantNotesByGuid={'VN0714935_2103343353_r0390_100': mock.ANY, 'VN0714937_2103343353_r0390_100': mock.ANY}, variantTagsByGuid={ 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0003_tes': mock.ANY, @@ -1449,6 +1436,26 @@ def test_variant_lookup(self, mock_liftover): }, ) + # Test error handling when the ClickHouse sampleId cannot be mapped to any Postgres Individual + self.reset_logs() + Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') + missing_gt_variant = { + **GRCH37_VARIANT, 'genotypes': {k: v for k, v in GRCH37_VARIANT['genotypes'].items() if k != 'I000006_hg00733'}, + } + self._assert_expected_lookup( + '7-143270172-A-G', missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) + unmapped_sample_logs = [ + ('Looking up variant 7-143270172-A-G with data type SNV_INDEL', None), + ('Unable to map sample HG00733 in family F000002_2 to an individual for variant 7-143270172-A-G', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ] + self.assert_json_logs(self.manager_user, unmapped_sample_logs) + # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() self.reset_logs() From 9778412f7c54929eba11f51740b7d700e827c570 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 10:10:01 -0400 Subject: [PATCH 108/155] fix test order --- clickhouse_search/all_search_tests.py | 44 +++++++++------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 61e7d3b0cf..d36baeac7e 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1334,7 +1334,7 @@ def test_variant_lookup(self, mock_liftover): 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], }, 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', + 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], }, @@ -1348,11 +1348,6 @@ def test_variant_lookup(self, mock_liftover): 'individualGuid': 'I000016_na20888', 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], }, - 'I000018_na21234': { - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', - 'individualGuid': 'I000018_na21234', - 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], - }, }, 'clinvar': None, 'hgmd': None, @@ -1365,23 +1360,9 @@ def test_variant_lookup(self, mock_liftover): 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, }, 'predictions': { - 'cadd': None, - 'eigen': None, - 'fathmm': None, - 'gnomad_noncoding': None, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': None, - 'splice_ai_consequence': None, - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - 'absplice': None, - 'pext': None, - 'promoter_ai': None, + 'cadd': None, 'eigen': None, 'fathmm': None, 'gnomad_noncoding': None, 'mpc': None, 'mut_pred': None, + 'primate_ai': None, 'splice_ai': None, 'splice_ai_consequence': None, 'vest': None, 'mut_taster': None, + 'polyphen': None, 'revel': None, 'sift': None, 'absplice': None, 'pext': None, 'promoter_ai': None, }, 'transcripts': {}, 'sortedMotifFeatureConsequences': None, @@ -1395,9 +1376,7 @@ def test_variant_lookup(self, mock_liftover): 'discoveryTags': [], 'noAccessDiscoveryFamilies': 1, 'genotypes': { - # TODO why are some genotypes missing?? - **{guid: gt for guid, gt in base_discovery_variant['genotypes'].items() if - guid not in {'I000006_hg00733', 'I000018_na21234'}}, + **base_discovery_variant['genotypes'], 'I0_F0_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], 'hasDiscoveryTag': True, @@ -1407,8 +1386,14 @@ def test_variant_lookup(self, mock_liftover): cached_discovery_variant = { **base_discovery_variant, 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], + 'genotypes': { + **base_discovery_variant['genotypes'], + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', 'individualGuid': 'I000018_na21234', + 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], + }, + }, } - self.maxDiff = None self._assert_expected_lookup( '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], @@ -1427,8 +1412,7 @@ def test_variant_lookup(self, mock_liftover): }, }, mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, - savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, - 'SV0000006_1248367227_r0003_tes': mock.ANY}, + savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0003_tes': mock.ANY}, variantNotesByGuid={'VN0714935_2103343353_r0390_100': mock.ANY, 'VN0714937_2103343353_r0390_100': mock.ANY}, variantTagsByGuid={ 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0003_tes': mock.ANY, @@ -1454,7 +1438,7 @@ def test_variant_lookup(self, mock_liftover): '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', }), ] - self.assert_json_logs(self.manager_user, unmapped_sample_logs) + self.assert_json_logs(self.analyst_user, unmapped_sample_logs) # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() From aa4e819a7f97b322446d9a22ef346f37470ff7bb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 10:12:54 -0400 Subject: [PATCH 109/155] clean up global tags --- clickhouse_search/all_search_tests.py | 1 - seqr/views/apis/variant_search_api.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index d36baeac7e..e6ae13ee5e 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1374,7 +1374,6 @@ def test_variant_lookup(self, mock_liftover): discovery_variant = { **base_discovery_variant, 'discoveryTags': [], - 'noAccessDiscoveryFamilies': 1, 'genotypes': { **base_discovery_variant['genotypes'], 'I0_F0_1-248367227-TC-T': { diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index d106fc7aee..2c5b894b4f 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -626,6 +626,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] discovery_families = variant.pop('lookupDiscoveryFamilies') + variant.pop('noAccessDiscoveryFamilies', None) for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): genotype.pop('metadata', None) From 6b3ca8ad351b7ec77162381799e493ab48acc057 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 10:33:22 -0400 Subject: [PATCH 110/155] debug --- clickhouse_search/all_search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index e6ae13ee5e..09d9451346 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -424,6 +424,7 @@ def test_both_sample_types_search(self): dataset.sample_type = 'WGS' dataset.save() dataset.active_individuals.add(4) + self.maxDiff = None # Variant 1 is de novo in exome but inherited and homozygous in genome. # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome. @@ -440,7 +441,6 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, From 764c94c2c8240c400cae987d60f48eb80699f974 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 11:09:22 -0400 Subject: [PATCH 111/155] split fixture data --- clickhouse_search/all_search_tests.py | 2 +- .../clickhouse_discovery_variants.json | 88 +++++++++++++++++++ .../fixtures/clickhouse_saved_variants.json | 88 +------------------ .../fixtures/clickhouse_search.json | 24 +++++ seqr/fixtures/report_variants.json | 2 +- 5 files changed, 115 insertions(+), 89 deletions(-) create mode 100644 clickhouse_search/fixtures/clickhouse_discovery_variants.json diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 09d9451346..53f82f5b14 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -109,7 +109,7 @@ def setUpTestData(cls): class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' - fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_saved_variants', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): self.MOCK_CACHE = {} diff --git a/clickhouse_search/fixtures/clickhouse_discovery_variants.json b/clickhouse_search/fixtures/clickhouse_discovery_variants.json new file mode 100644 index 0000000000..266d845cc1 --- /dev/null +++ b/clickhouse_search/fixtures/clickhouse_discovery_variants.json @@ -0,0 +1,88 @@ +[{ + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [6, 48], + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 1, 99, 0.6530612111091614, 49], + ["HG00731", 2, 48, 1, 16] + ] + } +}, { + "model": "clickhouse_search.keylookupsnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T" + } +}, { + "model": "clickhouse_search.variantssnvindel", + "pk": 100, + "fields": { + "key": 100, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "sorted_transcript_consequences": [] + } +},{ + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": "CA1501729", + "lifted_over_chrom": "1", + "lifted_over_pos": null, + "transcripts": [], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0004_non_analyst_project", + "family_guid": "F000014_14", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], + "filters": [], + "sign": 1, + "calls": [ + ["NA21234", 2, 99, 0, 49] + ] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0003_test", + "family_guid": "F000012_12", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], + "filters": [], + "sign": 1, + "calls": [ + ["NA20889", 1, 99, 0.0, 71], + ["NA20888", 1, 99, 0.5555556, 9] + ] + } +} +] \ No newline at end of file diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index ce860bc9c7..fd1d18bc21 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -58,25 +58,6 @@ "assertions": [], "pathogenicity": "Uncertain_significance" } -}, { - "model": "clickhouse_search.entriessnvindel", - "pk": 100, - "fields": { - "key": 100, - "project_guid": "R0001_1kg", - "family_guid": "F000002_2", - "sample_type": "WGS", - "xpos": 1248367227, - "is_gnomad_gt_5_percent": false, - "is_annotated_in_any_gene": true, - "geneId_ids": [6, 48], - "filters": [], - "sign": 1, - "calls": [ - ["HG00733", 1, 99, 0.6530612111091614, 49], - ["HG00731", 2, 48, 1, 16] - ] - } }, { "model": "clickhouse_search.keylookupgrch37snvindel", "pk": 101, @@ -119,7 +100,7 @@ "transcripts": [] } }, { - "model": "clickhouse_search.entriessnvindel", + "model": "clickhouse_search.entriesgrch37snvindel", "pk": 101, "fields": { "key": 101, @@ -172,73 +153,6 @@ "sift": null, "vest": null } -}, { - "model": "clickhouse_search.keylookupsnvindel", - "pk": 100, - "fields": { - "key": 100, - "variant_id": "1-248367227-TC-T" - } -}, { - "model": "clickhouse_search.variantssnvindel", - "pk": 100, - "fields": { - "key": 100, - "sorted_motif_feature_consequences": [], - "sorted_regulatory_feature_consequences": [], - "sorted_transcript_consequences": [] - } -},{ - "model": "clickhouse_search.variantdetailssnvindel", - "pk": 100, - "fields": { - "key": 100, - "variant_id": "1-248367227-TC-T", - "rsid": null, - "caid": "CA1501729", - "lifted_over_chrom": "1", - "lifted_over_pos": null, - "transcripts": [], - "sorted_motif_feature_consequences": [], - "sorted_regulatory_feature_consequences": [] - } -}, { - "model": "clickhouse_search.entriessnvindel", - "pk": 1000, - "fields": { - "key": 100, - "project_guid": "R0004_non_analyst_project", - "family_guid": "F000014_14", - "sample_type": "WGS", - "xpos": 1248367227, - "is_gnomad_gt_5_percent": false, - "is_annotated_in_any_gene": false, - "geneId_ids": [], - "filters": [], - "sign": 1, - "calls": [ - ["NA21234", 2, 99, 0, 49] - ] - } -}, { - "model": "clickhouse_search.entriessnvindel", - "pk": 1001, - "fields": { - "key": 100, - "project_guid": "R0003_test", - "family_guid": "F000012_12", - "sample_type": "WGS", - "xpos": 1248367227, - "is_gnomad_gt_5_percent": false, - "is_annotated_in_any_gene": false, - "geneId_ids": [], - "filters": [], - "sign": 1, - "calls": [ - ["NA20889", 1, 99, 0.0, 71], - ["NA20888", 1, 99, 0.5555556, 9] - ] - } }, { "model": "clickhouse_search.keylookupsv", "pk": 101, diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 207645a64e..82508b5e00 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -1054,6 +1054,24 @@ ] ] } +}, { + "model": "clickhouse_search.variantsmito", + "pk": 8, + "fields": { + "key": 8, + "variant_id": "M-14783-T-C", + "rsid": "rs193302982", + "lifted_over_pos": 14783, + "common_low_heteroplasmy": true, + "mitotip": null, + "haplogroup_defining": true, + "sorted_transcript_consequences": [ + [ + "L", "protein_coding", 1, "Tta/Cta", ["synonymous_variant"], "ENSG00000198727", "ENST00000361789.2:c.37T>C", + "ENSP00000354554.2:p.Leu13=", [null, []], "synonymous_variant", "ENST00000361789", 0 + ] + ] + } }, { "model": "clickhouse_search.keylookupmito", "pk": "M-4429-G-A", @@ -1066,6 +1084,12 @@ "fields": { "key": 7 } +}, { + "model": "clickhouse_search.keylookupmito", + "pk": "M-14783-T-C", + "fields": { + "key": 8 + } }, { "model": "clickhouse_search.clinvarallvariantsmito", "pk": "M-4429-G-A", diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 4b65b096ce..b35a39fce8 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -77,7 +77,7 @@ "ref": "T", "alt": "C", "variant_id": "M-14783-T-C", - "key": 8, + "key": 100, "dataset_type": "MITO", "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} From 322b90da085f934875a188a3ff0b8ae630ec0be3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 11:13:17 -0400 Subject: [PATCH 112/155] more fixture fixes --- .../fixtures/clickhouse_saved_variants.json | 19 +++++++++++++++++++ .../reload_saved_variant_genotypes_tests.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index fd1d18bc21..b697f1340d 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -58,6 +58,25 @@ "assertions": [], "pathogenicity": "Uncertain_significance" } +}, { + "model": "clickhouse_search.entriesgrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [6, 48], + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 1, 99, 0.6530612111091614, 49], + ["HG00731", 2, 48, 1, 16] + ] + } }, { "model": "clickhouse_search.keylookupgrch37snvindel", "pk": 101, diff --git a/seqr/management/tests/reload_saved_variant_genotypes_tests.py b/seqr/management/tests/reload_saved_variant_genotypes_tests.py index 50ec2d6003..272f9a8cac 100644 --- a/seqr/management/tests/reload_saved_variant_genotypes_tests.py +++ b/seqr/management/tests/reload_saved_variant_genotypes_tests.py @@ -5,7 +5,7 @@ class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants', 'clickhouse_discovery_variants'] def test_command(self): # Update fixture data From ab3cb05dfbb39c957bb346c7b43f30ef9cb26bdd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 11:14:01 -0400 Subject: [PATCH 113/155] more fixture fixes --- clickhouse_search/fixtures/clickhouse_saved_variants.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index b697f1340d..c4a1113b6c 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -266,16 +266,16 @@ } }, { "model": "clickhouse_search.keylookupmito", - "pk": 8, + "pk": 100, "fields": { - "key": 8, + "key": 100, "variant_id": "M-14783-T-C" } }, { "model": "clickhouse_search.variantsmito", - "pk": 8, + "pk": 100, "fields": { - "key": 8, + "key": 100, "variant_id": "M-14783-T-C", "rsid": "rs193302982", "lifted_over_pos": 14783, From 5ceefe423bf05255fe22c49a6fb67a24ed4e87d8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 11:23:24 -0400 Subject: [PATCH 114/155] new fixture updates --- .../management/tests/set_saved_variant_key_tests.py | 2 +- .../tests/check_for_new_samples_from_pipeline_tests.py | 9 +++++---- seqr/views/apis/saved_variant_api_tests.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 032364f550..daf46291fa 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -11,7 +11,7 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] MOCK_GCNV_DATA = MOCK_GCNV_DATA diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 285175b6c3..ab7ef8b581 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -358,8 +358,9 @@ class CheckNewSamplesTest(object): 'updateType': 'bulk_update'}} ), ('Reloading saved variants in 2 projects', None), - ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000012_12', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 2 families for project Test Reprocessed Project', None), ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), ('Updated 1 variants in 1 families for project Non-Analyst Project', None), @@ -749,7 +750,7 @@ def test_command(self): class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'clickhouse_discovery_variants'] databases = '__all__' MOCK_DATA_DIR = '/seqr/seqr-hail-search-data' @@ -821,7 +822,7 @@ def _assert_expected_airtable_calls(self, *args, **kwargs): class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'clickhouse_discovery_variants'] airtable_samples_url = 'http://testairtable/app3Y97xtbbaOopVR/Samples' airtable_pdo_url = 'http://testairtable/app3Y97xtbbaOopVR/PDO' diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7b4ff9f13e..954ecb7d03 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -1132,7 +1132,7 @@ def test_update_variant_acmg_classification(self): # Tests for AnVIL access disabled class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] def assert_no_list_ws_has_al(self, acl_call_count): @@ -1145,7 +1145,7 @@ def assert_no_list_ws_has_al(self, acl_call_count): # Test for permissions from AnVIL only class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] @classmethod def setUpTestData(cls): From 3bfeaa7f4ab7ad09ee2ade1d7a2aa9518105c2bd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 11:27:49 -0400 Subject: [PATCH 115/155] fix fixture data --- clickhouse_search/all_search_tests.py | 6 +++--- .../fixtures/clickhouse_discovery_variants.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 53f82f5b14..0333b5fc3c 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1331,7 +1331,7 @@ def test_variant_lookup(self, mock_liftover): 'I000004_hg00731': { 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], + 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], }, 'I000006_hg00733': { 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', @@ -1353,8 +1353,8 @@ def test_variant_lookup(self, mock_liftover): 'hgmd': None, 'screenRegionType': None, 'populations': { - 'seqr': {'ac': 7, 'hom': 2, 'ac_wes': 0, 'ac_wgs': 7, 'hom_wes': 0, 'hom_wgs': 2}, - 'seqr_affected': {'ac': 6, 'hom': 2}, + 'seqr': {'ac': 5, 'hom': 1, 'ac_wes': 0, 'ac_wgs': 5, 'hom_wes': 0, 'hom_wgs': 1}, + 'seqr_affected': {'ac': 4, 'hom': 1}, 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, diff --git a/clickhouse_search/fixtures/clickhouse_discovery_variants.json b/clickhouse_search/fixtures/clickhouse_discovery_variants.json index 266d845cc1..83ff2e903d 100644 --- a/clickhouse_search/fixtures/clickhouse_discovery_variants.json +++ b/clickhouse_search/fixtures/clickhouse_discovery_variants.json @@ -14,7 +14,7 @@ "sign": 1, "calls": [ ["HG00733", 1, 99, 0.6530612111091614, 49], - ["HG00731", 2, 48, 1, 16] + ["HG00731", 0, 48, 1, 16] ] } }, { From 093a7254ed4c529ccf126e40200abdf73fed9857 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 12:00:10 -0400 Subject: [PATCH 116/155] shared helper --- clickhouse_search/search.py | 38 +++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2938bd87e0..e87a5f8b40 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -875,16 +875,28 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only=False, hom_only=False): +def _clickhouse_variants_lookup(entries, genome_version, data_type, format_results, affected_only=False, hom_only=False): variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] entries = _filter_lookup_entries(entries, affected_only, hom_only) entries = entries.result_values(additional_expressions=_lookup_genotype_expressions()) results = variants_cls.objects.subquery_join(entries) + + return format_results(results).result_values() + + +def _add_results_override_annotations(results): if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) + return results + + +def _clickhouse_variant_lookup(entries, genome_version, data_type, **kwargs): + results = _clickhouse_variants_lookup( + entries, genome_version, data_type, format_results=_add_results_override_annotations, **kwargs, + ) - variant = results.result_values().first() + variant = results.first() if variant: variant = format_clickhouse_results([variant])[0] return variant @@ -964,16 +976,18 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] padding = int((variant['end'] - variant['pos']) * 0.2) - entries = other_entry_class.objects.search_padded_interval( - variant['chrom'], variant['pos'], padding, additional_expressions=_lookup_genotype_expressions(), - ) - entries = _filter_lookup_entries(entries, affected_only, hom_only) - results = other_variants_cls.objects.subquery_join(entries).search( - padded_interval_end=(variant['end'], padding), **other_variants_cls.objects.get_parsed_annotations_filters( - annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - ), - ) - variants += list(results.result_values()) + entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) + + def format_results(results): + return results.search( + padded_interval_end=(variant['end'], padding), **results.get_parsed_annotations_filters( + annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + ), + ) + + variants += list(_clickhouse_variants_lookup( + entries, genome_version, other_sample_type, format_results, affected_only=affected_only, hom_only=hom_only, + )) return variants From 1fc1cf1554c26ac9eb60e98c09247b05bac0ea2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 12:39:44 -0400 Subject: [PATCH 117/155] handle exclude tags --- clickhouse_search/all_search_tests.py | 109 +++++++++++++++++++------- clickhouse_search/managers.py | 8 +- clickhouse_search/search.py | 18 +++-- seqr/views/apis/variant_search_api.py | 9 ++- 4 files changed, 104 insertions(+), 40 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 0333b5fc3c..d03ff08f61 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1031,23 +1031,23 @@ def test_variant_lookup(self, mock_liftover): 'liftedFamilyGuids': ['F2_1-10439-AC-A'], 'genotypes': { 'I0_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, ], 'I1_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, - {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, ], 'I2_F0_1-10439-AC-A': [ - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, ], 'I0_F1_1-10439-AC-A': [ - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False}, - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, ], 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], - 'sampleType': 'WGS', 'hasDiscoveryTag': False}, + 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, }, } expected_individuals = { @@ -1089,15 +1089,15 @@ def test_variant_lookup(self, mock_liftover): **GCNV_VARIANT4, 'familyGuids': ['F0_suffix_140608_DUP'], 'genotypes': { - 'I0_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ + 'I0_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, - 'I1_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ + 'I1_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, - 'I2_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, **{ + 'I2_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, @@ -1107,15 +1107,15 @@ def test_variant_lookup(self, mock_liftover): **SV_VARIANT4, 'familyGuids': ['F0_phase2_DEL_chr14_4640'], 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ + 'I0_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, - 'I1_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ + 'I1_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, - 'I2_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, **{ + 'I2_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }}, @@ -1170,17 +1170,17 @@ def test_variant_lookup(self, mock_liftover): 'I000015_na20885': 'I0_F1_1-10439-AC-A', 'I000018_na21234': 'I0_F2_1-10439-AC-A', } - expected_individuals = { + access_expected_individuals = { individual_guid: { **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, **{k: v for k, v in expected_individuals[anon_individual_guid].items() if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, } for individual_guid, anon_individual_guid in individual_guid_map.items() } - expected_individuals.update( + access_expected_individuals.update( {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) self._assert_expected_lookup( - '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, + '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=access_expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], family_guids=['F000002_2', 'F000011_11', 'F000014_14'], ) @@ -1312,7 +1312,6 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) - self.login_analyst_user() base_discovery_variant = { 'key': 100, 'variantId': '1-248367227-TC-T', @@ -1326,7 +1325,6 @@ def test_variant_lookup(self, mock_liftover): 'liftedOverPos': None, 'xpos': 1248367227, 'rsid': None, - 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], 'genotypes': { 'I000004_hg00731': { 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', @@ -1374,17 +1372,34 @@ def test_variant_lookup(self, mock_liftover): discovery_variant = { **base_discovery_variant, 'discoveryTags': [], + 'familyGuids': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], 'genotypes': { - **base_discovery_variant['genotypes'], 'I0_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], + 'hasDiscoveryTag': True, 'hasExcludedTag': True, + }, + 'I1_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], + 'hasDiscoveryTag': True, 'hasExcludedTag': True, + }, + 'I0_F1_1-248367227-TC-T': { + 'sampleType': 'WGS', 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], + 'hasDiscoveryTag': True, 'hasExcludedTag': False, + }, + 'I1_F1_1-248367227-TC-T': { + 'sampleType': 'WGS', 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], + 'hasDiscoveryTag': True, 'hasExcludedTag': False, + }, + 'I0_F2_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], - 'hasDiscoveryTag': True, + 'hasDiscoveryTag': True, 'hasExcludedTag': False, }, }, } cached_discovery_variant = { **base_discovery_variant, 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], + 'excludeTagFamilies': ['F000002_2'], 'genotypes': { **base_discovery_variant['genotypes'], 'I000018_na21234': { @@ -1393,6 +1408,48 @@ def test_variant_lookup(self, mock_liftover): }, }, } + expected_individuals = { + 'I0_F0_1-248367227-TC-T': { + **expected_individuals['I0_F0_1-10439-AC-A'], + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I0_F0_1-248367227-TC-T', + }, 'I1_F0_1-248367227-TC-T': { + **expected_individuals['I2_F0_1-10439-AC-A'], + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I1_F0_1-248367227-TC-T', + }, 'I0_F1_1-248367227-TC-T': { + **expected_individuals['I0_F1_1-10439-AC-A'], + 'sex': 'F', + 'familyGuid': 'F1_1-248367227-TC-T', + 'individualGuid': 'I0_F1_1-248367227-TC-T', + }, 'I1_F1_1-248367227-TC-T': { + **expected_individuals['I0_F1_1-10439-AC-A'], + 'features': [], + 'familyGuid': 'F1_1-248367227-TC-T', + 'individualGuid': 'I1_F1_1-248367227-TC-T', + }, 'I0_F2_1-248367227-TC-T': { + **expected_individuals['I0_F2_1-10439-AC-A'], + 'familyGuid': 'F2_1-248367227-TC-T', + 'individualGuid': 'I0_F2_1-248367227-TC-T', + } + } + self.login_base_user() + self._assert_expected_lookup( + '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', + cached_variants=[cached_discovery_variant], expected_individuals=expected_individuals, skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, locusListsByGuid={}, + ) + + self.login_analyst_user() + discovery_variant = { + **discovery_variant, + 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], + 'genotypes': { + **base_discovery_variant['genotypes'], + 'I0_F0_1-248367227-TC-T': discovery_variant['genotypes']['I0_F2_1-248367227-TC-T'], + }, + } self._assert_expected_lookup( '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], @@ -1402,12 +1459,9 @@ def test_variant_lookup(self, mock_liftover): 'I000020_na20870', ]}, 'I0_F0_1-248367227-TC-T': { + **expected_individuals['I0_F2_1-248367227-TC-T'], 'familyGuid': 'F0_1-248367227-TC-T', 'individualGuid': 'I0_F0_1-248367227-TC-T', - 'affected': 'A', - 'sex': 'F', - 'features': [], - 'vlmContactEmail': 'vlm@broadinstitute.org', }, }, mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, @@ -1463,7 +1517,7 @@ def test_variant_lookup(self, mock_liftover): no_access_variant = { **GRCH37_VARIANT, 'familyGuids': ['F0_7-143270172-A-G'], - 'genotypes': {mapped_guid: {'hasDiscoveryTag': False, **{ + 'genotypes': {mapped_guid: {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ k: v for k, v in GRCH37_VARIANT['genotypes'][guid].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} }} for guid, mapped_guid in { @@ -1574,6 +1628,7 @@ def _cached_lookup_variant(self, variant, expected_individuals=None): }) return { 'discoveryFamilies': [], + 'excludeTagFamilies': [], **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': { family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 27a503a476..a1f64006d7 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -197,9 +197,9 @@ def annotation_values(self): if field.db_column and field.name != field.db_column and field.db_column } - def result_values(self, additional_fields=None, **kwargs): + def result_values(self, additional_fields=None, additional_values=None, **kwargs): fields = [*self.annotation_fields] + (additional_fields or []) - values = {**self.annotation_values} + values = {**self.annotation_values, **(additional_values or {})} values.update(self.conditional_selects(self, **kwargs)) override_model_annotations = set(values).intersection(fields) @@ -1478,9 +1478,9 @@ def _filter_locations_q(self, intervals, genes=None): def _can_filter_gene_interval(self, genes): return (not hasattr(self.model, 'geneId_ids')) or len(genes) < self.MAX_XPOS_FILTER_INTERVALS or self.filtered_chrom - def search_padded_interval(self, chrom, pos, padding, **kwargs): + def search_padded_interval(self, chrom, pos, padding): interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) - return self.filter(interval_q).result_values(**kwargs) + return self.filter(interval_q) @staticmethod def _interval_query(chrom, start, end, **kwargs): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index e87a5f8b40..42d5e88c26 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -14,7 +14,7 @@ ArrayMap, Modulo from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP -from clickhouse_search.models.postgres_dicts import SexDict, AffectedDict, IndividualMetadataDict +from clickhouse_search.models.postgres_dicts import SexDict, AffectedDict, IndividualMetadataDict, ExcludedVariantDict from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP @@ -882,15 +882,15 @@ def _clickhouse_variants_lookup(entries, genome_version, data_type, format_resul entries = entries.result_values(additional_expressions=_lookup_genotype_expressions()) results = variants_cls.objects.subquery_join(entries) - return format_results(results).result_values() - + return format_results(results).result_values(additional_values={ + 'excludeTagFamilies': ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + }) def _add_results_override_annotations(results): if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) return results - def _clickhouse_variant_lookup(entries, genome_version, data_type, **kwargs): results = _clickhouse_variants_lookup( entries, genome_version, data_type, format_results=_add_results_override_annotations, **kwargs, @@ -973,11 +973,8 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() if dt != data_type and dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) ) - other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] padding = int((variant['end'] - variant['pos']) * 0.2) - entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) - def format_results(results): return results.search( padded_interval_end=(variant['end'], padding), **results.get_parsed_annotations_filters( @@ -985,6 +982,7 @@ def format_results(results): ), ) + entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) variants += list(_clickhouse_variants_lookup( entries, genome_version, other_sample_type, format_results, affected_only=affected_only, hom_only=hom_only, )) @@ -1000,10 +998,14 @@ def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression(additional_expressions=_lookup_genotype_expressions()) - lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) + lifted_entry_data = lifted_entries.values('key').annotate( + excludeTagFamilies=ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + **{gt_field: GroupArrayArray(gt_expr)}, + ) if lifted_entry_data: variant['familyGenotypes'].update(lifted_entry_data[0]['familyGenotypes']) variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) + variant['excludeTagFamilies'] += lifted_entry_data[0]['excludeTagFamilies'] def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, additional_fields=None): diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 2c5b894b4f..37362d91ac 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -588,6 +588,7 @@ def variant_lookup_handler(request): for variant in variants: family_guids.update(variant['familyGenotypes'].keys()) family_guids.update(variant['discoveryFamilies']) + family_guids.update(variant['excludeTagFamilies']) family_guids = set(Family.objects.filter( guid__in=family_guids, @@ -596,6 +597,7 @@ def variant_lookup_handler(request): for variant in variants: variant['familyGuids'] = family_guids.intersection(variant['familyGenotypes'].keys()) variant['lookupDiscoveryFamilies'] = set(variant['discoveryFamilies']) - family_guids + variant['excludeTagFamilies'] = set(variant['excludeTagFamilies']) - family_guids saved_variants = _get_saved_variant_models(variants) if family_guids else None response = get_variants_response( @@ -626,6 +628,7 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] discovery_families = variant.pop('lookupDiscoveryFamilies') + exclude_tag_families = variant.pop('excludeTagFamilies') variant.pop('noAccessDiscoveryFamilies', None) for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): @@ -652,7 +655,11 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) individual = genotype.pop('metadata', {}) - genotype = {**genotype, 'hasDiscoveryTag': unmapped_family_guid in discovery_families} + genotype = { + **genotype, + 'hasDiscoveryTag': unmapped_family_guid in discovery_families, + 'hasExcludedTag': unmapped_family_guid in exclude_tag_families, + } if individual_key in individual_key_map: individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] From a52eee104353e60eef600a5226f01d89c7d8e7ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 13:23:52 -0400 Subject: [PATCH 118/155] use actual db query for context discovery tags --- clickhouse_search/all_search_tests.py | 108 +++++++------------------- clickhouse_search/test_utils.py | 52 ++++++++++++- 2 files changed, 78 insertions(+), 82 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index d03ff08f61..d129b24d92 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -34,7 +34,7 @@ VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ - MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, \ + MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, DISCOVERY_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, \ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant @@ -1312,65 +1312,8 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) - base_discovery_variant = { - 'key': 100, - 'variantId': '1-248367227-TC-T', - 'chrom': '1', - 'pos': 248367227, - 'ref': 'TC', - 'alt': 'T', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': None, - 'liftedOverPos': None, - 'xpos': 1248367227, - 'rsid': None, - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', - 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', - 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], - }, - 'I000017_na20889': { - 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', - 'individualGuid': 'I000017_na20889', - 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], - }, - 'I000016_na20888': { - 'sampleId': 'NA20888', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', - 'individualGuid': 'I000016_na20888', - 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], - }, - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 5, 'hom': 1, 'ac_wes': 0, 'ac_wgs': 5, 'hom_wes': 0, 'hom_wgs': 1}, - 'seqr_affected': {'ac': 4, 'hom': 1}, - 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - }, - 'predictions': { - 'cadd': None, 'eigen': None, 'fathmm': None, 'gnomad_noncoding': None, 'mpc': None, 'mut_pred': None, - 'primate_ai': None, 'splice_ai': None, 'splice_ai_consequence': None, 'vest': None, 'mut_taster': None, - 'polyphen': None, 'revel': None, 'sift': None, 'absplice': None, 'pext': None, 'promoter_ai': None, - }, - 'transcripts': {}, - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'CAID': None, - } discovery_variant = { - **base_discovery_variant, + **DISCOVERY_VARIANT, 'discoveryTags': [], 'familyGuids': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], 'genotypes': { @@ -1397,11 +1340,21 @@ def test_variant_lookup(self, mock_liftover): }, } cached_discovery_variant = { - **base_discovery_variant, + **DISCOVERY_VARIANT, 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], 'excludeTagFamilies': ['F000002_2'], 'genotypes': { - **base_discovery_variant['genotypes'], + **DISCOVERY_VARIANT['genotypes'], + 'I000017_na20889': { + 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000017_na20889', + 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], + }, + 'I000016_na20888': { + 'sampleId': 'NA20888', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000016_na20888', + 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], + }, 'I000018_na21234': { 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', 'individualGuid': 'I000018_na21234', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], @@ -1446,10 +1399,11 @@ def test_variant_lookup(self, mock_liftover): **discovery_variant, 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], 'genotypes': { - **base_discovery_variant['genotypes'], + **cached_discovery_variant['genotypes'], 'I0_F0_1-248367227-TC-T': discovery_variant['genotypes']['I0_F2_1-248367227-TC-T'], }, } + del discovery_variant['genotypes']['I000018_na21234'] self._assert_expected_lookup( '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], @@ -2644,14 +2598,7 @@ def test_search_context(self): '3': {'chrom': '1', 'end': 249055991, 'mimNumber': 600315, 'phenotypeDescription': '?Immunodeficiency 16', 'phenotypeInheritance': 'Autosomal recessive', 'phenotypeMimNumber': 615120, 'start': 249044482}, }) - # Test cross-project discovery for analyst users - self.set_cache('search_results__VRS0009876__xpos',[{ - 'key': 100, 'familyGuids': ['F000002_2'], 'xpos': 1248367227, 'genomeVersion': '38', 'sortedTranscriptConsequences': [], - }]) - response, _, _ = self._execute_search(search_hash=9876) - self.assertEqual(response.status_code, 200) - variants = response.json()['variantsById'] - self.assertEqual(len(variants), 1) + # Test cross-project discovery tags discovery_tag = { 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', @@ -2668,7 +2615,9 @@ def test_search_context(self): 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, } - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [discovery_tag, { + self._add_sample_type_samples('WGS', active_individuals__family__guid='F000002_2') + self._assert_expected_search( + [{**DISCOVERY_VARIANT, 'noAccessDiscoveryFamilies': 0, 'discoveryTags': [discovery_tag, { **discovery_tag, 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0004_non', @@ -2676,18 +2625,15 @@ def test_search_context(self): 'projectGuid': 'R0004_non_analyst_project', }, 'tagGuid': 'VT1726961_2103343353_r0005_tes', - }]) - self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 0) - self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY, 'F000014_14': mock.ANY}) + }]}], locus={'rawVariantItems': '1-248367227-TC-T'}, additional_response={ + 'familiesByGuid': {'F000012_12': mock.ANY, 'F000014_14': mock.ANY}, + }) self.login_analyst_user() - response, _, _ = self._execute_search(search_hash=9876) - self.assertEqual(response.status_code, 200) - variants = response.json()['variantsById'] - self.assertEqual(len(variants), 1) - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [discovery_tag]) - self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 1) - self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) + self._assert_expected_search( + [{**DISCOVERY_VARIANT, 'noAccessDiscoveryFamilies': 1, 'discoveryTags': [discovery_tag]}], + locus={'rawVariantItems': '1-248367227-TC-T'}, additional_response={'familiesByGuid': {'F000012_12': mock.ANY}}, + ) def test_cached_query_variants(self): search_hash = 987 diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 249782d113..fc60017d04 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1205,6 +1205,55 @@ 'CAID': None, } +DISCOVERY_VARIANT = { + 'key': 100, + 'variantId': '1-248367227-TC-T', + 'chrom': '1', + 'pos': 248367227, + 'ref': 'TC', + 'alt': 'T', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'xpos': 1248367227, + 'rsid': None, + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', + 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], + }, + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 5, 'hom': 1, 'ac_wes': 0, 'ac_wgs': 5, 'hom_wes': 0, 'hom_wgs': 1}, + 'seqr_affected': {'ac': 4, 'hom': 1}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + }, + 'predictions': { + 'cadd': None, 'eigen': None, 'fathmm': None, 'gnomad_noncoding': None, 'mpc': None, 'mut_pred': None, + 'primate_ai': None, 'splice_ai': None, 'splice_ai_consequence': None, 'vest': None, 'mut_taster': None, + 'polyphen': None, 'revel': None, 'sift': None, 'absplice': None, 'pext': None, 'promoter_ai': None, + }, + 'transcripts': {}, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'mainTranscriptId': None, + 'selectedMainTranscriptId': None, + 'CAID': None, +} + CACHED_CONSEQUENCES_BY_KEY = {1: [], 2: [{ 'alphamissensePathogenicity': 0.99779, 'canonical': 1, @@ -1356,6 +1405,7 @@ 'geneId': 'ENSG00000268903', 'isManeSelect': False, }], +100: [], } def format_cached_variant(variant): @@ -1365,7 +1415,7 @@ def format_cached_variant(variant): **{k: v for k, v in variant.items() if k not in [ 'mainTranscriptId', 'selectedMainTranscriptId', 'transcripts', 'CAID', 'chrom', 'pos', 'ref', 'alt', 'liftedOverChrom', 'liftedOverPos', 'rsid', 'variantId', 'sortedMotifFeatureConsequences', - 'sortedRegulatoryFeatureConsequences', + 'sortedRegulatoryFeatureConsequences', 'noAccessDiscoveryFamilies', 'discoveryTags', ]}, 'sortedTranscriptConsequences': CACHED_CONSEQUENCES_BY_KEY[variant['key']], } From 7be5139d23d90663b7b18a48881ad57189e5c4d4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 14:32:03 -0400 Subject: [PATCH 119/155] show no access discovery families --- .../components/panel/variants/Variants.jsx | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/ui/shared/components/panel/variants/Variants.jsx b/ui/shared/components/panel/variants/Variants.jsx index 350e3d8532..3a45d33ed9 100644 --- a/ui/shared/components/panel/variants/Variants.jsx +++ b/ui/shared/components/panel/variants/Variants.jsx @@ -1,5 +1,6 @@ import React from 'react' import PropTypes from 'prop-types' +import { NavLink } from 'react-router-dom' import styled from 'styled-components' import { Grid, Popup, Label, Button, Header, Tab } from 'semantic-ui-react' @@ -139,6 +140,29 @@ export const Variant = React.memo(( { variant, mainGeneId, reads, showReads, dispatch, isCompoundHet, updateReads, ...props }, ) => { const variantMainGeneId = mainGeneId || getVariantMainGeneId(variant) + const discoveryTags = [] + if (variant.discoveryTags?.length > 0) { + discoveryTags.push( + + ) + } + if (variant.noAccessDiscoveryFamilies > 0) { + discoveryTags.push( + + + ) + } return ( - {variant.discoveryTags && variant.discoveryTags.length > 0 && ( + {discoveryTags.length > 0 && ( Other Project Discovery Tags} - content={} + content={discoveryTags} /> )}
From f57191564163b53487c70f781da9958fdfb7c1c4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 14:57:58 -0400 Subject: [PATCH 120/155] restructure lookup tags response --- clickhouse_search/all_search_tests.py | 64 ++++++++++++++------------- clickhouse_search/search.py | 6 +-- seqr/views/apis/variant_search_api.py | 22 ++++----- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index d129b24d92..26fa1a865f 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1031,23 +1031,22 @@ def test_variant_lookup(self, mock_liftover): 'liftedFamilyGuids': ['F2_1-10439-AC-A'], 'genotypes': { 'I0_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, ], 'I1_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, - {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, ], 'I2_F0_1-10439-AC-A': [ - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, ], 'I0_F1_1-10439-AC-A': [ - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, ], - 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], - 'sampleType': 'WGS', 'hasDiscoveryTag': False, 'hasExcludedTag': False}, + 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, }, } expected_individuals = { @@ -1089,36 +1088,36 @@ def test_variant_lookup(self, mock_liftover): **GCNV_VARIANT4, 'familyGuids': ['F0_suffix_140608_DUP'], 'genotypes': { - 'I0_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + 'I0_F0_suffix_140608_DUP': { k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, - 'I1_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + }, + 'I1_F0_suffix_140608_DUP': { k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, - 'I2_F0_suffix_140608_DUP': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + }, + 'I2_F0_suffix_140608_DUP': { k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, + }, }, } expected_sv_variant = { **SV_VARIANT4, 'familyGuids': ['F0_phase2_DEL_chr14_4640'], 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + 'I0_F0_phase2_DEL_chr14_4640': { k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, - 'I1_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + }, + 'I1_F0_phase2_DEL_chr14_4640': { k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, - 'I2_F0_phase2_DEL_chr14_4640': {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + }, + 'I2_F0_phase2_DEL_chr14_4640': { k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }}, + }, }, } expected_sv_individuals = { @@ -1237,6 +1236,8 @@ def test_variant_lookup(self, mock_liftover): **VARIANT1, 'familyGuids': [], 'lookupFamilyGuids': VARIANT1['familyGuids'], + 'discoveryTagFamilies': [], + 'excludedTagFamilies': [], }}) cache_key = 'variant_lookup_results__7-143270172-A-G__37' @@ -1316,33 +1317,30 @@ def test_variant_lookup(self, mock_liftover): **DISCOVERY_VARIANT, 'discoveryTags': [], 'familyGuids': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], + 'discoveryTagFamilies': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], + 'excludedTagFamilies': ['F0_1-248367227-TC-T'], 'genotypes': { 'I0_F0_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], - 'hasDiscoveryTag': True, 'hasExcludedTag': True, }, 'I1_F0_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], - 'hasDiscoveryTag': True, 'hasExcludedTag': True, }, 'I0_F1_1-248367227-TC-T': { 'sampleType': 'WGS', 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], - 'hasDiscoveryTag': True, 'hasExcludedTag': False, }, 'I1_F1_1-248367227-TC-T': { 'sampleType': 'WGS', 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], - 'hasDiscoveryTag': True, 'hasExcludedTag': False, }, 'I0_F2_1-248367227-TC-T': { 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], - 'hasDiscoveryTag': True, 'hasExcludedTag': False, }, }, } cached_discovery_variant = { **DISCOVERY_VARIANT, 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], - 'excludeTagFamilies': ['F000002_2'], + 'excludedTagFamilies': ['F000002_2'], 'genotypes': { **DISCOVERY_VARIANT['genotypes'], 'I000017_na20889': { @@ -1398,6 +1396,8 @@ def test_variant_lookup(self, mock_liftover): discovery_variant = { **discovery_variant, 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], + 'discoveryTagFamilies': ['F0_1-248367227-TC-T'], + 'excludedTagFamilies': [], 'genotypes': { **cached_discovery_variant['genotypes'], 'I0_F0_1-248367227-TC-T': discovery_variant['genotypes']['I0_F2_1-248367227-TC-T'], @@ -1471,10 +1471,10 @@ def test_variant_lookup(self, mock_liftover): no_access_variant = { **GRCH37_VARIANT, 'familyGuids': ['F0_7-143270172-A-G'], - 'genotypes': {mapped_guid: {'hasDiscoveryTag': False, 'hasExcludedTag': False, **{ + 'genotypes': {mapped_guid: { k: v for k, v in GRCH37_VARIANT['genotypes'][guid].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }} for guid, mapped_guid in { + } for guid, mapped_guid in { 'I000004_hg00731': 'I1_F0_7-143270172-A-G', 'I000006_hg00733': 'I0_F0_7-143270172-A-G', }.items()} } @@ -1555,6 +1555,8 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, 'variantsById': {v['variantId']: { + 'discoveryTagFamilies': [], + 'excludedTagFamilies': [], **v, 'familyGuids': [], 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []), @@ -1582,7 +1584,7 @@ def _cached_lookup_variant(self, variant, expected_individuals=None): }) return { 'discoveryFamilies': [], - 'excludeTagFamilies': [], + 'excludedTagFamilies': [], **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': { family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 42d5e88c26..c7bae80842 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -883,7 +883,7 @@ def _clickhouse_variants_lookup(entries, genome_version, data_type, format_resul results = variants_cls.objects.subquery_join(entries) return format_results(results).result_values(additional_values={ - 'excludeTagFamilies': ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + 'excludedTagFamilies': ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), }) def _add_results_override_annotations(results): @@ -999,13 +999,13 @@ def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression(additional_expressions=_lookup_genotype_expressions()) lifted_entry_data = lifted_entries.values('key').annotate( - excludeTagFamilies=ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + excludedTagFamilies=ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), **{gt_field: GroupArrayArray(gt_expr)}, ) if lifted_entry_data: variant['familyGenotypes'].update(lifted_entry_data[0]['familyGenotypes']) variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) - variant['excludeTagFamilies'] += lifted_entry_data[0]['excludeTagFamilies'] + variant['excludedTagFamilies'] += lifted_entry_data[0]['excludedTagFamilies'] def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, additional_fields=None): diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 37362d91ac..722b89e8f2 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -588,7 +588,7 @@ def variant_lookup_handler(request): for variant in variants: family_guids.update(variant['familyGenotypes'].keys()) family_guids.update(variant['discoveryFamilies']) - family_guids.update(variant['excludeTagFamilies']) + family_guids.update(variant['excludedTagFamilies']) family_guids = set(Family.objects.filter( guid__in=family_guids, @@ -596,8 +596,8 @@ def variant_lookup_handler(request): ).values_list('guid', flat=True)) for variant in variants: variant['familyGuids'] = family_guids.intersection(variant['familyGenotypes'].keys()) - variant['lookupDiscoveryFamilies'] = set(variant['discoveryFamilies']) - family_guids - variant['excludeTagFamilies'] = set(variant['excludeTagFamilies']) - family_guids + variant['discoveryTagFamilies'] = set(variant['discoveryFamilies']) - family_guids + variant['excludedTagFamilies'] = set(variant['excludedTagFamilies']) - family_guids saved_variants = _get_saved_variant_models(variants) if family_guids else None response = get_variants_response( @@ -627,8 +627,6 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['genotypes'] = {} variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] - discovery_families = variant.pop('lookupDiscoveryFamilies') - exclude_tag_families = variant.pop('excludeTagFamilies') variant.pop('noAccessDiscoveryFamilies', None) for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): @@ -646,8 +644,10 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): variant['genotypes'][individual_guid] = genotype all_feature_ids = set() + family_guid_map = {} for i, (unmapped_family_guid, genotypes) in enumerate(sorted(variant.pop('familyGenotypes').items())): family_guid = f'F{i}_{variant["variantId"]}' + family_guid_map[unmapped_family_guid] = family_guid variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): variant['liftedFamilyGuids'][variant['liftedFamilyGuids'].index(unmapped_family_guid)] = family_guid @@ -655,11 +655,6 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) individual = genotype.pop('metadata', {}) - genotype = { - **genotype, - 'hasDiscoveryTag': unmapped_family_guid in discovery_families, - 'hasExcludedTag': unmapped_family_guid in exclude_tag_families, - } if individual_key in individual_key_map: individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] @@ -676,6 +671,13 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): } variant['genotypes'][individual_guid] = genotype + variant['discoveryTagFamilies'] = sorted([ + family_guid_map[family_guid] for family_guid in variant['discoveryTagFamilies'] if family_guid in family_guid_map + ]) + variant['excludedTagFamilies'] = sorted([ + family_guid_map[family_guid] for family_guid in variant['excludedTagFamilies'] if family_guid in family_guid_map + ]) + _parse_hpo_terms(all_feature_ids, response['individualsByGuid'].values()) From 8720ebf416434946907a9acda1daebd538bfaaf7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 15:19:20 -0400 Subject: [PATCH 121/155] display lookup tags --- ui/pages/Search/VariantLookup.jsx | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 4b9145a4bf..466e069d1a 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -193,6 +193,34 @@ InternalFamily.propTypes = { showReads: PropTypes.object, } +const ExternalFamily = ({ familyGuid, variant }) => { + const { discoveryTagFamilies = [], excludedTagFamilies = [] } = variant + const tags = [] + if (discoveryTagFamilies.includes(familyGuid)) { + tags.push({ color: 'teal', content: 'Discovery Variant' }) + } + if (excludedTagFamilies.includes(familyGuid)) { + tags.push({ color: 'grey', content: 'Excluded' }) + } + return ( + + {tags.length > 0 && ( + + {tags.map(tag => + )} + + + + + ) +} + +ExternalFamily.propTypes = { + familyGuid: PropTypes.string, + variant: PropTypes.object, +} + const BaseLookupVariant = ({ variant, familiesByContactEmail, vlmDefaultContactEmails }) => { const { internal, disabled, ...familiesByContact } = familiesByContactEmail return ( @@ -211,18 +239,14 @@ const BaseLookupVariant = ({ variant, familiesByContactEmail, vlmDefaultContactE > {families.map(familyGuid => ( - - - - - + ))} ))} {(disabled || []).map(familyGuid => ( - + ))} From 12b35b030d3900b024c80268d9816347ec63aa5e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 15:29:12 -0400 Subject: [PATCH 122/155] better ui linitng --- .../components/panel/variants/Variants.jsx | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/ui/shared/components/panel/variants/Variants.jsx b/ui/shared/components/panel/variants/Variants.jsx index 3a45d33ed9..72afadcc72 100644 --- a/ui/shared/components/panel/variants/Variants.jsx +++ b/ui/shared/components/panel/variants/Variants.jsx @@ -140,29 +140,6 @@ export const Variant = React.memo(( { variant, mainGeneId, reads, showReads, dispatch, isCompoundHet, updateReads, ...props }, ) => { const variantMainGeneId = mainGeneId || getVariantMainGeneId(variant) - const discoveryTags = [] - if (variant.discoveryTags?.length > 0) { - discoveryTags.push( - - ) - } - if (variant.noAccessDiscoveryFamilies > 0) { - discoveryTags.push( - - - ) - } return ( - {discoveryTags.length > 0 && ( + {(variant.discoveryTags?.length > 0 || variant.noAccessDiscoveryFamilies > 0) && ( Other Project Discovery Tags} - content={discoveryTags} + content={( + + {variant.discoveryTags?.length > 0 && ( + + )} + {variant.noAccessDiscoveryFamilies > 0 && ( + + + )} + + )} /> )}
From ea3954cd66b5ea25eb39560b9c5ff590f416f84a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 16:51:32 -0400 Subject: [PATCH 123/155] return additional metadata lookup fields --- clickhouse_search/all_search_tests.py | 51 ++++++++++++++++----------- clickhouse_search/search.py | 5 ++- seqr/fixtures/1kg_project.json | 2 +- seqr/views/apis/variant_search_api.py | 6 ++++ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 805dbc7afe..90670f5869 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1053,7 +1053,7 @@ def test_variant_lookup(self, mock_liftover): expected_individuals = { 'I0_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', + 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', 'isSolved': False, 'disease': 'OMIM:615123', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I0_F1_1-10439-AC-A': { @@ -1062,20 +1062,20 @@ def test_variant_lookup(self, mock_liftover): 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', }, 'I0_F2_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], + 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], 'isSolved': True, 'disease': '', 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', }, 'I1_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', + 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', 'isSolved': False, 'disease': 'OMIM:615123', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I2_F0_1-10439-AC-A': { 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', 'features': [{'category': 'HP:0000707', 'label': 'Morphological abnormality of the central nervous system', 'id': 'HP:0002011'}, {'category': 'HP:0001626', 'label': 'Arrhythmia', 'id': 'HP:0011675'}], - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, 'disease': 'OMIM:615123', }, } cache_key = 'variant_lookup_results__1-10439-AC-A__38' @@ -1125,31 +1125,34 @@ def test_variant_lookup(self, mock_liftover): 'I0_F0_phase2_DEL_chr14_4640': { 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I0_F0_suffix_140608_DUP': { 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, 'I1_F0_phase2_DEL_chr14_4640': { 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I1_F0_suffix_140608_DUP': { 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, 'I2_F0_phase2_DEL_chr14_4640': { 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I2_F0_suffix_140608_DUP': { 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', 'features': [{'category': 'HP:0000707', 'label': 'Morphological abnormality of the central nervous system', 'id': 'HP:0002011'}, {'category': 'HP:0001626', 'label': 'Arrhythmia', 'id': 'HP:0011675'}], 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, } sv_genes = { @@ -1174,7 +1177,7 @@ def test_variant_lookup(self, mock_liftover): individual_guid: { **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, **{k: v for k, v in expected_individuals[anon_individual_guid].items() - if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, + if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail', 'isSolved', 'disease'}}, } for individual_guid, anon_individual_guid in individual_guid_map.items() } expected_individuals.update( @@ -1337,19 +1340,23 @@ def test_variant_lookup(self, mock_liftover): self.reset_logs() expected_individuals = {'I0_F0_7-143270172-A-G': { 'affected': 'N', + 'disease': 'OMIM:615123', 'familyGuid': 'F0_7-143270172-A-G', 'features': [], 'individualGuid': 'I0_F0_7-143270172-A-G', + 'isSolved': False, 'sex': 'F', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I1_F0_7-143270172-A-G': { 'affected': 'A', + 'disease': 'OMIM:615123', 'familyGuid': 'F0_7-143270172-A-G', 'features': [ {'category': 'HP:0000707', 'id': 'HP:0002011', 'label': 'Morphological abnormality of the central nervous system'}, {'category': 'HP:0001626', 'id': 'HP:0011675', 'label': 'Arrhythmia'}, ], 'individualGuid': 'I1_F0_7-143270172-A-G', + 'isSolved': False, 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }} @@ -1373,36 +1380,38 @@ def test_variant_lookup(self, mock_liftover): INDIVIDUAL_METADATA = { 'I000006_hg00733': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, }, 'I000005_hg00732': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': False, + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', }, 'I000004_hg00731': { 'affected': 'A', 'features': '[{"id": "HP:0002011"}, {"id": "HP:0011675"}]', 'restrict_sharing': False, - 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, + 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', }, 'I000015_na20885': { 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 0, 'mondo_id': '', 'isSolved': False, }, 'I000018_na21234': { - 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', }, 'I000019_na21987': { - 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', }, 'I000021_na21654': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', }, 'I000002_na19678': { - 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': False, + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'omim_id': 615123, 'mondo_id': '', }, } diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2938bd87e0..83f37e2cc2 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -900,13 +900,16 @@ def _lookup_genotype_expressions(): affected_expr = AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U') sex_expr = SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U') metadata_expr = IndividualMetadataDict.dict_get_sql( - key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'features', 'vlm_contact_email'], + key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'is_solved', 'omim_id', 'mondo_id', 'features', 'vlm_contact_email'], ) return { f'tupleConcat(({affected_expr}, {sex_expr}), {metadata_expr})': ('metadata', NamedTupleField([ ('affected', StringField()), ('sex', StringField()), ('restrict_sharing', BoolField()), + ('isSolved', BoolField()), + ('omim_id', StringField()), + ('mondo_id', StringField()), ('features', StringField()), ('vlmContactEmail', StringField()), ])), diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 888e93019e..144c9cf2d5 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -377,7 +377,7 @@ "last_modified_date": "2017-03-12T22:37:17.555Z", "project": 4, "family_id": "fam14", - "analysis_status": "Rncc", + "analysis_status": "S_ng", "success_story": "Differential treatement", "success_story_types": ["A", "D"] } diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index f9e70e5b26..96f1004c1a 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -656,6 +656,12 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): individual_key_map[individual_key] = individual_guid features = json.loads(individual['features']) if individual.get('features') else [] all_feature_ids.update([feature['id'] for feature in features]) + omim_id = individual.pop('omim_id') + mondo_id = individual.pop('mondo_id') + if individual.get('restrict_sharing'): + individual.pop('isSolved') + else: + individual['disease'] = f'OMIM:{omim_id}' if omim_id else mondo_id response['individualsByGuid'][individual_guid] = { **individual, 'familyGuid': family_guid, From 0d0da244d9b39209be23504f9176ce6723916070 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 May 2026 17:54:32 -0400 Subject: [PATCH 124/155] display additional fields --- .../panel/variants/VariantIndividuals.jsx | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ui/shared/components/panel/variants/VariantIndividuals.jsx b/ui/shared/components/panel/variants/VariantIndividuals.jsx index 0558bbafa3..72c2a29aa4 100644 --- a/ui/shared/components/panel/variants/VariantIndividuals.jsx +++ b/ui/shared/components/panel/variants/VariantIndividuals.jsx @@ -466,14 +466,31 @@ Genotype.propTypes = { genesById: PropTypes.object, } -const INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_FEATURES] +const INDIVIDUAL_FIELD_IS_SOLVED = 'isSolved' +const INDIVIDUAL_FIELD_DISEASE = 'disease' +const INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_DISEASE, INDIVIDUAL_FIELD_IS_SOLVED, INDIVIDUAL_FIELD_FEATURES] const VARIANT_INDIVIDUAL_DETAIL_FIELDS = [ INDIVIDUAL_FIELD_FILTER_FLAGS, INDIVIDUAL_FIELD_POP_FILTERS, ...INDIVIDUAL_DETAIL_FIELDS, ] const SV_INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_SV_FLAGS, ...INDIVIDUAL_DETAIL_FIELDS] +const VARIANT_INDIVIDUAL_FIELD_LOOKUP = { + ...INDIVIDUAL_FIELD_LOOKUP, + [INDIVIDUAL_FIELD_IS_SOLVED]: { + fieldDisplay: isSolved => (isSolved ?
), }, + }, { + [ENGLISH]: { + header: 'Q. Can I get assistance with processing my data?', + content: ( +
+ Yes - If you need support processing your data for seqr, such as generating a joint-called VCF from CRAM files + or reprocessing data using DRAGEN, the Broad Institute’s Data Science Services group offers fee-for-service + support. These services are designed to help prepare your data for successful ingestion into seqr and are + available to both internal Broad cohorts and external research groups. To discuss project scope, timelines, + and pricing, please reach out to bclservices@broadinstitute.org. +
+ ), + }, + [SPANISH]: { + header: 'P. ¿Puedo recibir asistencia para procesar mis datos?', + content: ( +
+ Sí. Si necesita asistencia para el procesamiento de sus datos destinados a seqr - como la generación de un + archivo VCF de llamada conjunta a partir de archivos CRAM, o el reprocesamiento de datos mediante DRAGEN - el + grupo de Servicios de Ciencia de Datos del Broad Institute ofrece soporte bajo la modalidad de pago por + servicio. Estos servicios están diseñados para ayudarle a preparar sus datos y asegurar su correcta ingesta en + seqr, y se encuentran a disposición tanto de las cohortes internas del Broad como de grupos de investigación + externos. Para conversar sobre el alcance del proyecto, los plazos y las tarifas, por favor póngase en + contacto con bclservices@broadinstitute.org. +
+ ), + }, }, { [ENGLISH]: { header: 'Q. Who has access to my data in seqr?', From 645d368cea747b89243f8640a8ba9459a9aa65cd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:36:52 -0400 Subject: [PATCH 134/155] use pyliftover --- vlm/clickhouse_utils.py | 6 ++---- vlm/match.py | 30 ++++++++++++++++-------------- vlm/requirements-test.in | 1 + vlm/requirements-test.txt | 2 ++ vlm/web_app.py | 26 +------------------------- 5 files changed, 22 insertions(+), 43 deletions(-) diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py index dd32573776..75dbb717fe 100644 --- a/vlm/clickhouse_utils.py +++ b/vlm/clickhouse_utils.py @@ -1,5 +1,4 @@ import clickhouse_connect -import hail as hl import os CLICKHOUSE_CONNECTION_PARAMS = { @@ -11,13 +10,12 @@ } -def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - locus = hl.eval(locus) +def get_clickhouse_variant_counts(chrom: str, pos: int, genome_build: str, ref: str, alt: str) -> tuple[int, int]: client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) results = client.query( "SELECT plus(gt_stats.1, gt_stats.2), plus(gt_stats.3, gt_stats.4) FROM (SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) AS gt_stats FROM %(table_name)s WHERE variantId=%(variant_id)s)", parameters={ - 'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}', + 'variant_id': f'{chrom}-{pos}-{ref}-{alt}', 'table_name': f'{genome_build}/SNV_INDEL/key_lookup', 'dict_name': f'{genome_build}/SNV_INDEL/gt_stats_dict', }, diff --git a/vlm/match.py b/vlm/match.py index 07cbe57e60..a0eced8f37 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -1,6 +1,6 @@ from aiohttp.web import HTTPBadRequest -import hail as hl import os +from pyliftover.liftover import LiftOver import re from vlm.clickhouse_utils import get_clickhouse_variant_counts @@ -8,7 +8,7 @@ SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') - +LIFTOVER_DIR = f'{os.path.dirname(os.path.abspath(__file__))}/liftover_references' BEACON_HANDOVER_TYPE = { 'id': NODE_ID, @@ -39,29 +39,31 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) - locus = hl.locus(chrom, pos, reference_genome=genome_build) - ac, hom = get_clickhouse_variant_counts(locus, ref, alt, genome_build) + ac, hom = get_clickhouse_variant_counts(chrom, pos, genome_build, ref, alt) - liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = get_clickhouse_variant_counts(liftover_locus, ref, alt, liftover_genome_build) + liftover = _liftover_variant(chrom, pos, genome_build) + lift_ac, lift_hom = get_clickhouse_variant_counts(*liftover, ref, alt) if liftover else (0, 0) url = _get_contact_url( - chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, + chrom, pos, ref, alt, genome_build, liftover if lift_ac and not ac else None, ) return _format_results(ac+lift_ac, hom+lift_hom, url) -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover_genome_build: str, liftover_locus: hl.LocusExpression) -> str: +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> tuple[str, int, str]: + liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 + lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') + lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None + + +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: tuple[str, int, str]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL if liftover_locus is not None: - lifted = hl.eval(liftover_locus) - chrom = lifted.contig - pos = lifted.position - genome_build = liftover_genome_build + chrom, pos, genome_build = liftover genome_build = genome_build.replace('GRCh', '') return f'{SEQR_BASE_URL}variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' @@ -93,7 +95,7 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: if not re.fullmatch(r'[ATCG]', allele): raise HTTPBadRequest(reason=f'Invalid {allele_field}: {allele}') - return chrom, start, query['referenceBases'], query['alternateBases'], genome_build + return chrom.replace('chr', ''), start, query['referenceBases'], query['alternateBases'], genome_build def _format_results(ac: int, hom: int, url: str) -> dict: diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index c63f835ebe..19660ea18c 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,4 +1,5 @@ aioresponses coverage<5.2 +pyliftover pytest-aiohttp clickhouse-connect==0.8.18 diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index ccd84ac4f7..6c9313db33 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -44,6 +44,8 @@ pluggy==1.2.0 # via pytest propcache==0.2.0 # via yarl +pyliftover==0.4 + # via -r vlm/requirements-test.in pytest==7.4.0 # via # pytest-aiohttp diff --git a/vlm/web_app.py b/vlm/web_app.py index 72875e5817..90a5594f0c 100644 --- a/vlm/web_app.py +++ b/vlm/web_app.py @@ -1,20 +1,12 @@ from aiohttp import web -import hail as hl import logging -import os import traceback from vlm.auth import authenticate -from vlm.match import get_variant_match, GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 +from vlm.match import get_variant_match logger = logging.getLogger(__name__) -JAVA_OPTS_XSS = os.environ.get('JAVA_OPTS_XSS') -MACHINE_MEM = os.environ.get('MACHINE_MEM') -JVM_MEMORY_FRACTION = 0.9 - -LIFTOVER_DIR = f'{os.path.dirname(os.path.abspath(__file__))}/liftover_references' - def _handle_exception(e, request): logger.error(f'{request.headers.get("From")} "{e}"') @@ -42,22 +34,6 @@ async def match(request: web.Request) -> web.Response: async def init_web_app(): - spark_conf = {} - # memory limits adapted from https://github.com/hail-is/hail/blob/main/hail/python/hailtop/hailctl/dataproc/start.py#L321C17-L321C36 - if MACHINE_MEM: - spark_conf['spark.driver.memory'] = f'{int((int(MACHINE_MEM) - 11) * JVM_MEMORY_FRACTION)}g' - if JAVA_OPTS_XSS: - spark_conf.update( - {f'spark.{field}.extraJavaOptions': f'-Xss{JAVA_OPTS_XSS}' for field in ['driver', 'executor']}) - hl.init(idempotent=True, spark_conf=spark_conf or None) - - rg37 = hl.get_reference(GENOME_VERSION_GRCh37) - rg38 = hl.get_reference(GENOME_VERSION_GRCh38) - if not rg37.has_liftover(rg38): - rg37.add_liftover(f'{LIFTOVER_DIR}/grch37_to_grch38.over.chain.gz', rg38) - if not rg38.has_liftover(rg37): - rg38.add_liftover(f'{LIFTOVER_DIR}/grch38_to_grch37.over.chain.gz', rg37) - app = web.Application(middlewares=[error_middleware], client_max_size=(1024 ** 2) * 10) app.add_routes([ web.get('/vlm/match', match), From 2917f4db91fc2e715493a9fc1e7a018199964682 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:41:43 -0400 Subject: [PATCH 135/155] remove hail validation --- vlm/match.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index a0eced8f37..a2cc29d2cb 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -37,6 +37,14 @@ 'hg19': GENOME_VERSION_GRCh37, } +CHROMOSOMES = { + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', + '20', '21', '22', 'X', 'Y', 'M', +} +MIN_POS = 1 +MAX_POS = 3e8 + + def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) @@ -78,16 +86,14 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: raise HTTPBadRequest(reason=f'Invalid assemblyId: {query["assemblyId"]}') chrom = query['referenceName'].replace('chr', '') - if genome_build == GENOME_VERSION_GRCh38: - chrom = f'chr{chrom}' - if not hl.eval(hl.is_valid_contig(chrom, reference_genome=genome_build)): + if chrom not in CHROMOSOMES: raise HTTPBadRequest(reason=f'Invalid referenceName: {query["referenceName"]}') start = query['start'] if not start.isnumeric(): raise HTTPBadRequest(reason=f'Invalid start: {start}') start = int(start) - if not hl.eval(hl.is_valid_locus(chrom, start, reference_genome=genome_build)): + if start < MIN_POS or start > MAX_POS: raise HTTPBadRequest(reason=f'Invalid start: {start}') for allele_field in ['referenceBases', 'alternateBases']: @@ -95,7 +101,7 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: if not re.fullmatch(r'[ATCG]', allele): raise HTTPBadRequest(reason=f'Invalid {allele_field}: {allele}') - return chrom.replace('chr', ''), start, query['referenceBases'], query['alternateBases'], genome_build + return chrom, start, query['referenceBases'], query['alternateBases'], genome_build def _format_results(ac: int, hom: int, url: str) -> dict: From 90a0dea641fce482abb1fe81e11a124ec20fd01b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:45:32 -0400 Subject: [PATCH 136/155] fix typo --- vlm/match.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index a2cc29d2cb..cfecb17488 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -49,7 +49,6 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) ac, hom = get_clickhouse_variant_counts(chrom, pos, genome_build, ref, alt) - liftover = _liftover_variant(chrom, pos, genome_build) lift_ac, lift_hom = get_clickhouse_variant_counts(*liftover, ref, alt) if liftover else (0, 0) @@ -70,7 +69,7 @@ def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str if not SEQR_BASE_URL: return SEQR_BASE_URL - if liftover_locus is not None: + if liftover is not None: chrom, pos, genome_build = liftover genome_build = genome_build.replace('GRCh', '') return f'{SEQR_BASE_URL}variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' From 0eb5ce307aca113c65bc69e45fce43f2175cf0e4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:50:40 -0400 Subject: [PATCH 137/155] debug --- vlm/test_vlm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 9b0a1f02e3..eccd2c9c16 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -42,6 +42,7 @@ async def test_match(self, mocked_responses): jwt_body = {'iss': 'https://vlm-auth.us.auth0.com/', 'azp': REQUESTER_CLIENT_ID} headers = {'Authorization': f'Bearer {jwt.encode(jwt_body, "")}'} + self.maxDiff = None async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', headers=headers) as resp: self.assertEqual(resp.status, 200) resp_json = await resp.json() From 9806d7a5f8d8c63ef6ce6a988300360c44c8744b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:56:26 -0400 Subject: [PATCH 138/155] fix test --- vlm/match.py | 2 +- vlm/test_vlm.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index cfecb17488..0a9a7dc743 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -62,7 +62,7 @@ def _liftover_variant(chrom: str, pos: int, genome_build: str) -> tuple[str, int liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) - return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None + return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: tuple[str, int, str]) -> str: diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index eccd2c9c16..02134c5e70 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -42,7 +42,6 @@ async def test_match(self, mocked_responses): jwt_body = {'iss': 'https://vlm-auth.us.auth0.com/', 'azp': REQUESTER_CLIENT_ID} headers = {'Authorization': f'Bearer {jwt.encode(jwt_body, "")}'} - self.maxDiff = None async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', headers=headers) as resp: self.assertEqual(resp.status, 200) resp_json = await resp.json() @@ -53,7 +52,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-38724419-T-G', 'email': None, } ], @@ -196,7 +195,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=7-143270172-A-G', 'email': None, } ], From 1f69b6b9643d195fc05c75efa102c9dd63907830 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 16:59:50 -0400 Subject: [PATCH 139/155] coverage report --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2199726d0e..991040c3ba 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -57,5 +57,5 @@ jobs: export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ - coverage report --fail-under=95 + coverage report -m --fail-under=95 From 24f2f457876707fe37fafd45386e1599c2edaa7f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:12:29 -0400 Subject: [PATCH 140/155] real requiment files --- vlm/requirements-test.in | 3 +- vlm/requirements-test.txt | 83 ++++++++++++++++++++++----------------- vlm/requirements.in | 3 ++ vlm/requirements.txt | 46 ++++++++++++++++++++++ 4 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 vlm/requirements.in create mode 100644 vlm/requirements.txt diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index 19660ea18c..e444f7d6f2 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,5 +1,4 @@ +-c vlm/requirements.txt aioresponses coverage<5.2 -pyliftover pytest-aiohttp -clickhouse-connect==0.8.18 diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index 6c9313db33..92782e1402 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -1,60 +1,73 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile vlm/requirements-test.in # -aiohappyeyeballs==2.3.5 - # via aiohttp -aiohttp==3.10.11 +aiohappyeyeballs==2.6.2 # via + # -c requirements.txt + # aiohttp +aiohttp==3.13.5 + # via + # -c requirements.txt # aioresponses # pytest-aiohttp aioresponses==0.7.8 - # via -r vlm/requirements-test.in -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -clickhouse-connect==0.8.18 - # via -r vlm/requirements-test.in + # via -r requirements-test.in +aiosignal==1.4.0 + # via + # -c requirements.txt + # aiohttp +attrs==26.1.0 + # via + # -c requirements.txt + # aiohttp coverage==5.1 - # via -r vlm/requirements-test.in -exceptiongroup==1.1.3 - # via pytest -frozenlist==1.3.3 + # via -r requirements-test.in +frozenlist==1.8.0 # via + # -c requirements.txt # aiohttp # aiosignal -idna==3.7 - # via yarl -iniconfig==2.0.0 +idna==3.15 + # via + # -c requirements.txt + # yarl +iniconfig==2.3.0 # via pytest -multidict==6.0.4 +multidict==6.7.1 # via + # -c requirements.txt # aiohttp # yarl -packaging==23.1 +packaging==26.2 # via # aioresponses # pytest -pluggy==1.2.0 +pluggy==1.6.0 # via pytest -propcache==0.2.0 - # via yarl -pyliftover==0.4 - # via -r vlm/requirements-test.in -pytest==7.4.0 +propcache==0.5.2 + # via + # -c requirements.txt + # aiohttp + # yarl +pygments==2.20.0 + # via pytest +pytest==9.0.3 # via # pytest-aiohttp # pytest-asyncio -pytest-aiohttp==1.0.4 - # via -r vlm/requirements-test.in -pytest-asyncio==0.21.0 +pytest-aiohttp==1.1.0 + # via -r requirements-test.in +pytest-asyncio==1.3.0 # via pytest-aiohttp -tomli==2.0.1 - # via pytest -yarl==1.17.2 - # via aiohttp +typing-extensions==4.15.0 + # via + # -c requirements.txt + # aiosignal + # pytest-asyncio +yarl==1.24.2 + # via + # -c requirements.txt + # aiohttp diff --git a/vlm/requirements.in b/vlm/requirements.in new file mode 100644 index 0000000000..b56e5a23f2 --- /dev/null +++ b/vlm/requirements.in @@ -0,0 +1,3 @@ +aiohttp +pyliftover +clickhouse-connect==0.8.18 diff --git a/vlm/requirements.txt b/vlm/requirements.txt new file mode 100644 index 0000000000..bbd63258e9 --- /dev/null +++ b/vlm/requirements.txt @@ -0,0 +1,46 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile vlm/requirements.in +# +aiohappyeyeballs==2.6.2 + # via aiohttp +aiohttp==3.13.5 + # via -r requirements.in +aiosignal==1.4.0 + # via aiohttp +attrs==26.1.0 + # via aiohttp +certifi==2026.5.20 + # via clickhouse-connect +clickhouse-connect==0.8.18 + # via -r requirements.in +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal +idna==3.15 + # via yarl +lz4==4.4.5 + # via clickhouse-connect +multidict==6.7.1 + # via + # aiohttp + # yarl +propcache==0.5.2 + # via + # aiohttp + # yarl +pyliftover==0.4.1 + # via -r requirements.in +pytz==2026.2 + # via clickhouse-connect +typing-extensions==4.15.0 + # via aiosignal +urllib3==2.7.0 + # via clickhouse-connect +yarl==1.24.2 + # via aiohttp +zstandard==0.25.0 + # via clickhouse-connect From f8c4e7fdd74c9ccde30c7e944877e6d99cae905b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:13:49 -0400 Subject: [PATCH 141/155] no hail dockerfile --- vlm/deploy/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index bda44c60eb..a05445da7f 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -1,11 +1,11 @@ -FROM hailgenetics/hail:0.2.128 +FROM python:3.11-slim-bullseye LABEL maintainer="Broad TGG" -RUN pip install --no-cache-dir clickhouse-connect==0.8.18 - WORKDIR /vlm +RUN pip install --no-cache-dir -r requirements.txt + # Application Code COPY vlm/ . From 6f5483d7ba277e2d067f6aed0db78e6a7f38f86a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:14:18 -0400 Subject: [PATCH 142/155] update test image; --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 991040c3ba..909f19c31d 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -21,7 +21,7 @@ on: jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 + container: python:3.11-slim-bullseye services: clickhouse: From a979ec8546913f52402ff3aaa7d1c71151bea04a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:18:43 -0400 Subject: [PATCH 143/155] fix requiremnts --- .github/workflows/vlm-unit-tests.yaml | 1 + vlm/requirements-test.txt | 26 +++++++++++++------------- vlm/requirements.txt | 6 +++--- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 909f19c31d..6ac26e1a35 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -44,6 +44,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel + pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index 92782e1402..d459ee0e98 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -6,39 +6,39 @@ # aiohappyeyeballs==2.6.2 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp aiohttp==3.13.5 # via - # -c requirements.txt + # -c vlm/requirements.txt # aioresponses # pytest-aiohttp aioresponses==0.7.8 - # via -r requirements-test.in + # via -r vlm/requirements.in aiosignal==1.4.0 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp attrs==26.1.0 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp coverage==5.1 - # via -r requirements-test.in + # via -r vlm/requirements.in frozenlist==1.8.0 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp # aiosignal idna==3.15 # via - # -c requirements.txt + # -c vlm/requirements.txt # yarl iniconfig==2.3.0 # via pytest multidict==6.7.1 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp # yarl packaging==26.2 @@ -49,7 +49,7 @@ pluggy==1.6.0 # via pytest propcache==0.5.2 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp # yarl pygments==2.20.0 @@ -59,15 +59,15 @@ pytest==9.0.3 # pytest-aiohttp # pytest-asyncio pytest-aiohttp==1.1.0 - # via -r requirements-test.in + # via -r vlm/requirements.in pytest-asyncio==1.3.0 # via pytest-aiohttp typing-extensions==4.15.0 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiosignal # pytest-asyncio yarl==1.24.2 # via - # -c requirements.txt + # -c vlm/requirements.txt # aiohttp diff --git a/vlm/requirements.txt b/vlm/requirements.txt index bbd63258e9..466a7476f0 100644 --- a/vlm/requirements.txt +++ b/vlm/requirements.txt @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.2 # via aiohttp aiohttp==3.13.5 - # via -r requirements.in + # via -r vlm/requirements.in aiosignal==1.4.0 # via aiohttp attrs==26.1.0 @@ -15,7 +15,7 @@ attrs==26.1.0 certifi==2026.5.20 # via clickhouse-connect clickhouse-connect==0.8.18 - # via -r requirements.in + # via -r vlm/requirements.in frozenlist==1.8.0 # via # aiohttp @@ -33,7 +33,7 @@ propcache==0.5.2 # aiohttp # yarl pyliftover==0.4.1 - # via -r requirements.in + # via -r vlm/requirements.in pytz==2026.2 # via clickhouse-connect typing-extensions==4.15.0 From d645344c7afb095c60fe62d5ee9d5e2319726663 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:20:11 -0400 Subject: [PATCH 144/155] fix dependency install path --- vlm/deploy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index a05445da7f..33a0fce9e5 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,9 +2,9 @@ FROM python:3.11-slim-bullseye LABEL maintainer="Broad TGG" -WORKDIR /vlm +RUN pip install --no-cache-dir -r vlm/requirements.txt -RUN pip install --no-cache-dir -r requirements.txt +WORKDIR /vlm # Application Code COPY vlm/ . From 6ee1583faff605056434f530eb6e9feef2dbee7c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:23:22 -0400 Subject: [PATCH 145/155] add dependency --- vlm/requirements.in | 3 ++- vlm/requirements.txt | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/vlm/requirements.in b/vlm/requirements.in index b56e5a23f2..229f2edb7e 100644 --- a/vlm/requirements.in +++ b/vlm/requirements.in @@ -1,3 +1,4 @@ aiohttp -pyliftover clickhouse-connect==0.8.18 +jwt +pyliftover diff --git a/vlm/requirements.txt b/vlm/requirements.txt index 466a7476f0..90ae504b7c 100644 --- a/vlm/requirements.txt +++ b/vlm/requirements.txt @@ -14,14 +14,20 @@ attrs==26.1.0 # via aiohttp certifi==2026.5.20 # via clickhouse-connect +cffi==2.0.0 + # via cryptography clickhouse-connect==0.8.18 # via -r vlm/requirements.in +cryptography==48.0.0 + # via jwt frozenlist==1.8.0 # via # aiohttp # aiosignal idna==3.15 # via yarl +jwt==1.4.0 + # via -r vlm/requirements.in lz4==4.4.5 # via clickhouse-connect multidict==6.7.1 @@ -32,6 +38,8 @@ propcache==0.5.2 # via # aiohttp # yarl +pycparser==3.0 + # via cffi pyliftover==0.4.1 # via -r vlm/requirements.in pytz==2026.2 From 5718fb99f7eaa1057e8b671755bd1f0fb934f20c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:28:09 -0400 Subject: [PATCH 146/155] correct jwt package --- vlm/requirements.in | 2 +- vlm/requirements.txt | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/vlm/requirements.in b/vlm/requirements.in index 229f2edb7e..8f8e52a40f 100644 --- a/vlm/requirements.in +++ b/vlm/requirements.in @@ -1,4 +1,4 @@ aiohttp clickhouse-connect==0.8.18 -jwt +pyjwt pyliftover diff --git a/vlm/requirements.txt b/vlm/requirements.txt index 90ae504b7c..143556fe26 100644 --- a/vlm/requirements.txt +++ b/vlm/requirements.txt @@ -14,20 +14,14 @@ attrs==26.1.0 # via aiohttp certifi==2026.5.20 # via clickhouse-connect -cffi==2.0.0 - # via cryptography clickhouse-connect==0.8.18 # via -r vlm/requirements.in -cryptography==48.0.0 - # via jwt frozenlist==1.8.0 # via # aiohttp # aiosignal idna==3.15 # via yarl -jwt==1.4.0 - # via -r vlm/requirements.in lz4==4.4.5 # via clickhouse-connect multidict==6.7.1 @@ -38,8 +32,8 @@ propcache==0.5.2 # via # aiohttp # yarl -pycparser==3.0 - # via cffi +pyjwt==2.12.1 + # via -r vlm/requirements.in pyliftover==0.4.1 # via -r vlm/requirements.in pytz==2026.2 @@ -51,4 +45,4 @@ urllib3==2.7.0 yarl==1.24.2 # via aiohttp zstandard==0.25.0 - # via clickhouse-connect + # via clickhouse-connect \ No newline at end of file From 79365c4632b9e5e05e75f254b856c49634bb8c81 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:31:50 -0400 Subject: [PATCH 147/155] clean up --- vlm/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlm/requirements.txt b/vlm/requirements.txt index 143556fe26..d563dd3030 100644 --- a/vlm/requirements.txt +++ b/vlm/requirements.txt @@ -45,4 +45,4 @@ urllib3==2.7.0 yarl==1.24.2 # via aiohttp zstandard==0.25.0 - # via clickhouse-connect \ No newline at end of file + # via clickhouse-connect From 91aa892beda1ce7627e9cede3b8cba70a0137810 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:41:35 -0400 Subject: [PATCH 148/155] pr feedback --- vlm/match.py | 6 +++--- vlm/requirements-test.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index 0a9a7dc743..5ac548c04a 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -42,7 +42,7 @@ '20', '21', '22', 'X', 'Y', 'M', } MIN_POS = 1 -MAX_POS = 3e8 +MAX_POS = 300_000_000 def get_variant_match(query: dict) -> dict: @@ -58,14 +58,14 @@ def get_variant_match(query: dict) -> dict: return _format_results(ac+lift_ac, hom+lift_hom, url) -def _liftover_variant(chrom: str, pos: int, genome_build: str) -> tuple[str, int, str]: +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[tuple[str, int, str]]: liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: tuple[str, int, str]) -> str: +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[tuple[str, int, str]]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index d459ee0e98..0ae9f4ebc9 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -14,7 +14,7 @@ aiohttp==3.13.5 # aioresponses # pytest-aiohttp aioresponses==0.7.8 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in aiosignal==1.4.0 # via # -c vlm/requirements.txt @@ -24,7 +24,7 @@ attrs==26.1.0 # -c vlm/requirements.txt # aiohttp coverage==5.1 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in frozenlist==1.8.0 # via # -c vlm/requirements.txt @@ -59,7 +59,7 @@ pytest==9.0.3 # pytest-aiohttp # pytest-asyncio pytest-aiohttp==1.1.0 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in pytest-asyncio==1.3.0 # via pytest-aiohttp typing-extensions==4.15.0 From 56c6b6718016aa9a5e1c9f7c8bfeae8890f20f51 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:43:48 -0400 Subject: [PATCH 149/155] only initialize liftover once --- vlm/match.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index 5ac548c04a..5d880b4253 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -58,13 +58,6 @@ def get_variant_match(query: dict) -> dict: return _format_results(ac+lift_ac, hom+lift_hom, url) -def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[tuple[str, int, str]]: - liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') - lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) - return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None - - def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[tuple[str, int, str]]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL @@ -136,3 +129,19 @@ def _format_results(ac: int, hom: int, url: str) -> dict: ] } } + + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} + + +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[tuple[str, int, str]]: + liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 + if not LIFTOVERS[genome_version]: + LIFTOVERS[genome_version] = LiftOver( + f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz' + ) + lifted_coord = LIFTOVERS[genome_version].convert_coordinate(f'chr{chrom}', pos) + return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None From 1800d58ec84bb61f1805ec9724eb71758029bba9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 17:49:10 -0400 Subject: [PATCH 150/155] fix --- vlm/match.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index 5d880b4253..35f966374f 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -2,6 +2,7 @@ import os from pyliftover.liftover import LiftOver import re +from typing import Optional, Tuple from vlm.clickhouse_utils import get_clickhouse_variant_counts @@ -58,7 +59,7 @@ def get_variant_match(query: dict) -> dict: return _format_results(ac+lift_ac, hom+lift_hom, url) -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[tuple[str, int, str]]) -> str: +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[Tuple[str, int, str]]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL @@ -137,11 +138,11 @@ def _format_results(ac: int, hom: int, url: str) -> dict: } -def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[tuple[str, int, str]]: +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[Tuple[str, int, str]]: liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - if not LIFTOVERS[genome_version]: - LIFTOVERS[genome_version] = LiftOver( + if not LIFTOVERS[genome_build]: + LIFTOVERS[genome_build] = LiftOver( f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz' ) - lifted_coord = LIFTOVERS[genome_version].convert_coordinate(f'chr{chrom}', pos) + lifted_coord = LIFTOVERS[genome_build].convert_coordinate(f'chr{chrom}', pos) return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None From 69fd26529b89dc294e0e38bbddaf0ca190f995d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 19:09:51 +0000 Subject: [PATCH 151/155] Bump qs and express in /ui Bumps [qs](https://github.com/ljharb/qs) and [express](https://github.com/expressjs/express). These dependencies needed to be updated together. Updates `qs` from 6.14.0 to 6.15.2 - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.14.0...v6.15.2) Updates `express` from 4.22.1 to 4.22.2 - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/v4.22.2/History.md) - [Commits](https://github.com/expressjs/express/compare/v4.22.1...v4.22.2) --- updated-dependencies: - dependency-name: qs dependency-version: 6.15.2 dependency-type: indirect - dependency-name: express dependency-version: 4.22.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ui/package-lock.json | 269 ++++++++++++++++++++++++++++++------------- 1 file changed, 191 insertions(+), 78 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index f58320ac31..21e4d98631 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4707,23 +4707,23 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "dependencies": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8", @@ -4739,6 +4739,41 @@ "node": ">= 0.8" } }, + "node_modules/body-parser/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/body-parser/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/bonjour-service": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", @@ -7996,14 +8031,14 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -8022,7 +8057,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -8071,21 +8106,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "node_modules/express/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -14543,12 +14563,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -14645,20 +14665,64 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/raw-body/node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/react": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", @@ -22720,23 +22784,23 @@ "dev": true }, "body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "requires": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "dependencies": { "depd": { @@ -22744,6 +22808,31 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true } } }, @@ -25408,14 +25497,14 @@ } }, "express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -25434,7 +25523,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -25470,15 +25559,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "requires": { - "side-channel": "^1.1.0" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -30647,12 +30727,12 @@ } }, "qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "requires": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" } }, "query-string": { @@ -30731,15 +30811,48 @@ "dev": true }, "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true + } } }, "react": { From 17c3fb31889af5ac4fceb05b81d9c16935fee121 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 03:28:46 +0000 Subject: [PATCH 152/155] Bump tmp from 0.2.5 to 0.2.7 in /ui Bumps [tmp](https://github.com/raszi/node-tmp) from 0.2.5 to 0.2.7. - [Changelog](https://github.com/raszi/node-tmp/blob/master/CHANGELOG.md) - [Commits](https://github.com/raszi/node-tmp/compare/v0.2.5...v0.2.7) --- updated-dependencies: - dependency-name: tmp dependency-version: 0.2.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ui/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index f58320ac31..b487378b97 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -17303,9 +17303,9 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, "node_modules/tmp": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", - "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", "dev": true, "engines": { "node": ">=14.14" @@ -32902,9 +32902,9 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, "tmp": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", - "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", "dev": true }, "tmpl": { From 42ea2e311344b4105c8f72534df0491e60539082 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 12:25:16 -0400 Subject: [PATCH 153/155] fix igv reference file urls --- ui/shared/components/panel/family/constants.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/shared/components/panel/family/constants.js b/ui/shared/components/panel/family/constants.js index 0b74b27de4..37c4521d61 100644 --- a/ui/shared/components/panel/family/constants.js +++ b/ui/shared/components/panel/family/constants.js @@ -80,7 +80,7 @@ export const IGV_OPTIONS = { const REFERENCE_URLS = [ { key: 'fastaURL', - baseUrl: 'https://igv-genepattern-org.s3.amazonaws.com/genomes/seq', + baseUrl: 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq', path: { 37: 'hg19/hg19.fasta', 38: 'hg38/hg38.fa', @@ -96,10 +96,10 @@ const REFERENCE_URLS = [ }, { key: 'aliasURL', - baseUrl: undefined, + baseUrl: 'https://igv.org/genomes/data', path: { - 37: 'https://igv.org/genomes/data/hg19/hg19_alias.tab', - 38: 'https://igv-genepattern-org.s3.amazonaws.com/genomes/hg38/hg38_alias.tab', + 37: 'hg19/hg19_alias.tab', + 38: 'hg38/hg38_alias.tab', }, }, ] From 6d166abd3061cc9e465f46d54fa90a0b85336269 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Thu, 28 May 2026 14:00:36 -0400 Subject: [PATCH 154/155] Add missing eigen reference dataset refresh (#5463) --- .../migrations/0040_gnomadnoncodingconstraintdict.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py index 35376dfd70..274fb30bf1 100644 --- a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py +++ b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py @@ -4620,4 +4620,9 @@ class Migration(migrations.Migration): reference_dataset="promoterAI", ), ), + migrations.RunPython( + conditionally_refresh_reference_dataset( + reference_dataset="eigen", + ), + ), ] From 8186ee6f642695f1f947b0ab95da2bedc2df84d4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 13:38:38 -0400 Subject: [PATCH 155/155] fix test merge conflict --- clickhouse_search/all_search_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 5143850e98..74d4c11442 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -1522,10 +1522,12 @@ def test_variant_lookup(self, mock_liftover): 'I000016_na20888': { 'affected': 'A', 'features': '', 'restrict_sharing': True, 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 616126, 'mondo_id': '0008788', 'isSolved': False, }, 'I000017_na20889': { 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, 'sex': 'F', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 616126, 'mondo_id': '0008788', 'isSolved': False, }, 'I000018_na21234': { 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'isSolved': True,