From e03fdd3a20e56b95f0b2e38080c9a60b790a7eb7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 11:16:21 -0400 Subject: [PATCH 01/36] move test fixtures from hail folder to clickhouse --- clickhouse_search/test_utils.py | 979 +++++++++++++++++++++++++++++--- 1 file changed, 886 insertions(+), 93 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 0395005abf..89df6c0635 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1,112 +1,905 @@ from copy import deepcopy -from hail_search.test_utils import ( - VARIANT1 as HAIL_VARIANT1, - VARIANT2 as HAIL_VARIANT2, - VARIANT3 as HAIL_VARIANT3, - VARIANT4 as HAIL_VARIANT4, - PROJECT_2_VARIANT as HAIL_PROJECT_2_VARIANT, - GRCH37_VARIANT as HAIL_GRCH37_VARIANT, - MITO_VARIANT1 as HAIL_MITO_VARIANT1, - MITO_VARIANT2 as HAIL_MITO_VARIANT2, - MITO_VARIANT3 as HAIL_MITO_VARIANT3, - SV_VARIANT1 as HAIL_SV_VARIANT1, - SV_VARIANT2 as HAIL_SV_VARIANT2, - SV_VARIANT3 as HAIL_SV_VARIANT3, - SV_VARIANT4 as HAIL_SV_VARIANT4, - GCNV_VARIANT1 as HAIL_GCNV_VARIANT1, - GCNV_VARIANT2 as HAIL_GCNV_VARIANT2, - GCNV_VARIANT3 as HAIL_GCNV_VARIANT3, - GCNV_VARIANT4 as HAIL_GCNV_VARIANT4, -) - -VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 8, 'hom': 3}}} -VARIANT2 = {**deepcopy(HAIL_VARIANT2), 'key': 2, 'populations': {**deepcopy(HAIL_VARIANT2)['populations'], 'seqr': {'ac': 10, 'hom': 3}}} -VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 7, 'hom': 0}}} -VARIANT4 = {**deepcopy(HAIL_VARIANT4), 'key': 4, 'populations': {**deepcopy(HAIL_VARIANT4)['populations'], 'seqr': {'ac': 5, 'hom': 1}}} -PROJECT_2_VARIANT = {**deepcopy(HAIL_PROJECT_2_VARIANT), 'key': 5, 'populations': {**deepcopy(HAIL_PROJECT_2_VARIANT)['populations'], 'seqr': {'ac': 2, 'hom': 0}}} -MITO_VARIANT1 = {**deepcopy(HAIL_MITO_VARIANT1), 'key': 6, 'populations': {**deepcopy(HAIL_MITO_VARIANT1)['populations'], 'seqr': {'ac': 0}, 'seqr_heteroplasmy': {'ac': 1}}} -MITO_VARIANT2 = {**deepcopy(HAIL_MITO_VARIANT2), 'key': 7, 'populations': {**deepcopy(HAIL_MITO_VARIANT2)['populations'], 'seqr': {'ac': 0}, 'seqr_heteroplasmy': {'ac': 1}}} -MITO_VARIANT3 = {**deepcopy(HAIL_MITO_VARIANT3), 'key': 8, 'populations': {**deepcopy(HAIL_MITO_VARIANT3)['populations'], 'seqr': {'ac': 1}, 'seqr_heteroplasmy': {'ac': 0}}} -for variant in [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3]: - variant['genotypes'] = { +VARIANT1 = { + 'key': 1, + 'variantId': '1-10439-AC-A', + 'chrom': '1', + 'pos': 10439, + 'ref': 'AC', + 'alt': 'A', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 10439, + 'xpos': 1000010439, + 'rsid': 'rs112766696', + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 10, 'gq': 99, 'ab': 0.5, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 60, 'gq': 20, 'ab': 0.0, 'filters': [], + }, + }, + 'clinvar': { + 'alleleId': 19473, + 'conflictingPathogenicities': None, + 'goldStars': None, + 'pathogenicity': 'Likely_pathogenic', + 'assertions': None, + 'submitters': None, + 'conditions': None, + }, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 8, 'hom': 3}, + 'topmed': {'af': 0.07842, 'ac': 20757, 'an': 264690, 'hom': 0, 'het': 20757}, + 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.03445, 'ac': 927, 'an': 26912, 'hom': 48, 'hemi': 0, 'filter_af': 0.04028}, + }, + 'predictions': { + 'cadd': 4.668, + 'eigen': None, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + }, + 'transcripts': {}, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'mainTranscriptId': None, + 'selectedMainTranscriptId': None, + 'CAID': 'CA16717152', +} +VARIANT2 = { + 'key': 2, + 'variantId': '1-38724419-T-G', + 'chrom': '1', + 'pos': 38724419, + 'ref': 'T', + 'alt': 'G', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 11854476, + 'xpos': 1038724419, + 'rsid': 'rs1801131', + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 2, 'dp': 36, 'gq': 99, 'ab': 1.0, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 32, 'gq': 99, 'ab': 0.625, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 33, 'gq': 40, 'ab': 0.0, 'filters': [], + }, + }, + 'clinvar': { + 'alleleId': 18560, + 'conflictingPathogenicities': [ + {'count': 1, 'pathogenicity': 'Likely_pathogenic'}, + {'count': 1, 'pathogenicity': 'Uncertain_significance'}, + {'count': 1, 'pathogenicity': 'Likely_benign'}, + {'count': 5, 'pathogenicity': 'Benign'}, + ], + 'goldStars': 1, + 'pathogenicity': 'Conflicting_classifications_of_pathogenicity', + 'assertions': ['other'], + 'submitters': [ + 'Broad Center for Mendelian Genomics, Broad Institute of MIT and Harvard', + 'Illumina Laboratory Services, Illumina', + 'Blueprint Genetics', + 'GenomeConnect, ClinGen' + ], + 'conditions': [ + 'ABCA4-Related Disorders', + 'Severe early-childhood-onset retinal dystrophy', + 'not specified', + 'not provided' + ], + }, + 'hgmd': {'accession': 'CM981315', 'class': 'DFP'}, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 10, 'hom': 3}, + 'topmed': {'af': 0.24615, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, + 'exac': {'af': 0.295, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.4153}, + 'gnomad_exomes': {'af': 0.289, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.41165}, + 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0}, + }, + 'predictions': { + 'cadd': 20.9, + 'eigen': 2.001, + 'fathmm': 0, + 'gnomad_noncoding': 5.86851, + 'mpc': 0.28205, + 'mut_pred': None, + 'primate_ai': 0.46558, + 'splice_ai': 0.0, + 'splice_ai_consequence': 'No consequence', + 'vest': 0.211, + 'mut_taster': 'P', + 'polyphen': 0.1, + 'revel': 0.197, + 'sift': 0.05, + }, + 'transcripts': { + 'ENSG00000177000': [ + {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { + 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', + 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, + }}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426925.5:c.-677A>C', 'hgvsp': None, 'transcriptId': 'ENST00000426925', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 11}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 2, 'utrannotator': { + 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 8,'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', + 'fiveutrAnnotation': {'type': None, 'KozakContext': 'TCAATGC', 'KozakStrength': 'Weak', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 588, 'FrameWithCDS': 'inFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None, 'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, + }}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000412806.1:c.138+1722A>C', 'hgvsp': None, 'transcriptId': 'ENST00000412806', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3},'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 3}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000459627.1:n.298A>C', 'hgvsp': None, 'transcriptId': 'ENST00000459627', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 3}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 4}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000465894.6:n.33A>C', 'hgvsp': None, 'transcriptId': 'ENST00000465894', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 5}, + ], + 'ENSG00000277258': [ + {'aminoAcids': 'L/F', 'canonical': None, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000277258', 'hgvsc': 'ENST00000450625.1:c.156A>C', 'hgvsp': 'ENSP00000389484.1:p.Leu52Phe', 'transcriptId': 'ENST00000450625', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant', 'NMD_transcript_variant'], 'transcriptRank': 0}, + ] + }, + 'mainTranscriptId': 'ENST00000383791', + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'selectedMainTranscriptId': None, + 'CAID': None, +} +VARIANT3 = { + 'key': 3, + 'variantId': '1-91502721-G-A', + 'chrom': '1', + 'pos': 91502721, + 'ref': 'G', + 'alt': 'A', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 91968278, + 'xpos': 1091502721, + 'rsid': 'rs13447464', + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 40, 'gq': 99, 'ab': 1.0, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 37, 'gq': 99, 'ab': 0.45946, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 27, 'gq': 99, 'ab': 0.40741, 'filters': [], + }, + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 7, 'hom': 0}, + 'topmed': {'af': 0.36268, 'ac': 95998, 'an': 264690, 'hom': 19369, 'het': 57260}, + 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.38041, 'ac': 57757, 'an': 151828, 'hom': 12204, 'hemi': 0, 'filter_af': 0.47978}, + }, + 'predictions': { + 'cadd': 2.754, + 'eigen': 1.378, + 'fathmm': None, + 'gnomad_noncoding': 0.73896, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': 0.01, + 'splice_ai_consequence': 'Donor gain', + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + }, + 'transcripts': { + 'ENSG00000097046': [ + {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000234626.11:c.-63-251G>A', 'hgvsp': None, 'transcriptId': 'ENST00000234626', 'maneSelect': 'NM_003503.4', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_003503.4', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000428239.5:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000428239', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001134420.2', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 1}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000497611.1:n.244G>A', 'hgvsp': None, 'transcriptId': 'ENST00000497611', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 4}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 2}, + ], + 'ENSG00000177000': [ + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426137.1:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000426137', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 5}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, + ], + }, + 'mainTranscriptId': 'ENST00000234626', + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': [{'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000009706'}], + 'selectedMainTranscriptId': None, + 'CAID': 'CA10960369', +} +VARIANT4 = { + 'key': 4, + 'variantId': '1-91511686-T-G', + 'chrom': '1', + 'pos': 91511686, + 'ref': 'T', + 'alt': 'G', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 91977243, + 'xpos': 1091511686, + 'rsid': None, + 'familyGuids': ['F000002_2'], + 'genotypes': { 'I000004_hg00731': { - **variant['genotypes']['I000006_hg00733'], 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 29, 'gq': 58, 'ab': 0.17241, 'filters': ['VQSRTrancheSNP99.95to100.00'], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 45, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], + }, + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': 'CTCF-only', + 'populations': { + 'seqr': {'ac': 5, 'hom': 1}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.00027, 'ac': 39, 'an': 147062, 'hom': 0, 'hemi': 0, 'filter_af': 0.0015}, + }, + 'predictions': { + 'cadd': 29.9, + 'eigen': 9.491, + 'fathmm': 0, + 'gnomad_noncoding': 0.23005, + 'mpc': 0.83268, + 'mut_pred': 0.687, + 'primate_ai': 0.69959, + 'splice_ai': 0.0, + 'splice_ai_consequence': 'No consequence', + 'vest': 0.858, + 'mut_taster': 'N', + 'polyphen': 0, + 'revel': 0.526, + 'sift': 0, + }, + 'transcripts': { + 'ENSG00000097046': [ + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000466716.5:c.-264+1G>A', 'hgvsp': None, 'transcriptId': 'ENST00000466716', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': ['5UTR_SPLICE']}, 'spliceregion': {'extended_intronic_splice_region_variant': True}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'splice_donor_variant', 'consequenceTerms': ['splice_donor_variant'], 'transcriptRank': 0}, + {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1}, + {'aminoAcids': 'T/I', 'canonical': None, 'codons': 'aCc/aTc', 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'transcriptId': 'ENST00000257261', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 12}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001281501.1', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 2}, + ], + }, + 'mainTranscriptId': 'ENST00000466716', + 'sortedMotifFeatureConsequences': [ + {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00093424674'}, + {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00036268032'}, + ], + 'sortedRegulatoryFeatureConsequences': [ + {'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000040341'}, + ], + 'selectedMainTranscriptId': None, + 'CAID': 'CA341062623', +} + +PROJECT_2_VARIANT = { + 'key': 5, + 'variantId': '1-10146-ACC-A', + 'chrom': '1', + 'pos': 10146, + 'ref': 'ACC', + 'alt': 'A', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 10146, + 'xpos': 1000010146, + 'rsid': 'rs375931351', + 'familyGuids': ['F000011_11'], + 'genotypes': { + 'I000015_na20885': { + 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', + 'numAlt': 1, 'dp': 8, 'gq': 14, 'ab': 0.875, 'filters': [], } - } - if variant['clinvar']: - variant['clinvar'].update({'assertions': None, 'conditions': None, 'submitters': None}) - variant['populations'].update({ - 'seqr': {'ac': variant['populations']['seqr']['ac']}, - 'seqr_heteroplasmy': {'ac': variant['populations']['seqr_heteroplasmy']['ac']}, - }) -MITO_VARIANT3['predictions']['haplogroup_defining'] = True + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 2, 'hom': 0}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.00012, 'ac': 2, 'an': 16090, 'hom': 0, 'hemi': 0, 'filter_af': 0.00234}, + }, + 'predictions': { + 'cadd': 4.653, + 'eigen': None, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + }, + 'transcripts': {}, + 'mainTranscriptId': None, + 'selectedMainTranscriptId': None, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'CAID': 'CA520798130', +} + GRCH37_VARIANT = { - **deepcopy(HAIL_GRCH37_VARIANT), 'key': 11, + 'variantId': '7-143270172-A-G', + 'xpos': 7143270172, + 'chrom': '7', + 'pos': 143270172, + 'ref': 'A', + 'alt': 'G', + 'genomeVersion': '37', 'liftedOverGenomeVersion': '38', 'liftedOverChrom': '7', 'liftedOverPos': 143271368, - 'populations': {**deepcopy(HAIL_GRCH37_VARIANT)['populations'], 'seqr': {'ac': 3, 'hom': 1}}, -} -for genotype in GRCH37_VARIANT['genotypes'].values(): - genotype['sampleType'] = 'WES' -GRCH37_VARIANT['predictions'].update({'fathmm': None, 'mut_pred': None, 'vest': None}) -for variant in [GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3]: - for transcripts in variant['transcripts'].values(): - for transcript in transcripts: - transcript['loftee'] = {field: transcript.pop(field) for field in ['isLofNagnag', 'lofFilters']} -SV_VARIANT1 = {**deepcopy(HAIL_SV_VARIANT1), 'key': 12, 'populations': {**HAIL_SV_VARIANT1['populations'], 'sv_callset': {'ac': 1, 'hom': 0}}} -SV_VARIANT2 = {**deepcopy(HAIL_SV_VARIANT2), 'key': 13, 'populations': {**HAIL_SV_VARIANT2['populations'], 'sv_callset': {'ac': 2, 'hom': 0}}} -SV_VARIANT3 = {**deepcopy(HAIL_SV_VARIANT3), 'key': 14, 'populations': {**HAIL_SV_VARIANT3['populations'], 'sv_callset': {'ac': 4, 'hom': 1}}} + 'rsid': 'rs72611576', + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1, 'filters': ['VQSRTrancheSNP99.90to99.95'], + }, 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', + 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, + 'filters': ['VQSRTrancheSNP99.90to99.95'], + }, + }, + 'populations': { + 'seqr': {'ac': 3, 'hom': 1}, + 'topmed': {'af': 0.52132, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, + 'exac': {'af': 0.63, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.81988}, + 'gnomad_exomes': {'af': 0.63542, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.82261}, + 'gnomad_genomes': {'af': 0.61365, 'ac': 14649, 'an': 23872, 'hom': 4584, 'hemi': 0, 'filter_af': 0.82844}, + }, + 'predictions': { + 'cadd': 13.02, 'eigen': 3.951, 'primate_ai': 0.49064, + 'splice_ai': 0, 'splice_ai_consequence': 'No consequence', 'fathmm': None, + 'mpc': None, 'mut_taster': None, 'polyphen': None, 'revel': None, 'sift': None, 'mut_pred': None, 'vest': None, + }, + 'clinvar': None, + 'hgmd': None, + 'transcripts': { + 'ENSG00000271079': [ + {'aminoAcids': 'E/G', 'canonical': 1, 'codons': 'gAa/gGa', 'geneId': 'ENSG00000271079', + 'hgvsc': 'ENST00000420911.2:c.1262A>G', 'hgvsp': 'ENSP00000474204.1:p.Glu421Gly', + 'transcriptId': 'ENST00000420911', 'loftee': {'isLofNagnag': None, 'lofFilters': None},'transcriptRank': 0, + 'consequenceTerms': ['missense_variant'], 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant'}, + ], + 'ENSG00000176227': [ + {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000176227', + 'hgvsc': 'ENST00000447022.1:n.1354A>G', 'hgvsp': None, + 'transcriptId': 'ENST00000447022', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'transcriptRank': 0, + 'biotype': 'processed_pseudogene', 'majorConsequence': 'non_coding_transcript_exon_variant', + 'consequenceTerms': ['non_coding_transcript_exon_variant', 'non_coding_transcript_variant']}, + ], + }, + 'mainTranscriptId': 'ENST00000420911', + 'selectedMainTranscriptId': None, + 'CAID': 'CA4540310', +} + +MITO_VARIANT1 = { + 'key': 6, + 'variantId': 'M-4429-G-A', + 'chrom': 'M', + 'pos': 4429, + 'ref': 'G', + 'alt': 'A', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': 'MT', + 'liftedOverPos': 4429, + 'xpos': 25000004429, + 'rsid': 'rs1603219456', + 'familyGuids': ['F000002_2'], + 'genotypes': {'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 3955, 'hl': 0.083, 'mitoCn': 230, 'contamination': 0.0, 'filters': [], + }}, + 'populations': { + 'seqr': {'ac': 0}, + 'seqr_heteroplasmy': {'ac': 1}, + 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 56419}, + 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 56419, 'max_hl': 0.0}, + 'helix': {'af': 0.0, 'ac': 0, 'an': 195983}, + 'helix_heteroplasmy': {'af': 2e-05, 'ac': 3, 'an': 195983, 'max_hl': 0.20635}, + }, + 'predictions': { + 'apogee': None, + 'haplogroup_defining': None, + 'hmtvar': 0.05, + 'mitotip': 'likely_pathogenic', + 'mut_taster': None, + 'sift': None, + 'mlc': 3.38874, + }, + 'commonLowHeteroplasmy': False, + 'mitomapPathogenic': None, + 'clinvar': { + 'alleleId': 677748, + 'conflictingPathogenicities': None, + 'goldStars': 1, + 'pathogenicity': 'Uncertain_significance', + 'assertions': None, + 'conditions': None, + 'submitters': None, + }, + 'transcripts': { + 'ENSG00000210112': [ + {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000210112', 'hgvsc': 'ENST00000387377.1:n.28G>A', 'hgvsp': None, 'transcriptId': 'ENST00000387377', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'transcriptRank': 0, 'biotype': 'Mt_tRNA', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant']}, + ], + }, + 'mainTranscriptId': 'ENST00000387377', + 'selectedMainTranscriptId': None, +} +MITO_VARIANT2 = { + 'key': 7, + 'variantId': 'M-11597-A-G', + 'chrom': 'M', + 'pos': 11597, + 'ref': 'A', + 'alt': 'G', + 'xpos': 25000011597, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': 'MT', + 'liftedOverPos': 11597, + 'rsid': None, + 'familyGuids': ['F000002_2'], + 'genotypes': {'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 3845, 'hl': 0.029, 'mitoCn': 247, 'contamination': 0.015, 'filters': [], + }}, + 'populations': { + 'seqr': {'ac': 0}, + 'seqr_heteroplasmy': {'ac': 1}, + 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 0}, + 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, + 'helix': {'af': 0.0, 'ac': 0, 'an': 0}, + 'helix_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, + }, + 'predictions': { + 'apogee': 0.58, + 'haplogroup_defining': None, + 'hmtvar': 0.75, + 'mitotip': None, + 'mut_taster': 'N', + 'sift': 0, + 'mlc': None, + }, + 'commonLowHeteroplasmy': False, + 'mitomapPathogenic': None, + 'clinvar': None, + 'transcripts': { + 'ENSG00000198886': [ + {'aminoAcids': 'T/A', 'canonical': 1, 'codons': 'Aca/Gca', 'geneId': 'ENSG00000198886', 'hgvsc': 'ENST00000361381.2:c.838A>G', 'hgvsp': 'ENSP00000354961.2:p.Thr280Ala', 'transcriptId': 'ENST00000361381', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'transcriptRank': 0, 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant']}, + ], + }, + 'mainTranscriptId': 'ENST00000361381', + 'selectedMainTranscriptId': None, +} +MITO_VARIANT3 = { + 'key': 8, + 'variantId': 'M-14783-T-C', + 'chrom': 'M', + 'pos': 14783, + 'ref': 'T', + 'alt': 'C', + 'xpos': 25000014783, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': 'MT', + 'liftedOverPos': 14783, + 'rsid': 'rs193302982', + 'familyGuids': ['F000002_2'], + 'genotypes': {'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 2, 'dp': 3943, 'hl': 1.0, 'mitoCn': 214, 'contamination': 0.0, 'filters': ['artifact_prone_site'], + }}, + 'populations': { + 'seqr': {'ac': 1}, + 'seqr_heteroplasmy': {'ac': 0}, + 'gnomad_mito': {'af': 0.05535, 'ac': 3118, 'an': 56336}, + 'gnomad_mito_heteroplasmy': {'af': 5e-05, 'ac': 3, 'an': 56336, 'max_hl': 1.0}, + 'helix': {'af': 0.04885, 'ac': 9573, 'an': 195983}, + 'helix_heteroplasmy': {'af': 9e-05, 'ac': 18, 'an': 195983, 'max_hl': 0.96269}, + }, + 'predictions': { + 'apogee': None, + 'haplogroup_defining': True, + 'hmtvar': None, + 'mitotip': None, + 'mut_taster': None, + 'sift': None, + 'mlc': 0.7514, + }, + 'commonLowHeteroplasmy': True, + 'mitomapPathogenic': True, + 'clinvar': { + 'alleleId': 150280, + 'conflictingPathogenicities': None, + 'goldStars': 0, + 'pathogenicity': 'Likely_pathogenic', + 'assertions': None, + 'conditions': None, + 'submitters': None, + }, + 'transcripts': { + 'ENSG00000198727': [ + {'aminoAcids': 'L', 'canonical': 1, 'codons': 'Tta/Cta', 'geneId': 'ENSG00000198727', 'hgvsc': 'ENST00000361789.2:c.37T>C', 'hgvsp': 'ENSP00000354554.2:p.Leu13=', 'transcriptId': 'ENST00000361789', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'transcriptRank': 0, 'biotype': 'protein_coding', 'majorConsequence': 'synonymous_variant', 'consequenceTerms': ['synonymous_variant']}, + ], + }, + 'mainTranscriptId': 'ENST00000361789', + 'selectedMainTranscriptId': None, +} + +SV_VARIANT1 = { + 'key': 12, + 'variantId': 'phase2_DEL_chr1_625', + 'chrom': '1', + 'endChrom': None, + 'pos': 9310023, + 'end': 9310127, + 'rg37LocusEnd': {'contig': '1', 'position': 9370186}, + 'xpos': 1009310023, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 9370082, + 'algorithms': 'manta', + 'bothsidesSupport': False, + 'familyGuids': ['F000014_14'], + 'genotypes': { + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': 2, 'gq': 0, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000019_na21987': { + 'sampleId': 'NA21987', 'sampleType': 'WGS', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14', + 'numAlt': 0, 'cn': 2, 'gq': 6, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000021_na21654': { + 'sampleId': 'NA21654', 'sampleType': 'WGS', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14', + 'numAlt': 0, 'cn': 2, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, + }, + 'populations': { + 'sv_callset': {'ac': 1, 'hom': 0}, + 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, + }, + 'predictions': {'strvctvre': None}, + 'cpxIntervals': None, + 'svSourceDetail': None, + 'svType': 'DEL', + 'svTypeDetail': None, + 'transcripts': { + 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'INTRONIC'}], + }, +} +SV_VARIANT2 = { + 'key': 13, + 'variantId': 'cohort_2911.chr1.final_cleanup_INS_chr1_160', + 'chrom': '1', + 'endChrom': None, + 'pos': 9380254, + 'end': 9380286, + 'rg37LocusEnd': {'contig': '4', 'position': 9382012}, + 'xpos': 1009380254, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 9440313, + 'algorithms': 'manta', + 'bothsidesSupport': True, + 'familyGuids': ['F000014_14'], + 'genotypes': { + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000019_na21987': { + 'sampleId': 'NA21987', 'sampleType': 'WGS', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14', + 'numAlt': 0, 'cn': None, 'gq': 99, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000021_na21654': { + 'sampleId': 'NA21654', 'sampleType': 'WGS', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, + }, + 'populations': { + 'sv_callset': {'ac': 2, 'hom': 0}, + 'gnomad_svs': {'af': 0.00542, 'id': 'gnomAD-SV_v3_INS_1_299', 'hom': 10359, 'het': 35634}, + }, + 'predictions': {'strvctvre': None}, + 'cpxIntervals': None, + 'svSourceDetail': {'chrom': '4'}, + 'svType': 'INS', + 'svTypeDetail': None, + 'transcripts': { + 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'NEAREST_TSS'}], + }, +} +SV_VARIANT3 = { + 'key': 14, + 'variantId': 'phase2_CPX_chr13_35', + 'chrom': '13', + 'endChrom': '17', + 'pos': 63036002, + 'end': 63036041, + 'rg37LocusEnd': {'contig': '17', 'position': 61113402}, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '13', + 'liftedOverPos': 63610135, + 'xpos': 13063036002, + 'algorithms': 'manta', + 'bothsidesSupport': False, + 'familyGuids': ['F000014_14'], + 'genotypes': { + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': None, 'gq': 62, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000019_na21987': { + 'sampleId': 'NA21987', 'sampleType': 'WGS', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14', + 'numAlt': 2, 'cn': None, 'gq': 42, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, 'I000021_na21654': { + 'sampleId': 'NA21654', 'sampleType': 'WGS', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': None, 'gq': 79, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], + }, + }, 'populations': { + 'sv_callset': {'ac': 4, 'hom': 1}, + 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, + }, + 'predictions': {'strvctvre': None}, + 'cpxIntervals': [{'chrom': '17', 'start': 22150735, 'end': 22151179, 'type': 'DUP'}], + 'svSourceDetail': None, + 'svType': 'CPX', + 'svTypeDetail': 'dDUP', + 'transcripts': { + 'ENSG00000083544': [{'geneId': 'ENSG00000083544', 'majorConsequence': 'NEAREST_TSS'}], + 'null': [{'geneId': None, 'majorConsequence': 'NEAREST_TSS'}], + }, +} SV_VARIANT4 = { - **deepcopy(HAIL_SV_VARIANT4), 'key': 15, - 'populations': {**HAIL_SV_VARIANT4['populations'], 'sv_callset': {'ac': 4, 'hom': 1}}, - 'xpos': 17038719997, + 'variantId': 'phase2_DEL_chr14_4640', 'chrom': '17', - 'liftedOverChrom': '17', + 'endChrom': None, 'pos': 38719997, 'end': 38737237, + 'rg37LocusEnd': None, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '17', + 'liftedOverPos': 107150261, + 'xpos': 17038719997, + 'algorithms': 'depth,manta', + 'bothsidesSupport': False, + 'familyGuids': ['F000014_14'], + 'genotypes': { + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14', + 'numAlt': 2, 'cn': 0, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': [], + }, 'I000019_na21987': { + 'sampleId': 'NA21987', 'sampleType': 'WGS', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], + }, 'I000021_na21654': { + 'sampleId': 'NA21654', 'sampleType': 'WGS', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], + }, + }, + 'populations': { + 'sv_callset': {'ac': 4, 'hom': 1}, + 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, + }, + 'predictions': {'strvctvre': 0.161}, + 'cpxIntervals': None, + 'svSourceDetail': None, + 'svType': 'DEL', + 'svTypeDetail': None, + 'transcripts': { + 'ENSG00000184986': [{'geneId': 'ENSG00000184986', 'majorConsequence': 'NEAREST_TSS'}], + }, } -for variant in [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4]: - variant['familyGuids'] = ['F000014_14'] - variant['genotypes'] = { - 'I000018_na21234': {**variant['genotypes']['I000015_na20885'], 'sampleId': 'NA21234', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14'}, - 'I000019_na21987': {**variant['genotypes']['I000025_na20884'], 'sampleId': 'NA21987', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14'}, - 'I000021_na21654': {**variant['genotypes']['I000035_na20883'], 'sampleId': 'NA21654', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14'}, - } -SV_VARIANT3['cpxIntervals'][0]['chrom'] = SV_VARIANT3['cpxIntervals'][0]['chrom'].replace('chr', '') -GCNV_VARIANT1 = {**deepcopy(HAIL_GCNV_VARIANT1), 'key': 16} -GCNV_VARIANT2 = {**deepcopy(HAIL_GCNV_VARIANT2), 'key': 17} -GCNV_VARIANT3 = {**deepcopy(HAIL_GCNV_VARIANT3), 'key': 18} -GCNV_VARIANT4 = {**deepcopy(HAIL_GCNV_VARIANT4), 'key': 19} - -for variant in [VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT, GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, - SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4]: - # clickhouse uses fixed length decimals so values are rounded relative to hail backend - for genotype in variant['genotypes'].values(): - if 'ab' in genotype: - genotype['ab'] = round(genotype['ab'], 5) - for pred, pred_val in variant['predictions'].items(): - if isinstance(pred_val, float): - variant['predictions'][pred] = round(pred_val, 5) - for pop in variant['populations'].values(): - if 'af' in pop: - pop['af'] = round(pop['af'], 5) - if 'filter_af' in pop: - pop['filter_af'] = round(pop['filter_af'], 5) - if 'max_hl' in pop: - pop['max_hl'] = round(pop['max_hl'], 5) - for transcripts in variant['transcripts'].values(): - for transcript in transcripts: - if transcript.get('alphamissense', {}).get('pathogenicity'): - transcript['alphamissense']['pathogenicity'] = round(transcript['alphamissense']['pathogenicity'], 5) - # sort is not computed/annotated at query time - del variant['_sort'] - if variant.get('clinvar'): - del variant['clinvar']['version'] +GCNV_VARIANT1 = { + 'key': 16, + 'variantId': 'suffix_95340_DUP', + 'chrom': '14', + 'pos': 22438910, + 'end': 22469796, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '14', + 'liftedOverPos': 22886546, + 'rg37LocusEnd': {'contig': '14', 'position': 23058228}, + 'xpos': 14022417556, + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 38, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], + } + }, + 'populations': {'sv_callset': {'af': 0.07649, 'ac': 1763, 'an': 23048, 'hom': 0, 'het': 0}}, + 'predictions': {'strvctvre': 0.181}, + 'numExon': 0, + 'svType': 'DUP', + 'transcripts': {}, +} +GCNV_VARIANT2 = { + 'key': 17, + 'variantId': 'suffix_124520_DUP', + 'chrom': '16', + 'pos': 29809156, + 'end': 29815990, + 'xpos': 16029802672, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '16', + 'liftedOverPos': 29813993, + 'rg37LocusEnd': {'contig': '16', 'position': 29831761}, + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 46, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 37, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + } + }, + 'populations': {'sv_callset': {'af': 0.01232, 'ac': 284, 'an': 23047, 'hom': 0, 'het': 0}}, + 'predictions': {'strvctvre': 0.548}, + 'numExon': 8, + 'svType': 'DUP', + 'transcripts': { + 'ENSG00000103495': [{'geneId': 'ENSG00000103495', 'majorConsequence': 'COPY_GAIN'}], + 'ENSG00000167371': [{'geneId': 'ENSG00000167371', 'majorConsequence': 'COPY_GAIN'}], + 'ENSG00000280893': [{'geneId': 'ENSG00000280893', 'majorConsequence': 'COPY_GAIN'}], + }, +} +GCNV_VARIANT3 = { + 'key': 18, + 'variantId': 'suffix_140593_DUP', + 'chrom': '17', + 'pos': 38717327, + 'end': 38719636, + 'xpos': 17038717327, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '17', + 'liftedOverPos': 36873580, + 'rg37LocusEnd': {'contig': '17', 'position': 36876246}, + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 2, 'cn': 4, 'qs': 13, 'defragged': True, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': True, 'prevCall': False, 'prevOverlap': False, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 7, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': False, 'prevOverlap': True, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', 'numAlt': 0, + 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, 'geneIds': None, + 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], + }, + }, + 'populations': {'sv_callset': {'af': 0.00152, 'ac': 35, 'an': 23048, 'hom': 0, 'het': 0}}, + 'predictions': {'strvctvre': 0.786}, + 'numExon': 3, + 'svType': 'DUP', + 'transcripts': { + 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], + }, +} +GCNV_VARIANT4 = { + 'key': 19, + 'variantId': 'suffix_140608_DUP', + 'chrom': '17', + 'pos': 38721781, + 'end': 38735703, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '17', + 'liftedOverPos': 36878034, + 'rg37LocusEnd': {'contig': '17', 'position': 36892521}, + 'familyGuids': ['F000002_2'], + 'xpos': 17038721781, + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 28, 'defragged': False, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + }, + 'I000005_hg00732': { + 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', + 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, + 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', + 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': 38734440, 'numExon': None, + 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], + } + }, + 'populations': {'sv_callset': {'af': 0.00499, 'ac': 115, 'an': 23048, 'hom': 0, 'het': 0}}, + 'predictions': {'strvctvre': 0.71}, + 'numExon': 7, + 'svType': 'DEL', + 'transcripts': { + 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], + 'ENSG00000277258': [{'geneId': 'ENSG00000277258', 'majorConsequence': 'LOF'}], + 'ENSG00000277972': [{'geneId': 'ENSG00000277972', 'majorConsequence': 'COPY_GAIN'}], + }, +} FAMILY_3_VARIANT = deepcopy(VARIANT3) FAMILY_3_VARIANT['familyGuids'] = ['F000003_3'] From a49373ccc2d950555e0cb3c1d65c4cc18b425750 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 11:27:20 -0400 Subject: [PATCH 02/36] update test import --- seqr/utils/search/search_utils_tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 6bfacf5cfc..e068739aec 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -4,8 +4,8 @@ import json import mock -from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant -from hail_search.test_utils import GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_VARIANT4, SV_VARIANT1 +from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant, \ + GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_VARIANT4, SV_VARIANT1 from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ @@ -18,6 +18,12 @@ NON_SNP_INDEL_SAMPLES = SV_SAMPLES + MITO_SAMPLES FAMILY_3_SAMPLE = 'S000135_na20870' +GENE_COUNTS = { + **GENE_COUNTS, + 'ENSG00000177000': {**GENE_COUNTS['ENSG00000177000'], 'total': 3}, + 'ENSG00000277258': {**GENE_COUNTS['ENSG00000277258'], 'total': 2}, +} + class SearchTestHelper(object): def set_up(self): @@ -83,7 +89,7 @@ def _mock_get_variants(samples, search, user, previous_search_results, genome_ve mock_sv_variant_lookup.assert_called_with( self.user, 'phase2_DEL_chr14_4640', 'SV_WGS', genome_version='38', samples=mock.ANY) mock_get_variants.assert_called_with(mock.ANY, { - 'parsed_locus': {'padded_interval': {'chrom': '14', 'start': 106694244, 'end': 106740587, 'padding': 0.2}}, + 'parsed_locus': {'padded_interval': {'chrom': '17', 'start': 38719997, 'end': 38737237, 'padding': 0.2}}, 'annotations': {'structural': ['DEL', 'gCNV_DEL']}, }, self.user, mock.ANY, '38') cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38__test_user' @@ -696,7 +702,7 @@ def test_cached_get_variant_query_gene_counts(self): self.assertDictEqual(gene_counts, { 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000011_11': 1}}, + 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, }) @@ -787,7 +793,7 @@ def test_cached_get_variant_query_gene_counts(self): 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000011_11': 1}}, + 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, }) @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') From bdc87958029809e1b81f5e542c58123fdc5498db Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 Aug 2025 11:39:01 -0400 Subject: [PATCH 03/36] first pass isoform expression --- ui/shared/components/graph/Gtex.jsx | 161 ++++++------------- ui/shared/components/graph/GtexLauncher.jsx | 5 +- ui/shared/components/graph/GtexViolin.jsx | 166 ++++++++++++++++++++ 3 files changed, 215 insertions(+), 117 deletions(-) create mode 100644 ui/shared/components/graph/GtexViolin.jsx diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 1ed8c45ee7..741b16a8bb 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -1,5 +1,5 @@ import React from 'react' -import { deviation, extent, max, mean, median, min, quantile } from 'd3-array' +import { deviation, extent, max, mean, min, quantile } from 'd3-array' import { randomNormal } from 'd3-random' import { scaleBand, scaleLinear } from 'd3-scale' import { area } from 'd3-shape' @@ -19,125 +19,54 @@ const DIMENSIONS = { height: 400, } -// Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/GeneExpressionViolinPlot.js +// Code adapted from https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/TranscriptBrowser.js +// and https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/modules/Heatmap.js -const drawViolin = (svg, scale, tooltip) => (entry) => { - const violinG = svg.append('g') - .datum(entry) - - const eDomain = extent(entry.values) // get the max and min in entry.values - const q1 = quantile(entry.values, 0.25) - const q3 = quantile(entry.values, 0.75) - - // get the vertices - const rangeDeviation = (q3 - q1) / 1.34 - const kernelBandwidth = 1.06 * Math.min(deviation(entry.values), rangeDeviation) * (entry.values.length ** -0.2) - // use up to 100 vertices along the Y axis (to create the violin path) - const vertices = scale.y.ticks(100).map(x => [x, mean(entry.values, (v) => { - const u = (x - v) / kernelBandwidth - return (1 / Math.sqrt(2 * Math.PI)) * Math.exp(-0.5 * u * u) - }) / kernelBandwidth]).filter( - // filter the vertices that aren't in the entry.values - d => d[0] >= eDomain[0] && d[0] <= eDomain[1], - ) - - if (vertices.length < 1 || vertices.some(Number.isNaN)) { - return - } - - // define the z scale -- the violin width - const zMax = max(vertices, d => Math.abs(d[1])) - scale.z - .domain([-zMax, zMax]) - .range([scale.x(entry.label), scale.x(entry.label) + scale.x.bandwidth()]) - - // visual rendering - const violin = area() - .x0(d => scale.z(d[1])) - .x1(d => scale.z(-d[1])) - .y(d => scale.y(d[0])) - - const vPath = violinG.append('path') - .datum(vertices) - .attr('d', violin) - .style('fill', () => entry.color) - .style('opacity', 0.6) - - // boxplot - const z = scale.z.domain()[1] / 3 - const x = scale.z(-z) - - // interquartile range - violinG.append('rect') - .attr('x', x) - .attr('y', scale.y(q3)) - .attr('width', Math.abs(x - scale.z(z))) - .attr('height', Math.abs(scale.y(q3) - scale.y(q1))) - .style('fill', '#555f66') - - // the median line - const medianY = scale.y(entry.median) - violinG.append('line') - .attr('x1', x) - .attr('x2', scale.z(z)) - .attr('y1', medianY) - .attr('y2', medianY) - .style('stroke', '#fff') - .style('stroke-width', '2px') - - const jitter = randomNormal(0, z / 2) - const iqr = Math.abs(q3 - q1) - const upper = max(entry.values.filter(d => d <= q3 + (iqr * 1.5))) - const lower = min(entry.values.filter(d => d >= q1 - (iqr * 1.5))) - const outliers = entry.values.filter(d => d < lower || d > upper) - violinG.append('g') - .selectAll('circle') - .data(outliers) - .enter() - .append('circle') - .attr('cx', () => scale.z(jitter())) - .attr('cy', d => scale.y(d)) - .attr('fill', entry.color) - .attr('r', 1) - - // mouse events - violinG.on('mouseover', () => { - vPath.style('opacity', 1) - tooltip.show( - `${entry.label}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, - x + 70, - medianY < 40 ? 10 : medianY - 40, - ) - }) - violinG.on('mouseout', () => { - vPath.style('opacity', 0.6) - }) -} - -const renderViolinPlot = (violinPlotData, containerElement) => { - const xDomain = violinPlotData.map(({ label }) => label) - const yDomain = extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), [])) +const renderIsoformHeatmap = (isoformData, containerElement) => { + const xDomain = [...(new Set(...isoformData.map(({ x }) => x)))].sort(compareObjects('x')) + const yDomain = [...(new Set(...isoformData.map(({ y }) => y)))].sort(compareObjects('y')) const scale = { x: scaleBand() .rangeRound([0, DIMENSIONS.width]) .domain(xDomain) .paddingInner(0.2), - y: scaleLinear() + y: scaleBand() .rangeRound([DIMENSIONS.height, 0]) .domain(yDomain), z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin } - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, { - y: { - text: 'TPM', - transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), - }, - }) + const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale) const tooltip = new Tooltip(containerElement) - violinPlotData.forEach(drawViolin(svg, scale, tooltip)) + // TODO fix actually render heatmap + isoformData.forEach(({ x, y, value }) => { + svg.append('rect') + // .attr("row", (d) => `x${this.xList.indexOf(d.x)}`) + // .attr("col", (d) => `y${this.yList.indexOf(d.y)}`) + .attr('x', scale.x(x)) + .attr('y', scale.y(y)) + .attr('rx', 5) + .attr('ry', 5) + // .attr('class', 'exp-map-cell') + .attr('width', scale.x.bandwidth()) + .attr('height', scale.y.bandwidth()) + .style('fill', '#eeeeee') + .on('mouseover', () => { + tooltip.show( + `Tissue: ${x}
Isoform: ${y}
TPM: ${value}`, + x, + y, + ) + }) + .on('mouseout', () => { + tooltip.hide() + }) + // .style('fill', (d) => { + // return useNullColor&&d.value==0?nullColor:this.useLog?this.colorScale(this._log(d.value)):this.colorScale(d.value) + // }) + }) } // seqr-specific code @@ -145,22 +74,24 @@ const renderViolinPlot = (violinPlotData, containerElement) => { const loadTissueData = onSuccess => queryGtex('dataset/tissueSiteDetail', {}, onSuccess) const renderGtex = (expressionData, tissueData, containerElement) => { - if ((expressionData?.data || []).length < 1) { + if ((expressionData?.medianTranscriptExpression || []).length < 1) { return } const tissueLookup = tissueData.data.reduce( (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, ) - const violinPlotData = expressionData.data.map(({ tissueSiteDetailId, data }) => ({ - values: data.sort(), - median: median(data), - label: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, - color: `#${tissueLookup[tissueSiteDetailId]?.colorHex}`, + // TODO better property names + const isoformData = expressionData.medianTranscriptExpression.map(({ median, transcriptId, tissueSiteDetailId, gencodeId }) => ({ + value: Number(median), + displayValue: Number(median), + y: transcriptId, + x: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, + tissueId: tissueSiteDetailId, + id: gencodeId, })) - violinPlotData.sort(compareObjects('label')) - renderViolinPlot(violinPlotData, containerElement) + renderIsoformHeatmap(isoformData, containerElement) } export default props => ( - + ) diff --git a/ui/shared/components/graph/GtexLauncher.jsx b/ui/shared/components/graph/GtexLauncher.jsx index e7686ab2ac..ef02bec135 100644 --- a/ui/shared/components/graph/GtexLauncher.jsx +++ b/ui/shared/components/graph/GtexLauncher.jsx @@ -21,16 +21,17 @@ class GtexLauncher extends React.PureComponent { fetchAdditionalData: PropTypes.func.isRequired, getAdditionalExpressionParams: PropTypes.func, renderOnError: PropTypes.bool, + expressionPath: PropTypes.string, } state = { loading: false } loadGeneExpression = (gencodeId, additionalData) => { - const { getAdditionalExpressionParams } = this.props + const { getAdditionalExpressionParams, expressionPath = 'geneExpression' } = this.props const params = getAdditionalExpressionParams ? getAdditionalExpressionParams(additionalData) : {} const onComplete = this.onExpressionLoadComplete(additionalData) queryGtex( - 'expression/geneExpression', + `expression/${expressionPath}`, { gencodeId, ...params }, onComplete, () => onComplete(null), diff --git a/ui/shared/components/graph/GtexViolin.jsx b/ui/shared/components/graph/GtexViolin.jsx new file mode 100644 index 0000000000..1ed8c45ee7 --- /dev/null +++ b/ui/shared/components/graph/GtexViolin.jsx @@ -0,0 +1,166 @@ +import React from 'react' +import { deviation, extent, max, mean, median, min, quantile } from 'd3-array' +import { randomNormal } from 'd3-random' +import { scaleBand, scaleLinear } from 'd3-scale' +import { area } from 'd3-shape' + +import { compareObjects } from 'shared/utils/sortUtils' +import { initializeD3, Tooltip } from './d3Utils' +import GtexLauncher, { queryGtex } from './GtexLauncher' + +const MARGINS = { + top: 10, + right: 50, + bottom: 150, + left: 50, +} +const DIMENSIONS = { + width: window.innerWidth * 0.8, + height: 400, +} + +// Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/GeneExpressionViolinPlot.js + +const drawViolin = (svg, scale, tooltip) => (entry) => { + const violinG = svg.append('g') + .datum(entry) + + const eDomain = extent(entry.values) // get the max and min in entry.values + const q1 = quantile(entry.values, 0.25) + const q3 = quantile(entry.values, 0.75) + + // get the vertices + const rangeDeviation = (q3 - q1) / 1.34 + const kernelBandwidth = 1.06 * Math.min(deviation(entry.values), rangeDeviation) * (entry.values.length ** -0.2) + // use up to 100 vertices along the Y axis (to create the violin path) + const vertices = scale.y.ticks(100).map(x => [x, mean(entry.values, (v) => { + const u = (x - v) / kernelBandwidth + return (1 / Math.sqrt(2 * Math.PI)) * Math.exp(-0.5 * u * u) + }) / kernelBandwidth]).filter( + // filter the vertices that aren't in the entry.values + d => d[0] >= eDomain[0] && d[0] <= eDomain[1], + ) + + if (vertices.length < 1 || vertices.some(Number.isNaN)) { + return + } + + // define the z scale -- the violin width + const zMax = max(vertices, d => Math.abs(d[1])) + scale.z + .domain([-zMax, zMax]) + .range([scale.x(entry.label), scale.x(entry.label) + scale.x.bandwidth()]) + + // visual rendering + const violin = area() + .x0(d => scale.z(d[1])) + .x1(d => scale.z(-d[1])) + .y(d => scale.y(d[0])) + + const vPath = violinG.append('path') + .datum(vertices) + .attr('d', violin) + .style('fill', () => entry.color) + .style('opacity', 0.6) + + // boxplot + const z = scale.z.domain()[1] / 3 + const x = scale.z(-z) + + // interquartile range + violinG.append('rect') + .attr('x', x) + .attr('y', scale.y(q3)) + .attr('width', Math.abs(x - scale.z(z))) + .attr('height', Math.abs(scale.y(q3) - scale.y(q1))) + .style('fill', '#555f66') + + // the median line + const medianY = scale.y(entry.median) + violinG.append('line') + .attr('x1', x) + .attr('x2', scale.z(z)) + .attr('y1', medianY) + .attr('y2', medianY) + .style('stroke', '#fff') + .style('stroke-width', '2px') + + const jitter = randomNormal(0, z / 2) + const iqr = Math.abs(q3 - q1) + const upper = max(entry.values.filter(d => d <= q3 + (iqr * 1.5))) + const lower = min(entry.values.filter(d => d >= q1 - (iqr * 1.5))) + const outliers = entry.values.filter(d => d < lower || d > upper) + violinG.append('g') + .selectAll('circle') + .data(outliers) + .enter() + .append('circle') + .attr('cx', () => scale.z(jitter())) + .attr('cy', d => scale.y(d)) + .attr('fill', entry.color) + .attr('r', 1) + + // mouse events + violinG.on('mouseover', () => { + vPath.style('opacity', 1) + tooltip.show( + `${entry.label}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, + x + 70, + medianY < 40 ? 10 : medianY - 40, + ) + }) + violinG.on('mouseout', () => { + vPath.style('opacity', 0.6) + }) +} + +const renderViolinPlot = (violinPlotData, containerElement) => { + const xDomain = violinPlotData.map(({ label }) => label) + const yDomain = extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), [])) + + const scale = { + x: scaleBand() + .rangeRound([0, DIMENSIONS.width]) + .domain(xDomain) + .paddingInner(0.2), + y: scaleLinear() + .rangeRound([DIMENSIONS.height, 0]) + .domain(yDomain), + z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin + } + + const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, { + y: { + text: 'TPM', + transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), + }, + }) + + const tooltip = new Tooltip(containerElement) + violinPlotData.forEach(drawViolin(svg, scale, tooltip)) +} + +// seqr-specific code + +const loadTissueData = onSuccess => queryGtex('dataset/tissueSiteDetail', {}, onSuccess) + +const renderGtex = (expressionData, tissueData, containerElement) => { + if ((expressionData?.data || []).length < 1) { + return + } + const tissueLookup = tissueData.data.reduce( + (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, + ) + const violinPlotData = expressionData.data.map(({ tissueSiteDetailId, data }) => ({ + values: data.sort(), + median: median(data), + label: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, + color: `#${tissueLookup[tissueSiteDetailId]?.colorHex}`, + })) + violinPlotData.sort(compareObjects('label')) + renderViolinPlot(violinPlotData, containerElement) +} + +export default props => ( + +) From 414cfb6f6d50b5a0b7505802de8fa62d83167183 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 12:54:18 -0400 Subject: [PATCH 04/36] fix axis render --- ui/shared/components/graph/Gtex.jsx | 8 ++++---- ui/shared/components/graph/d3Utils.jsx | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 741b16a8bb..d826193fa8 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -12,7 +12,7 @@ const MARGINS = { top: 10, right: 50, bottom: 150, - left: 50, + left: 100, } const DIMENSIONS = { width: window.innerWidth * 0.8, @@ -23,8 +23,8 @@ const DIMENSIONS = { // and https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/modules/Heatmap.js const renderIsoformHeatmap = (isoformData, containerElement) => { - const xDomain = [...(new Set(...isoformData.map(({ x }) => x)))].sort(compareObjects('x')) - const yDomain = [...(new Set(...isoformData.map(({ y }) => y)))].sort(compareObjects('y')) + const xDomain = [...new Set(isoformData.map(({ x }) => x))].sort(compareObjects('x')) + const yDomain = [...new Set(isoformData.map(({ y }) => y))].sort(compareObjects('y')) const scale = { x: scaleBand() @@ -37,7 +37,7 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin } - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale) + const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, {}) const tooltip = new Tooltip(containerElement) // TODO fix actually render heatmap diff --git a/ui/shared/components/graph/d3Utils.jsx b/ui/shared/components/graph/d3Utils.jsx index 793b24c54b..de3f24cf92 100644 --- a/ui/shared/components/graph/d3Utils.jsx +++ b/ui/shared/components/graph/d3Utils.jsx @@ -30,17 +30,24 @@ export const initializeD3 = (containerElement, dimensions, margins, scales, axis // render y-axis let yAxis = axisLeft(scales.y) - if (axis.y.transform) { + if (axis.y?.transform) { yAxis = axis.y.transform(yAxis) } - svg.append('g') - .attr('transform', `translate(-${buffer / 2}, 0)`) + const yAxisOffset = axis.y?.text ? `-${buffer / 2}` : buffer + const yAxisElement = svg.append('g') + .attr('transform', `translate(${yAxisOffset}, 0)`) .call(yAxis) // y-axis label - svg.append('text') - .attr('text-anchor', 'middle') - .attr('transform', `translate(${buffer - margins.left}, ${dimensions.height / 2}) rotate(-90)`) - .text(axis.y.text) + if (axis.y?.text) { + svg.append('text') + .attr('text-anchor', 'middle') + .attr('transform', `translate(${buffer - margins.left}, ${dimensions.height / 2}) rotate(-90)`) + .text(axis.y.text) + } else { + yAxisElement.selectAll('text') + .attr('text-anchor', 'start') + .attr('transform', `translate(${buffer - margins.left}, 0)`) + } return svg } From 8c45ecfab80fb85e4cce168063f0a92c8558e51e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 14:03:48 -0400 Subject: [PATCH 05/36] fix cell height --- ui/shared/components/graph/Gtex.jsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index d826193fa8..bfe59b236c 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -26,18 +26,24 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { const xDomain = [...new Set(isoformData.map(({ x }) => x))].sort(compareObjects('x')) const yDomain = [...new Set(isoformData.map(({ y }) => y))].sort(compareObjects('y')) + const dimensions = { + ...DIMENSIONS, + height: yDomain.length * 15, + } + const scale = { x: scaleBand() - .rangeRound([0, DIMENSIONS.width]) + .rangeRound([0, dimensions.width]) .domain(xDomain) .paddingInner(0.2), y: scaleBand() - .rangeRound([DIMENSIONS.height, 0]) - .domain(yDomain), + .rangeRound([dimensions.height, 0]) + .domain(yDomain) + .paddingInner(0.1), z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin } - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, {}) + const svg = initializeD3(containerElement, dimensions, MARGINS, scale, {}) const tooltip = new Tooltip(containerElement) // TODO fix actually render heatmap From 7bc959694c11f9436f38588bfa69d7a092f98d0e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 14:43:26 -0400 Subject: [PATCH 06/36] actually ender isoform --- ui/config/d3-bundle.js | 1 + ui/package-lock.json | 22 ++++++++++++++++++++++ ui/package.json | 1 + ui/shared/components/graph/Gtex.jsx | 26 ++++++++++---------------- ui/shared/components/graph/d3Utils.jsx | 2 ++ 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/ui/config/d3-bundle.js b/ui/config/d3-bundle.js index 49240569e2..2be223131d 100644 --- a/ui/config/d3-bundle.js +++ b/ui/config/d3-bundle.js @@ -1,6 +1,7 @@ // create a d3 bundle to provide as the global d3 for seqr, containing only modules which are needed for pedigreejs export * from 'd3-drag' export * from 'd3-hierarchy' +export * from 'd3-scale-chromatic' export * from 'd3-shape' export * from 'd3-selection' export * from 'd3-zoom' diff --git a/ui/package-lock.json b/ui/package-lock.json index 18347e0581..7bac15c4ea 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -17,6 +17,7 @@ "d3-hierarchy": "^1.1.9", "d3-random": "^3.0.1", "d3-scale": "^3.3.0", + "d3-scale-chromatic": "^3.1.0", "d3-selection": "^1.4.2", "d3-shape": "^3.2.0", "d3-zoom": "^1.8.3", @@ -5962,6 +5963,18 @@ "d3-time-format": "2 - 3" } }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-scale/node_modules/d3-array": { "version": "2.12.1", "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", @@ -23655,6 +23668,15 @@ } } }, + "d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "requires": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + } + }, "d3-selection": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.2.tgz", diff --git a/ui/package.json b/ui/package.json index a40ffb986f..16d07b045a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -76,6 +76,7 @@ "d3-hierarchy": "^1.1.9", "d3-random": "^3.0.1", "d3-scale": "^3.3.0", + "d3-scale-chromatic": "^3.1.0", "d3-selection": "^1.4.2", "d3-shape": "^3.2.0", "d3-zoom": "^1.8.3", diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index bfe59b236c..0c5358abf0 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -1,11 +1,12 @@ import React from 'react' import { deviation, extent, max, mean, min, quantile } from 'd3-array' import { randomNormal } from 'd3-random' -import { scaleBand, scaleLinear } from 'd3-scale' +import { scaleBand, scaleLinear, scaleSequential } from 'd3-scale' +import { interpolatePurples } from 'd3-scale-chromatic' import { area } from 'd3-shape' import { compareObjects } from 'shared/utils/sortUtils' -import { initializeD3, Tooltip } from './d3Utils' +import { initializeD3, log, Tooltip } from './d3Utils' import GtexLauncher, { queryGtex } from './GtexLauncher' const MARGINS = { @@ -30,7 +31,7 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { ...DIMENSIONS, height: yDomain.length * 15, } - + const scale = { x: scaleBand() .rangeRound([0, dimensions.width]) @@ -40,38 +41,31 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { .rangeRound([dimensions.height, 0]) .domain(yDomain) .paddingInner(0.1), - z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin + color: scaleSequential(interpolatePurples) + .domain([0, max(isoformData.map(({ value }) => log(value)))]), } const svg = initializeD3(containerElement, dimensions, MARGINS, scale, {}) const tooltip = new Tooltip(containerElement) - // TODO fix actually render heatmap isoformData.forEach(({ x, y, value }) => { svg.append('rect') - // .attr("row", (d) => `x${this.xList.indexOf(d.x)}`) - // .attr("col", (d) => `y${this.yList.indexOf(d.y)}`) + .attr('row', `x${xDomain.indexOf(x)}`) + .attr('col', `y${yDomain.indexOf(y)}`) .attr('x', scale.x(x)) .attr('y', scale.y(y)) .attr('rx', 5) .attr('ry', 5) - // .attr('class', 'exp-map-cell') .attr('width', scale.x.bandwidth()) .attr('height', scale.y.bandwidth()) .style('fill', '#eeeeee') .on('mouseover', () => { - tooltip.show( - `Tissue: ${x}
Isoform: ${y}
TPM: ${value}`, - x, - y, - ) + tooltip.show(`Tissue: ${x}
Isoform: ${y}
TPM: ${value}`, scale.x(x), scale.y(y)) }) .on('mouseout', () => { tooltip.hide() }) - // .style('fill', (d) => { - // return useNullColor&&d.value==0?nullColor:this.useLog?this.colorScale(this._log(d.value)):this.colorScale(d.value) - // }) + .style('fill', value === 0 ? '#DDDDDD' : scale.color(log(value))) }) } diff --git a/ui/shared/components/graph/d3Utils.jsx b/ui/shared/components/graph/d3Utils.jsx index de3f24cf92..5d16856d28 100644 --- a/ui/shared/components/graph/d3Utils.jsx +++ b/ui/shared/components/graph/d3Utils.jsx @@ -81,3 +81,5 @@ export class Tooltip { hide = () => this.tooltip.style('display', 'none') } + +export const log = x => Math.log(x + 1) / Math.log(10) From 24624218735cac9de89a1e13c84d519f1ef841fb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 14:53:30 -0400 Subject: [PATCH 07/36] fix order --- ui/shared/components/graph/Gtex.jsx | 10 ++++++---- ui/shared/components/graph/d3Utils.jsx | 2 -- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 0c5358abf0..09202d3928 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -6,7 +6,7 @@ import { interpolatePurples } from 'd3-scale-chromatic' import { area } from 'd3-shape' import { compareObjects } from 'shared/utils/sortUtils' -import { initializeD3, log, Tooltip } from './d3Utils' +import { initializeD3, Tooltip } from './d3Utils' import GtexLauncher, { queryGtex } from './GtexLauncher' const MARGINS = { @@ -25,7 +25,9 @@ const DIMENSIONS = { const renderIsoformHeatmap = (isoformData, containerElement) => { const xDomain = [...new Set(isoformData.map(({ x }) => x))].sort(compareObjects('x')) - const yDomain = [...new Set(isoformData.map(({ y }) => y))].sort(compareObjects('y')) + const yDomain = Object.entries( + isoformData.reduce((acc, { y, value }) => ({ ...acc, [y]: (acc[y] || 0) + value }), {}), + ).sort((a, b) => a[1] - b[1]).map(([y]) => y) const dimensions = { ...DIMENSIONS, @@ -42,7 +44,7 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { .domain(yDomain) .paddingInner(0.1), color: scaleSequential(interpolatePurples) - .domain([0, max(isoformData.map(({ value }) => log(value)))]), + .domain([0, max(isoformData.map(({ value }) => Math.log10(value + 1)))]), } const svg = initializeD3(containerElement, dimensions, MARGINS, scale, {}) @@ -65,7 +67,7 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { .on('mouseout', () => { tooltip.hide() }) - .style('fill', value === 0 ? '#DDDDDD' : scale.color(log(value))) + .style('fill', scale.color(Math.log10(value + 1))) }) } diff --git a/ui/shared/components/graph/d3Utils.jsx b/ui/shared/components/graph/d3Utils.jsx index 5d16856d28..de3f24cf92 100644 --- a/ui/shared/components/graph/d3Utils.jsx +++ b/ui/shared/components/graph/d3Utils.jsx @@ -81,5 +81,3 @@ export class Tooltip { hide = () => this.tooltip.style('display', 'none') } - -export const log = x => Math.log(x + 1) / Math.log(10) From b807064e101efb181e48daf1c412f7c0dc609f5f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 14:59:05 -0400 Subject: [PATCH 08/36] combine files --- ui/shared/components/graph/Gtex.jsx | 183 ++++++++++++++++++++-- ui/shared/components/graph/GtexViolin.jsx | 166 -------------------- 2 files changed, 169 insertions(+), 180 deletions(-) delete mode 100644 ui/shared/components/graph/GtexViolin.jsx diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 09202d3928..466405aaa0 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -1,5 +1,5 @@ import React from 'react' -import { deviation, extent, max, mean, min, quantile } from 'd3-array' +import { deviation, extent, max, mean, median, min, quantile } from 'd3-array' import { randomNormal } from 'd3-random' import { scaleBand, scaleLinear, scaleSequential } from 'd3-scale' import { interpolatePurples } from 'd3-scale-chromatic' @@ -13,13 +13,134 @@ const MARGINS = { top: 10, right: 50, bottom: 150, - left: 100, + left: 50, } const DIMENSIONS = { width: window.innerWidth * 0.8, height: 400, } +// Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/GeneExpressionViolinPlot.js + +const drawViolin = (svg, scale, tooltip) => (entry) => { + const violinG = svg.append('g') + .datum(entry) + + const eDomain = extent(entry.values) // get the max and min in entry.values + const q1 = quantile(entry.values, 0.25) + const q3 = quantile(entry.values, 0.75) + + // get the vertices + const rangeDeviation = (q3 - q1) / 1.34 + const kernelBandwidth = 1.06 * Math.min(deviation(entry.values), rangeDeviation) * (entry.values.length ** -0.2) + // use up to 100 vertices along the Y axis (to create the violin path) + const vertices = scale.y.ticks(100).map(x => [x, mean(entry.values, (v) => { + const u = (x - v) / kernelBandwidth + return (1 / Math.sqrt(2 * Math.PI)) * Math.exp(-0.5 * u * u) + }) / kernelBandwidth]).filter( + // filter the vertices that aren't in the entry.values + d => d[0] >= eDomain[0] && d[0] <= eDomain[1], + ) + + if (vertices.length < 1 || vertices.some(Number.isNaN)) { + return + } + + // define the z scale -- the violin width + const zMax = max(vertices, d => Math.abs(d[1])) + scale.z + .domain([-zMax, zMax]) + .range([scale.x(entry.label), scale.x(entry.label) + scale.x.bandwidth()]) + + // visual rendering + const violin = area() + .x0(d => scale.z(d[1])) + .x1(d => scale.z(-d[1])) + .y(d => scale.y(d[0])) + + const vPath = violinG.append('path') + .datum(vertices) + .attr('d', violin) + .style('fill', () => entry.color) + .style('opacity', 0.6) + + // boxplot + const z = scale.z.domain()[1] / 3 + const x = scale.z(-z) + + // interquartile range + violinG.append('rect') + .attr('x', x) + .attr('y', scale.y(q3)) + .attr('width', Math.abs(x - scale.z(z))) + .attr('height', Math.abs(scale.y(q3) - scale.y(q1))) + .style('fill', '#555f66') + + // the median line + const medianY = scale.y(entry.median) + violinG.append('line') + .attr('x1', x) + .attr('x2', scale.z(z)) + .attr('y1', medianY) + .attr('y2', medianY) + .style('stroke', '#fff') + .style('stroke-width', '2px') + + const jitter = randomNormal(0, z / 2) + const iqr = Math.abs(q3 - q1) + const upper = max(entry.values.filter(d => d <= q3 + (iqr * 1.5))) + const lower = min(entry.values.filter(d => d >= q1 - (iqr * 1.5))) + const outliers = entry.values.filter(d => d < lower || d > upper) + violinG.append('g') + .selectAll('circle') + .data(outliers) + .enter() + .append('circle') + .attr('cx', () => scale.z(jitter())) + .attr('cy', d => scale.y(d)) + .attr('fill', entry.color) + .attr('r', 1) + + // mouse events + violinG.on('mouseover', () => { + vPath.style('opacity', 1) + tooltip.show( + `${entry.label}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, + x + 70, + medianY < 40 ? 10 : medianY - 40, + ) + }) + violinG.on('mouseout', () => { + vPath.style('opacity', 0.6) + }) +} + +const renderViolinPlot = (violinPlotData, containerElement) => { + const xDomain = violinPlotData.map(({ label }) => label) + const yDomain = extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), [])) + + const scale = { + x: scaleBand() + .rangeRound([0, DIMENSIONS.width]) + .domain(xDomain) + .paddingInner(0.2), + y: scaleLinear() + .rangeRound([DIMENSIONS.height, 0]) + .domain(yDomain), + z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin + } + + const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, { + y: { + text: 'TPM', + transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), + }, + }) + + const tooltip = new Tooltip(containerElement) + violinPlotData.forEach(drawViolin(svg, scale, tooltip)) +} + // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/TranscriptBrowser.js // and https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/modules/Heatmap.js @@ -33,6 +154,10 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { ...DIMENSIONS, height: yDomain.length * 15, } + const margins = { + ...MARGINS, + left: 100, + } const scale = { x: scaleBand() @@ -47,7 +172,7 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { .domain([0, max(isoformData.map(({ value }) => Math.log10(value + 1)))]), } - const svg = initializeD3(containerElement, dimensions, MARGINS, scale, {}) + const svg = initializeD3(containerElement, dimensions, margins, scale, {}) const tooltip = new Tooltip(containerElement) isoformData.forEach(({ x, y, value }) => { @@ -75,7 +200,24 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { const loadTissueData = onSuccess => queryGtex('dataset/tissueSiteDetail', {}, onSuccess) -const renderGtex = (expressionData, tissueData, containerElement) => { +const renderGtexViolin = (expressionData, tissueData, containerElement) => { + if ((expressionData?.data || []).length < 1) { + return + } + const tissueLookup = tissueData.data.reduce( + (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, + ) + const violinPlotData = expressionData.data.map(({ tissueSiteDetailId, data }) => ({ + values: data.sort(), + median: median(data), + label: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, + color: `#${tissueLookup[tissueSiteDetailId]?.colorHex}`, + })) + violinPlotData.sort(compareObjects('label')) + renderViolinPlot(violinPlotData, containerElement) +} + +const renderGtexIsoform = (expressionData, tissueData, containerElement) => { if ((expressionData?.medianTranscriptExpression || []).length < 1) { return } @@ -83,17 +225,30 @@ const renderGtex = (expressionData, tissueData, containerElement) => { (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, ) // TODO better property names - const isoformData = expressionData.medianTranscriptExpression.map(({ median, transcriptId, tissueSiteDetailId, gencodeId }) => ({ - value: Number(median), - displayValue: Number(median), - y: transcriptId, - x: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, - tissueId: tissueSiteDetailId, - id: gencodeId, - })) + const isoformData = expressionData.medianTranscriptExpression.map( + ({ median: medianValue, transcriptId, tissueSiteDetailId, gencodeId }) => ({ + value: Number(medianValue), + displayValue: Number(medianValue), + y: transcriptId, + x: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, + tissueId: tissueSiteDetailId, + id: gencodeId, + }), + ) renderIsoformHeatmap(isoformData, containerElement) } -export default props => ( - +const GtexViolin = props => ( + ) + +const GtexIsoform = props => ( + +) + +export default GtexViolin diff --git a/ui/shared/components/graph/GtexViolin.jsx b/ui/shared/components/graph/GtexViolin.jsx deleted file mode 100644 index 1ed8c45ee7..0000000000 --- a/ui/shared/components/graph/GtexViolin.jsx +++ /dev/null @@ -1,166 +0,0 @@ -import React from 'react' -import { deviation, extent, max, mean, median, min, quantile } from 'd3-array' -import { randomNormal } from 'd3-random' -import { scaleBand, scaleLinear } from 'd3-scale' -import { area } from 'd3-shape' - -import { compareObjects } from 'shared/utils/sortUtils' -import { initializeD3, Tooltip } from './d3Utils' -import GtexLauncher, { queryGtex } from './GtexLauncher' - -const MARGINS = { - top: 10, - right: 50, - bottom: 150, - left: 50, -} -const DIMENSIONS = { - width: window.innerWidth * 0.8, - height: 400, -} - -// Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/GeneExpressionViolinPlot.js - -const drawViolin = (svg, scale, tooltip) => (entry) => { - const violinG = svg.append('g') - .datum(entry) - - const eDomain = extent(entry.values) // get the max and min in entry.values - const q1 = quantile(entry.values, 0.25) - const q3 = quantile(entry.values, 0.75) - - // get the vertices - const rangeDeviation = (q3 - q1) / 1.34 - const kernelBandwidth = 1.06 * Math.min(deviation(entry.values), rangeDeviation) * (entry.values.length ** -0.2) - // use up to 100 vertices along the Y axis (to create the violin path) - const vertices = scale.y.ticks(100).map(x => [x, mean(entry.values, (v) => { - const u = (x - v) / kernelBandwidth - return (1 / Math.sqrt(2 * Math.PI)) * Math.exp(-0.5 * u * u) - }) / kernelBandwidth]).filter( - // filter the vertices that aren't in the entry.values - d => d[0] >= eDomain[0] && d[0] <= eDomain[1], - ) - - if (vertices.length < 1 || vertices.some(Number.isNaN)) { - return - } - - // define the z scale -- the violin width - const zMax = max(vertices, d => Math.abs(d[1])) - scale.z - .domain([-zMax, zMax]) - .range([scale.x(entry.label), scale.x(entry.label) + scale.x.bandwidth()]) - - // visual rendering - const violin = area() - .x0(d => scale.z(d[1])) - .x1(d => scale.z(-d[1])) - .y(d => scale.y(d[0])) - - const vPath = violinG.append('path') - .datum(vertices) - .attr('d', violin) - .style('fill', () => entry.color) - .style('opacity', 0.6) - - // boxplot - const z = scale.z.domain()[1] / 3 - const x = scale.z(-z) - - // interquartile range - violinG.append('rect') - .attr('x', x) - .attr('y', scale.y(q3)) - .attr('width', Math.abs(x - scale.z(z))) - .attr('height', Math.abs(scale.y(q3) - scale.y(q1))) - .style('fill', '#555f66') - - // the median line - const medianY = scale.y(entry.median) - violinG.append('line') - .attr('x1', x) - .attr('x2', scale.z(z)) - .attr('y1', medianY) - .attr('y2', medianY) - .style('stroke', '#fff') - .style('stroke-width', '2px') - - const jitter = randomNormal(0, z / 2) - const iqr = Math.abs(q3 - q1) - const upper = max(entry.values.filter(d => d <= q3 + (iqr * 1.5))) - const lower = min(entry.values.filter(d => d >= q1 - (iqr * 1.5))) - const outliers = entry.values.filter(d => d < lower || d > upper) - violinG.append('g') - .selectAll('circle') - .data(outliers) - .enter() - .append('circle') - .attr('cx', () => scale.z(jitter())) - .attr('cy', d => scale.y(d)) - .attr('fill', entry.color) - .attr('r', 1) - - // mouse events - violinG.on('mouseover', () => { - vPath.style('opacity', 1) - tooltip.show( - `${entry.label}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, - x + 70, - medianY < 40 ? 10 : medianY - 40, - ) - }) - violinG.on('mouseout', () => { - vPath.style('opacity', 0.6) - }) -} - -const renderViolinPlot = (violinPlotData, containerElement) => { - const xDomain = violinPlotData.map(({ label }) => label) - const yDomain = extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), [])) - - const scale = { - x: scaleBand() - .rangeRound([0, DIMENSIONS.width]) - .domain(xDomain) - .paddingInner(0.2), - y: scaleLinear() - .rangeRound([DIMENSIONS.height, 0]) - .domain(yDomain), - z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin - } - - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, { - y: { - text: 'TPM', - transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), - }, - }) - - const tooltip = new Tooltip(containerElement) - violinPlotData.forEach(drawViolin(svg, scale, tooltip)) -} - -// seqr-specific code - -const loadTissueData = onSuccess => queryGtex('dataset/tissueSiteDetail', {}, onSuccess) - -const renderGtex = (expressionData, tissueData, containerElement) => { - if ((expressionData?.data || []).length < 1) { - return - } - const tissueLookup = tissueData.data.reduce( - (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, - ) - const violinPlotData = expressionData.data.map(({ tissueSiteDetailId, data }) => ({ - values: data.sort(), - median: median(data), - label: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, - color: `#${tissueLookup[tissueSiteDetailId]?.colorHex}`, - })) - violinPlotData.sort(compareObjects('label')) - renderViolinPlot(violinPlotData, containerElement) -} - -export default props => ( - -) From 32264af8e7a32584bd94cdacb1a2f6e12294ec63 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 16:35:57 -0400 Subject: [PATCH 09/36] shared utility for viloin plot --- ui/shared/components/graph/Gtex.jsx | 80 +++++++++++++++-------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 466405aaa0..a31f9e7bfb 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -15,8 +15,9 @@ const MARGINS = { bottom: 150, left: 50, } +const WIDTH = window.innerWidth * 0.8 const DIMENSIONS = { - width: window.innerWidth * 0.8, + width: WIDTH, height: 400, } @@ -50,7 +51,7 @@ const drawViolin = (svg, scale, tooltip) => (entry) => { const zMax = max(vertices, d => Math.abs(d[1])) scale.z .domain([-zMax, zMax]) - .range([scale.x(entry.label), scale.x(entry.label) + scale.x.bandwidth()]) + .range([scale.x(entry.tissue), scale.x(entry.tissue) + scale.x.bandwidth()]) // visual rendering const violin = area() @@ -105,7 +106,7 @@ const drawViolin = (svg, scale, tooltip) => (entry) => { violinG.on('mouseover', () => { vPath.style('opacity', 1) tooltip.show( - `${entry.label}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, + `${entry.tissue}
Sample size: ${entry.values.length}
Median TPM: ${entry.median.toPrecision(4)}
`, x + 70, medianY < 40 ? 10 : medianY - 40, ) @@ -115,31 +116,18 @@ const drawViolin = (svg, scale, tooltip) => (entry) => { }) } -const renderViolinPlot = (violinPlotData, containerElement) => { - const xDomain = violinPlotData.map(({ label }) => label) - const yDomain = extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), [])) - - const scale = { - x: scaleBand() - .rangeRound([0, DIMENSIONS.width]) - .domain(xDomain) - .paddingInner(0.2), - y: scaleLinear() - .rangeRound([DIMENSIONS.height, 0]) - .domain(yDomain), - z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin - } - - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, { - y: { - text: 'TPM', - transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), - }, - }) - - const tooltip = new Tooltip(containerElement) - violinPlotData.forEach(drawViolin(svg, scale, tooltip)) -} +const getViolinScales = violinPlotData => ({ + y: scaleLinear() + .rangeRound([DIMENSIONS.height, 0]) + .domain(extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), []))), + z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin +}) +const getViolinAxis = scale => ({ + y: { + text: 'TPM', + transform: yAxis => yAxis.tickValues(scale.y.ticks(5)), + }, +}) // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/TranscriptBrowser.js // and https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/modules/Heatmap.js @@ -200,23 +188,39 @@ const renderIsoformHeatmap = (isoformData, containerElement) => { const loadTissueData = onSuccess => queryGtex('dataset/tissueSiteDetail', {}, onSuccess) -const renderGtexViolin = (expressionData, tissueData, containerElement) => { - if ((expressionData?.data || []).length < 1) { +const renderGtex = ( + responseKey, parseExpressionData, renderDataPoint, getScales, getAxis, +) => (expressionData, tissueData, containerElement) => { + if (((expressionData || {})[responseKey] || []).length < 1) { return } const tissueLookup = tissueData.data.reduce( (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, ) - const violinPlotData = expressionData.data.map(({ tissueSiteDetailId, data }) => ({ - values: data.sort(), - median: median(data), - label: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, - color: `#${tissueLookup[tissueSiteDetailId]?.colorHex}`, - })) - violinPlotData.sort(compareObjects('label')) - renderViolinPlot(violinPlotData, containerElement) + const plotData = expressionData[responseKey].map( + ({ tissueSiteDetailId, ...data }) => parseExpressionData(data, tissueLookup[tissueSiteDetailId]), + ) + const xDomain = plotData.map(({ tissue }) => tissue).sort() + const scale = { + x: scaleBand() + .rangeRound([0, WIDTH]) + .domain(xDomain) + .paddingInner(0.2), + ...getScales(plotData), + } + const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, getAxis ? getAxis(scale) : {}) + const tooltip = new Tooltip(containerElement) + plotData.forEach(renderDataPoint(svg, scale, tooltip)) } +const parseViolinExpressionData = ({ data }, tissue) => ({ + values: data.sort(), + median: median(data), + tissue: (tissue || {}).tissueSiteDetail, + color: `#${(tissue || {}).colorHex}`, +}) +const renderGtexViolin = renderGtex('data', parseViolinExpressionData, drawViolin, getViolinScales, getViolinAxis) + const renderGtexIsoform = (expressionData, tissueData, containerElement) => { if ((expressionData?.medianTranscriptExpression || []).length < 1) { return From e024b8ccdfc12ac7d7bff2c71b5c46dae77250a3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 Aug 2025 17:01:33 -0400 Subject: [PATCH 10/36] shared isoform render logic --- ui/shared/components/graph/Gtex.jsx | 125 +++++++++++----------------- 1 file changed, 48 insertions(+), 77 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index a31f9e7bfb..18c0b4a446 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -5,7 +5,6 @@ import { scaleBand, scaleLinear, scaleSequential } from 'd3-scale' import { interpolatePurples } from 'd3-scale-chromatic' import { area } from 'd3-shape' -import { compareObjects } from 'shared/utils/sortUtils' import { initializeD3, Tooltip } from './d3Utils' import GtexLauncher, { queryGtex } from './GtexLauncher' @@ -13,13 +12,9 @@ const MARGINS = { top: 10, right: 50, bottom: 150, - left: 50, } const WIDTH = window.innerWidth * 0.8 -const DIMENSIONS = { - width: WIDTH, - height: 400, -} +const VIOLIN_HEIGHT = 400 // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/GeneExpressionViolinPlot.js @@ -117,10 +112,14 @@ const drawViolin = (svg, scale, tooltip) => (entry) => { } const getViolinScales = violinPlotData => ({ - y: scaleLinear() - .rangeRound([DIMENSIONS.height, 0]) - .domain(extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), []))), - z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin + scales: { + y: scaleLinear() + .rangeRound([VIOLIN_HEIGHT, 0]) + .domain(extent(violinPlotData.reduce((acc, { values }) => ([...acc, ...values]), []))), + z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin + }, + height: VIOLIN_HEIGHT, + left: 50, }) const getViolinAxis = scale => ({ y: { @@ -132,56 +131,44 @@ const getViolinAxis = scale => ({ // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/TranscriptBrowser.js // and https://github.com/broadinstitute/gtex-viz/blob/f9d18299b2bcb69e4a919a4afa359c99a33fbc3b/src/modules/Heatmap.js -const renderIsoformHeatmap = (isoformData, containerElement) => { - const xDomain = [...new Set(isoformData.map(({ x }) => x))].sort(compareObjects('x')) - const yDomain = Object.entries( - isoformData.reduce((acc, { y, value }) => ({ ...acc, [y]: (acc[y] || 0) + value }), {}), - ).sort((a, b) => a[1] - b[1]).map(([y]) => y) +const parseIsoformExpressionData = ({ median: medianValue, transcriptId }) => ({ + value: Number(medianValue), transcriptId, +}) - const dimensions = { - ...DIMENSIONS, - height: yDomain.length * 15, - } - const margins = { - ...MARGINS, - left: 100, - } +const getIsoformScales = (isoformData) => { + const yDomain = Object.entries(isoformData.reduce( + (acc, { transcriptId, value }) => ({ ...acc, [transcriptId]: (acc[transcriptId] || 0) + value }), {}, + )).sort((a, b) => a[1] - b[1]).map(([transcriptId]) => transcriptId) - const scale = { - x: scaleBand() - .rangeRound([0, dimensions.width]) - .domain(xDomain) - .paddingInner(0.2), + const height = yDomain.length * 15 + + const scales = { y: scaleBand() - .rangeRound([dimensions.height, 0]) + .rangeRound([height, 0]) .domain(yDomain) .paddingInner(0.1), color: scaleSequential(interpolatePurples) .domain([0, max(isoformData.map(({ value }) => Math.log10(value + 1)))]), } + return { scales, height, left: 100 } +} - const svg = initializeD3(containerElement, dimensions, margins, scale, {}) - - const tooltip = new Tooltip(containerElement) - isoformData.forEach(({ x, y, value }) => { - svg.append('rect') - .attr('row', `x${xDomain.indexOf(x)}`) - .attr('col', `y${yDomain.indexOf(y)}`) - .attr('x', scale.x(x)) - .attr('y', scale.y(y)) - .attr('rx', 5) - .attr('ry', 5) - .attr('width', scale.x.bandwidth()) - .attr('height', scale.y.bandwidth()) - .style('fill', '#eeeeee') - .on('mouseover', () => { - tooltip.show(`Tissue: ${x}
Isoform: ${y}
TPM: ${value}`, scale.x(x), scale.y(y)) - }) - .on('mouseout', () => { - tooltip.hide() - }) - .style('fill', scale.color(Math.log10(value + 1))) - }) +const drawIsoformCell = (svg, scale, tooltip) => ({ tissue, transcriptId, value }) => { + svg.append('rect') + .attr('x', scale.x(tissue)) + .attr('y', scale.y(transcriptId)) + .attr('rx', 5) + .attr('ry', 5) + .attr('width', scale.x.bandwidth()) + .attr('height', scale.y.bandwidth()) + .style('fill', '#eeeeee') + .on('mouseover', () => { + tooltip.show(`Tissue: ${tissue}
Isoform: ${transcriptId}
TPM: ${value}`, scale.x(tissue), scale.y(transcriptId)) + }) + .on('mouseout', () => { + tooltip.hide() + }) + .style('fill', scale.color(Math.log10(value + 1))) } // seqr-specific code @@ -197,18 +184,22 @@ const renderGtex = ( const tissueLookup = tissueData.data.reduce( (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, ) - const plotData = expressionData[responseKey].map( - ({ tissueSiteDetailId, ...data }) => parseExpressionData(data, tissueLookup[tissueSiteDetailId]), - ) + const plotData = expressionData[responseKey].map(({ tissueSiteDetailId, ...data }) => ({ + ...parseExpressionData(data, tissueLookup[tissueSiteDetailId]), + tissue: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, + })) const xDomain = plotData.map(({ tissue }) => tissue).sort() + const { scales, height, left } = getScales(plotData) + const dimensions = { width: WIDTH, height } + const margins = { ...MARGINS, left } const scale = { x: scaleBand() .rangeRound([0, WIDTH]) .domain(xDomain) .paddingInner(0.2), - ...getScales(plotData), + ...scales, } - const svg = initializeD3(containerElement, DIMENSIONS, MARGINS, scale, getAxis ? getAxis(scale) : {}) + const svg = initializeD3(containerElement, dimensions, margins, scale, getAxis ? getAxis(scale) : {}) const tooltip = new Tooltip(containerElement) plotData.forEach(renderDataPoint(svg, scale, tooltip)) } @@ -216,31 +207,11 @@ const renderGtex = ( const parseViolinExpressionData = ({ data }, tissue) => ({ values: data.sort(), median: median(data), - tissue: (tissue || {}).tissueSiteDetail, color: `#${(tissue || {}).colorHex}`, }) const renderGtexViolin = renderGtex('data', parseViolinExpressionData, drawViolin, getViolinScales, getViolinAxis) -const renderGtexIsoform = (expressionData, tissueData, containerElement) => { - if ((expressionData?.medianTranscriptExpression || []).length < 1) { - return - } - const tissueLookup = tissueData.data.reduce( - (acc, { tissueSiteDetailId, ...data }) => ({ ...acc, [tissueSiteDetailId]: data }), {}, - ) - // TODO better property names - const isoformData = expressionData.medianTranscriptExpression.map( - ({ median: medianValue, transcriptId, tissueSiteDetailId, gencodeId }) => ({ - value: Number(medianValue), - displayValue: Number(medianValue), - y: transcriptId, - x: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, - tissueId: tissueSiteDetailId, - id: gencodeId, - }), - ) - renderIsoformHeatmap(isoformData, containerElement) -} +const renderGtexIsoform = renderGtex('medianTranscriptExpression', parseIsoformExpressionData, drawIsoformCell, getIsoformScales) const GtexViolin = props => ( From 01ea49191d41cd18a846b3fec23b84886451de2c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 14:45:24 -0400 Subject: [PATCH 11/36] render gtex tabs --- ui/shared/components/graph/Gtex.jsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 18c0b4a446..845d89faef 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -4,6 +4,7 @@ import { randomNormal } from 'd3-random' import { scaleBand, scaleLinear, scaleSequential } from 'd3-scale' import { interpolatePurples } from 'd3-scale-chromatic' import { area } from 'd3-shape' +import { Tab } from 'semantic-ui-react' import { initializeD3, Tooltip } from './d3Utils' import GtexLauncher, { queryGtex } from './GtexLauncher' @@ -226,4 +227,23 @@ const GtexIsoform = props => ( /> ) -export default GtexViolin +const gtexPanes = props => [ + { + menuItem: 'Gene Expression', + render: () => ( + + + + ), + }, + { + menuItem: 'Isoform Expression', + render: () => ( + + + + ), + }, +] + +export default props => From 281167dd2882fc785b6d90a6c664bd32f6921602 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 14:53:01 -0400 Subject: [PATCH 12/36] fix scale --- ui/shared/components/graph/Gtex.jsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 845d89faef..905a9015ad 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -13,6 +13,7 @@ const MARGINS = { top: 10, right: 50, bottom: 150, + left: 50, } const WIDTH = window.innerWidth * 0.8 const VIOLIN_HEIGHT = 400 @@ -120,7 +121,7 @@ const getViolinScales = violinPlotData => ({ z: scaleLinear(), // the violin width, domain and range are determined later individually for each violin }, height: VIOLIN_HEIGHT, - left: 50, + leftOffset: 0, }) const getViolinAxis = scale => ({ y: { @@ -151,7 +152,7 @@ const getIsoformScales = (isoformData) => { color: scaleSequential(interpolatePurples) .domain([0, max(isoformData.map(({ value }) => Math.log10(value + 1)))]), } - return { scales, height, left: 100 } + return { scales, height, leftOffset: 50 } } const drawIsoformCell = (svg, scale, tooltip) => ({ tissue, transcriptId, value }) => { @@ -190,12 +191,12 @@ const renderGtex = ( tissue: tissueLookup[tissueSiteDetailId]?.tissueSiteDetail, })) const xDomain = plotData.map(({ tissue }) => tissue).sort() - const { scales, height, left } = getScales(plotData) + const { scales, height, leftOffset } = getScales(plotData) const dimensions = { width: WIDTH, height } - const margins = { ...MARGINS, left } + const margins = { ...MARGINS, left: MARGINS.left + leftOffset } const scale = { x: scaleBand() - .rangeRound([0, WIDTH]) + .rangeRound([0, WIDTH - leftOffset]) .domain(xDomain) .paddingInner(0.2), ...scales, From ae58f94269a9b7b9e7aa644048f1dc0ec8e7e34c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 14:57:56 -0400 Subject: [PATCH 13/36] clean up --- ui/shared/components/graph/Gtex.jsx | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 905a9015ad..119e539ea0 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -215,25 +215,12 @@ const renderGtexViolin = renderGtex('data', parseViolinExpressionData, drawVioli const renderGtexIsoform = renderGtex('medianTranscriptExpression', parseIsoformExpressionData, drawIsoformCell, getIsoformScales) -const GtexViolin = props => ( - -) - -const GtexIsoform = props => ( - -) - const gtexPanes = props => [ { menuItem: 'Gene Expression', render: () => ( - + ), }, @@ -241,7 +228,12 @@ const gtexPanes = props => [ menuItem: 'Isoform Expression', render: () => ( - + ), }, From dd99651e42b59797b3853c5d7ee82d0f6ae34504 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 14:58:52 -0400 Subject: [PATCH 14/36] Revert "clean up" This reverts commit ae58f94269a9b7b9e7aa644048f1dc0ec8e7e34c. --- ui/shared/components/graph/Gtex.jsx | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ui/shared/components/graph/Gtex.jsx b/ui/shared/components/graph/Gtex.jsx index 119e539ea0..905a9015ad 100644 --- a/ui/shared/components/graph/Gtex.jsx +++ b/ui/shared/components/graph/Gtex.jsx @@ -215,12 +215,25 @@ const renderGtexViolin = renderGtex('data', parseViolinExpressionData, drawVioli const renderGtexIsoform = renderGtex('medianTranscriptExpression', parseIsoformExpressionData, drawIsoformCell, getIsoformScales) +const GtexViolin = props => ( + +) + +const GtexIsoform = props => ( + +) + const gtexPanes = props => [ { menuItem: 'Gene Expression', render: () => ( - + ), }, @@ -228,12 +241,7 @@ const gtexPanes = props => [ menuItem: 'Isoform Expression', render: () => ( - + ), }, From d76b89bce7f8e5a9103cd4d90df575276063000d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:01:26 -0400 Subject: [PATCH 15/36] remove unused dag triggers --- seqr/urls.py | 3 +-- seqr/views/apis/data_manager_api.py | 26 +++---------------- .../TriggerSearchDataUpdatePages.jsx | 10 ------- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index f93c3a603a..1ce292f0d2 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -125,7 +125,7 @@ from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, proxy_to_kibana, load_phenotype_prioritization_data, \ validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ - trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data + trigger_delete_project, trigger_delete_family from seqr.views.apis.report_api import \ anvil_export, \ family_metadata, \ @@ -342,7 +342,6 @@ 'data_management/add_igv': receive_bulk_igv_table_handler, 'data_management/trigger_delete_project': trigger_delete_project, 'data_management/trigger_delete_family': trigger_delete_family, - 'data_management/trigger_update_search_reference_data': trigger_update_search_reference_data, 'summary_data/saved_variants/(?P[^/]+)': saved_variants_page, 'summary_data/hpo/(?P[^/]+)': hpo_summary_data, diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 5992cd9e2e..8f89ddddcb 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -22,7 +22,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list -from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled +from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled # TODO still used anywhere? from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers @@ -438,37 +438,19 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(project, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) - - -@data_manager_required -def trigger_update_search_reference_data(request): - request_json = json.loads(request.body) - return _trigger_data_update(project=None, dag_id='UPDATE_REFERENCE_DATASETS', **request_json) + return _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) def _raise_backend_not_implemented(*args, **kwargs): raise ErrorsWarningsException(['This functionality is not available in the current search backend']) -def _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, **kwargs): +def _trigger_data_update(project, clickhouse_func, **kwargs): kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} - info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(project, **kwargs) + info = backend_specific_call(_raise_backend_not_implemented, _raise_backend_not_implemented, clickhouse_func)(project, **kwargs) return create_json_response({'info': [info]}) -def _trigger_dag(project, dag_id=None, family_guid=None, **kwargs): - if not is_airflow_enabled(): - raise PermissionDenied() - if family_guid: - kwargs['family_guids'] = [family_guid] - try: - dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) - except Exception as e: - raise ErrorsWarningsException([str(e)]) - return f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}' - - # Hop-by-hop HTTP response headers shouldn't be forwarded. # More info at: http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1 EXCLUDE_HTTP_RESPONSE_HEADERS = { diff --git a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index 7aa4ca250b..6a0e6fdfaa 100644 --- a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -9,7 +9,6 @@ import { DATASET_TYPE_SNV_INDEL_CALLS, DATASET_TYPE_SV_CALLS, DATASET_TYPE_MITO_CALLS, - GENOME_VERSION_FIELD, } from 'shared/utils/constants' import { snakecaseToTitlecase } from 'shared/utils/stringUtils' @@ -46,10 +45,6 @@ const FAMILY_FIELDS = [ }, DATASET_TYPE_FIELD, ] -const REFERENCE_DATASET_FIELDS = [ - { ...GENOME_VERSION_FIELD, validate: validators.required }, - DATASET_TYPE_FIELD, -] const TriggerSearchDataUpdateForm = ({ path, fields }) => ( ( ) -const TriggerUpdateReferenceDataset = () => ( - -) - export default [ { path: 'delete_search_project', component: TriggerDeleteProjects }, { path: 'delete_search_family', component: TriggerDeleteFamilies }, - { path: 'update_search_reference_data', component: TriggerUpdateReferenceDataset }, ] From 81ae3b524dfdc40267b97ad9f43149e014f7f6cf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:59:16 -0400 Subject: [PATCH 16/36] remve airflow from deactivate project --- seqr/management/commands/deactivate_project_search.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 57088c089e..19001db8ff 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -3,7 +3,6 @@ from clickhouse_search.search import delete_clickhouse_project from seqr.models import Project, Sample from seqr.utils.search.utils import backend_specific_call -from seqr.views.utils.airflow_utils import is_airflow_enabled, trigger_airflow_dag, DELETE_PROJECTS_DAG_NAME import logging logger = logging.getLogger(__name__) @@ -27,15 +26,9 @@ def handle(self, *args, **options): dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() for dataset_type, sample_type in dataset_types: backend_specific_call( - lambda *args, **kwargs: True, self._trigger_delete_dag, self._delete_clickhouse_project, + lambda *args, **kwargs: True, self._delete_clickhouse_project, )(project, dataset_type, sample_type) - @staticmethod - def _trigger_delete_dag(project, dataset_type, sample_type): - if is_airflow_enabled(): - trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') - logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') - @staticmethod def _delete_clickhouse_project(project, dataset_type, sample_type): if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: From 021488533b35b36d10425bb5bbacbaacf46cdcf4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:32:04 -0400 Subject: [PATCH 17/36] remove dag trigger from transfer families --- .../transfer_families_to_different_project.py | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 27a44c6a85..2fa6c2581e 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,16 +1,13 @@ -from collections import defaultdict - from django.core.management.base import BaseCommand from seqr.models import Project, Family, VariantTag, VariantTagType, Sample from seqr.utils.search.utils import backend_specific_call -from seqr.views.utils.airflow_utils import trigger_airflow_dag, DELETE_FAMILIES_DAG_NAME, DagRunningException import logging logger = logging.getLogger(__name__) -def _disable_search(families, from_project): +def _disable_search(families): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) updated_family_dataset_types = None if search_samples: @@ -23,19 +20,6 @@ def _disable_search(families, from_project): ) return updated_family_dataset_types -def _disable_search_trigger_delete_families_dags(families, from_project): - updated_family_dataset_types = _disable_search(families, from_project) - updated_families_by_dataset_type = defaultdict(list) - for dataset_type, family_guid in updated_family_dataset_types: - updated_families_by_dataset_type[dataset_type].append(family_guid) - - for dataset_type, family_guids in sorted(updated_families_by_dataset_type.items()): - try: - trigger_airflow_dag(DELETE_FAMILIES_DAG_NAME, from_project, dataset_type, family_guids=sorted(family_guids)) - logger.info(f'Successfully triggered DELETE_FAMILIES DAG for {len(family_guids)} {dataset_type} families') - except Exception as e: - logger_call = logger.warning if isinstance(e, DagRunningException) else logger.error - logger_call(str(e)) class Command(BaseCommand): def add_arguments(self, parser): @@ -64,7 +48,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search_trigger_delete_families_dags, _disable_search)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) From aac719283cb7a39451d8e1e104a29a563ac93a7b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:34:14 -0400 Subject: [PATCH 18/36] remove generic airflow dag triggering --- seqr/utils/search/add_data_utils.py | 4 ++-- seqr/views/utils/airflow_utils.py | 29 +---------------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 3241d42159..848ba1e5a7 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -103,7 +103,7 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N update={'Status': status, **(additional_update or {})}, ) -def format_loading_pipeline_variables( +def _format_loading_pipeline_variables( projects: list[Project], genome_version: str, dataset_type: str, sample_type: str = None, **kwargs ): variables = { @@ -119,7 +119,7 @@ def format_loading_pipeline_variables( def prepare_data_loading_request(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str, genome_version: str, data_path: str, user: User, pedigree_dir: str, raise_pedigree_error: bool = False, skip_validation: bool = False, skip_check_sex_and_relatedness: bool = False, vcf_sample_id_map=None): - variables = format_loading_pipeline_variables( + variables = _format_loading_pipeline_variables( projects, genome_version, dataset_type, diff --git a/seqr/views/utils/airflow_utils.py b/seqr/views/utils/airflow_utils.py index 634903a192..2bc1d089bf 100644 --- a/seqr/views/utils/airflow_utils.py +++ b/seqr/views/utils/airflow_utils.py @@ -4,17 +4,14 @@ from google.auth.transport.requests import AuthorizedSession import json -from seqr.models import Project from seqr.utils.communication_utils import safe_post_to_slack -from seqr.utils.search.add_data_utils import prepare_data_loading_request, format_loading_pipeline_variables +from seqr.utils.search.add_data_utils import prepare_data_loading_request from seqr.utils.logging_utils import SeqrLogger from settings import AIRFLOW_WEBSERVER_URL, SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL logger = SeqrLogger(__name__) LOADING_PIPELINE_DAG_NAME = 'LOADING_PIPELINE' -DELETE_FAMILIES_DAG_NAME = 'DELETE_FAMILIES' -DELETE_PROJECTS_DAG_NAME = 'DELETE_PROJECTS' AIRFLOW_AUTH_SCOPE = "https://www.googleapis.com/auth/cloud-platform" SEQR_V3_PEDIGREE_GS_PATH = 'gs://seqr-loading-temp/v3.1' @@ -23,10 +20,6 @@ class DagRunningException(Exception): pass -def is_airflow_enabled(): - return bool(AIRFLOW_WEBSERVER_URL) - - def trigger_airflow_data_loading(*args, user: User, success_message: str, success_slack_channel: str, error_message: str, is_internal: bool = False, **kwargs): success = True @@ -52,20 +45,6 @@ def trigger_airflow_data_loading(*args, user: User, success_message: str, succes return success -def trigger_airflow_dag(dag_id: str, project: Project, dataset_type: str, genome_version: str = None, **kwargs): - variables = format_loading_pipeline_variables( - [project] if project else [], - project.genome_version if project else genome_version, - dataset_type, - **kwargs - ) - _check_dag_running_state(dag_id) - _update_variables(variables, dag_id) - _wait_for_dag_variable_update(variables, dag_id) - _trigger_dag(dag_id) - return variables - - def _send_load_data_slack_msg(messages: list[str], channel: str, dag: dict): message = '\n\n '.join(messages) message_content = f"""{message} @@ -100,12 +79,6 @@ def _wait_for_dag_variable_update_via_tasks(projects: list[str], dag_name: str): dag_projects = _get_task_ids(dag_name) -def _wait_for_dag_variable_update(variables: dict, dag_name: str): - existing_variables = _get_variables(dag_name) - while any(existing_variables.get(k) != v for k, v in variables.items()): - existing_variables = _get_variables(dag_name) - - def _update_variables(val: dict, dag_name: str): endpoint = f'variables/{dag_name}' val_str = json.dumps(val) From 7da9d9ffec67afb924da4a72bb76b9b0d8fa3e76 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:43:15 -0400 Subject: [PATCH 19/36] placeholder delete family function --- clickhouse_search/search.py | 4 ++++ .../commands/transfer_families_to_different_project.py | 9 +++++---- seqr/views/apis/data_manager_api.py | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 6f42e1aa2a..e4e55c23e3 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -633,3 +633,7 @@ def delete_clickhouse_project(project, dataset_type=None, **kwargs): cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') return f'Deleted all {dataset_type} search data for project {project.name}' + + +def delete_clickhouse_family(project, family_guid, dataset_type=None, **kwargs): + return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}' diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 2fa6c2581e..2882583ba4 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,5 +1,6 @@ from django.core.management.base import BaseCommand +from clickhouse_search.search import delete_clickhouse_family from seqr.models import Project, Family, VariantTag, VariantTagType, Sample from seqr.utils.search.utils import backend_specific_call @@ -7,9 +8,8 @@ logger = logging.getLogger(__name__) -def _disable_search(families): +def _disable_search(families, from_project): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) - updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() updated_family_dataset_types = list(search_samples.values_list('dataset_type', 'individual__family__guid').distinct()) @@ -18,7 +18,8 @@ def _disable_search(families): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - return updated_family_dataset_types + for dataset_type, family_guid in updated_family_dataset_types: + logger.info(delete_clickhouse_family(from_project, family_guid=family_guid, dataset_type=dataset_type)) class Command(BaseCommand): @@ -48,7 +49,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families) + backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 8f89ddddcb..9e63df6879 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError -from clickhouse_search.search import delete_clickhouse_project +from clickhouse_search.search import delete_clickhouse_project, delete_clickhouse_family from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call @@ -429,7 +429,7 @@ def trigger_delete_project(request): project_guid = request_json.pop('project') project = Project.objects.get(guid=project_guid) return _trigger_data_update( - project, dag_id='DELETE_PROJECTS', clickhouse_func=delete_clickhouse_project, **request_json, + project, delete_clickhouse_project, **request_json, ) @@ -438,7 +438,7 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) + return _trigger_data_update(project, delete_clickhouse_family, family_guid=family_guid, **request_json) def _raise_backend_not_implemented(*args, **kwargs): From e553a2ac51bfd68c11a523fba1654c874345ff34 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 17:23:53 -0400 Subject: [PATCH 20/36] clean up --- seqr/management/commands/deactivate_project_search.py | 2 +- .../commands/transfer_families_to_different_project.py | 10 +++++++--- seqr/views/apis/data_manager_api.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 19001db8ff..b0885bc5df 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -26,7 +26,7 @@ def handle(self, *args, **options): dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() for dataset_type, sample_type in dataset_types: backend_specific_call( - lambda *args, **kwargs: True, self._delete_clickhouse_project, + lambda *args, **kwargs: True, lambda *args, **kwargs: True, self._delete_clickhouse_project, )(project, dataset_type, sample_type) @staticmethod diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 2882583ba4..197871f99c 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -18,8 +18,12 @@ def _disable_search(families, from_project): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - for dataset_type, family_guid in updated_family_dataset_types: - logger.info(delete_clickhouse_family(from_project, family_guid=family_guid, dataset_type=dataset_type)) + return updated_family_dataset_types + +def _disable_search_clickhouse(families, from_project): + updated_family_dataset_types = _disable_search(families, from_project) + for dataset_type, family_guid in updated_family_dataset_types: + logger.info(delete_clickhouse_family(from_project, family_guid=family_guid, dataset_type=dataset_type)) class Command(BaseCommand): @@ -49,7 +53,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search, _disable_search_clickhouse)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 9e63df6879..4c7af6a934 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -22,7 +22,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list -from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled # TODO still used anywhere? +from seqr.views.utils.airflow_utils import trigger_airflow_data_loading from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers From 4aeb6e469b0eb83eeeb184ff6b7b57a7ce8504bd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 2 Sep 2025 17:26:39 -0400 Subject: [PATCH 21/36] clean up --- .../commands/transfer_families_to_different_project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 197871f99c..e4921bb6e1 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -10,6 +10,7 @@ def _disable_search(families, from_project): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) + updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() updated_family_dataset_types = list(search_samples.values_list('dataset_type', 'individual__family__guid').distinct()) @@ -18,7 +19,7 @@ def _disable_search(families, from_project): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - return updated_family_dataset_types + return updated_family_dataset_types def _disable_search_clickhouse(families, from_project): updated_family_dataset_types = _disable_search(families, from_project) From 52805dfc2b22769b2963979486f04d109913effa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Sep 2025 10:29:49 -0400 Subject: [PATCH 22/36] fix manage command tests --- .../tests/deactivate_project_search_tests.py | 29 +--------- ...fer_families_to_different_project_tests.py | 55 ++----------------- 2 files changed, 9 insertions(+), 75 deletions(-) diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index 3a32a408e0..cbd13ae46b 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -9,7 +9,7 @@ from clickhouse_search.models import EntriesSnvIndel, EntriesMito, EntriesGcnv, ProjectGtStatsSnvIndel, \ ProjectGtStatsMito, AnnotationsSnvIndel, AnnotationsMito from seqr.models import Sample, Project -from seqr.views.utils.test_utils import AirflowTestCase, AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase PROJECT_GUID = 'R0001_1kg' VARIANT_ID = '21-3343353-GAGA-G' @@ -100,31 +100,8 @@ def _assert_expected_delete(self): }) -class HailBackendDeactivateProjectSearchTest(AirflowTestCase, DeactivateProjectSearchTest): +class HailBackendDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest): fixtures = ['users', '1kg_project'] CLICKHOUSE_HOSTNAME = '' - DAG_NAME = 'DELETE_PROJECTS' - DAG_VARIABLES = { - 'projects_to_run': [PROJECT_GUID], - 'dataset_type': 'SNV_INDEL', - 'reference_genome': 'GRCh38', - } - - DELETE_SUCCESS_LOGS = [ - ('Successfully triggered DELETE_PROJECTS DAG for MITO R0001_1kg', None), - ('Successfully triggered DELETE_PROJECTS DAG for SNV_INDEL R0001_1kg', None), - ('Successfully triggered DELETE_PROJECTS DAG for SV R0001_1kg', None), - ] - - def _assert_expected_delete(self): - self.assert_airflow_calls(self.DAG_VARIABLES, 5) - - def _add_update_check_dag_responses(self, **kwargs): - return self._add_check_dag_variable_responses(self.DAG_VARIABLES, **kwargs) - - def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): - variables_update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' - super()._assert_update_check_airflow_calls(call_count, offset, variables_update_check_path) - - + ES_HOSTNAME = '' diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index b4a93735d4..07f9062a96 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -2,7 +2,7 @@ from django.core.management import call_command from seqr.models import Family, VariantTagType, VariantTag, Sample -from seqr.views.utils.test_utils import AirflowTestCase, AuthenticationTestCase, AnvilAuthenticationTestCase +from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase class TransferFamiliesTest(object): @@ -60,61 +60,18 @@ class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTe ES_HOSTNAME = '' LOGS = [ ('Disabled search for 7 samples in the following 1 families: 2', None), + ('Clickhouse does not support deleting individual families from project. Manually delete MITO data for F000002_2 in project R0001_1kg', None), + ('Clickhouse does not support deleting individual families from project. Manually delete SNV_INDEL data for F000002_2 in project R0001_1kg', None), + ('Clickhouse does not support deleting individual families from project. Manually delete SV data for F000002_2 in project R0001_1kg', None), ] -class TransferFamiliesAirflowTest(TransferFamiliesTest, AirflowTestCase): +class TransferFamiliesHailBackendTest(TransferFamiliesTest, AuthenticationTestCase): fixtures = ['users', '1kg_project'] - PROJECT_GUID = 'R0001_1kg' # from-project - DAG_NAME = 'DELETE_FAMILIES' + ES_HOSTNAME = '' CLICKHOUSE_HOSTNAME = '' LOGS = [ ('Disabled search for 7 samples in the following 1 families: 2', None), - ('Successfully triggered DELETE_FAMILIES DAG for 1 MITO families', None), - ('Successfully triggered DELETE_FAMILIES DAG for 1 SNV_INDEL families', None), - ('400 Client Error: Bad Request for url: http://testairflowserver/api/v1/variables/DELETE_FAMILIES', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }) ] - - def setUp(self): - super().setUp() - self.set_up_one_dag(dataset_type='SNV_INDEL') - self.set_up_one_dag(dataset_type='SV', status=400) - - def set_up_one_dag(self, **kwargs): - dataset_type = kwargs.pop('dataset_type', 'MITO') - super().set_up_one_dag(dataset_type=dataset_type, **kwargs) - - def _get_dag_variables(self, dataset_type, **kwargs): - return { - 'projects_to_run': [self.PROJECT_GUID], - 'family_guids': ['F000002_2'], - 'reference_genome': 'GRCh37', - 'dataset_type': dataset_type - } - - def _add_update_check_dag_responses(self, **kwargs): - return self._add_check_dag_variable_responses(self._get_dag_variables(**kwargs), **kwargs) - - def assert_airflow_delete_families_calls(self): - self._assert_call_counts(13) - call_count_per_dag = 5 - offset = 0 - for dataset_type in ['MITO', 'SNV_INDEL']: - self.assert_airflow_calls(self._get_dag_variables(dataset_type), call_count_per_dag, offset) - offset += call_count_per_dag - - self._assert_update_variables_airflow_calls(self._get_dag_variables('SV'), offset) - - def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): - variables_update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' - super()._assert_update_check_airflow_calls(call_count, offset, variables_update_check_path) - - @responses.activate - def test_command(self): - super().test_command() - self.assert_airflow_delete_families_calls() From 006e5c183ca76310f2a3ff8ceaf3de451697298b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Sep 2025 10:51:32 -0400 Subject: [PATCH 23/36] fix dag trigger tests --- seqr/views/apis/data_manager_api_tests.py | 115 +++++----------------- 1 file changed, 26 insertions(+), 89 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 33a6b9bc60..d31d87ae25 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -10,7 +10,7 @@ from seqr.utils.communication_utils import _set_bulk_notification_stream from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ - get_loaded_projects, load_data, trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data + get_loaded_projects, load_data, trigger_delete_project, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AirflowTestCase, AirtableTest from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses @@ -1520,55 +1520,28 @@ def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) - self._test_trigger_search_data_update( - url, - 'DELETE_PROJECTS', - {'project': PROJECT_GUID, 'datasetType': 'SNV_INDEL'}, - { - 'projects_to_run': [PROJECT_GUID], - 'dataset_type': 'SNV_INDEL', - 'reference_genome': 'GRCh37', - } + Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') + response = self.client.post( + url, content_type='application/json', data=json.dumps({'project': PROJECT_GUID, 'datasetType': 'SNV_INDEL'}) ) + self._assert_expected_delete_project(response) @responses.activate def test_trigger_delete_family(self): url = reverse(trigger_delete_family) self.check_data_manager_login(url) - self._test_trigger_search_data_update( - url, - 'DELETE_FAMILIES', - {'family': 'F000012_12', 'datasetType': 'MITO'}, - { - 'projects_to_run': ['R0003_test'], - 'dataset_type': 'MITO', - 'reference_genome': 'GRCh37', - 'family_guids': ['F000012_12'], - } - ) - - @responses.activate - def test_trigger_update_search_reference_data(self): - url = reverse(trigger_update_search_reference_data) - self.check_data_manager_login(url) - - body = {'genomeVersion': '38', 'datasetType': 'SV'} - self._test_trigger_search_data_update(url, 'UPDATE_REFERENCE_DATASETS', body, { - 'projects_to_run': None, - 'dataset_type': 'SV', - 'reference_genome': 'GRCh38', - }) + Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') + response = self.client.post(url, content_type='application/json', data=json.dumps({'family': 'F000012_12', 'datasetType': 'MITO'})) + self._assert_expected_delete_family(response) - def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): - self._setup_trigger_search_data_update(dag_id=dag_id, dag_variables=dag_variables) - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_search_data_update(response, dag_id, dag_variables) + def _assert_expected_delete_project(self, response): + self._assert_expected_search_data_update(response) - def _setup_trigger_search_data_update(self, *args, **kwargs): - return + def _assert_expected_delete_family(self, response): + self._assert_expected_search_data_update(response) - def _assert_expected_search_data_update(self, response, dag_id, variables): + def _assert_expected_search_data_update(self, response): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) @@ -2011,21 +1984,6 @@ def _test_validate_dataset_type(self, url): self._set_file_not_found() - def _add_update_check_dag_responses(self, variables=None, **kwargs): - if variables: - return self._add_check_dag_variable_responses(variables) - - return super()._add_update_check_dag_responses(**kwargs) - - def _setup_trigger_search_data_update(self, *args, **kwargs): - Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') - - def _assert_expected_search_data_update(self, response, dag_id, variables): - if dag_id == 'DELETE_PROJECTS': - self._assert_expected_delete_project(response) - else: - super()._assert_expected_search_data_update(response, dag_id, variables) - def _assert_expected_delete_project(self, response): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { @@ -2045,6 +2003,12 @@ def _assert_expected_delete_project(self, response): 22: (0, 3, 0, 1), }) + def _assert_expected_delete_family(self, response): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': ['Clickhouse does not support deleting individual families from project. Manually delete MITO data for F000012_12 in project R0003_test'], + }) + def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', @@ -2057,43 +2021,16 @@ def _assert_expected_airtable_errors(self, url): }) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') -class HailBackendDataManagerAPITest(AnvilDataManagerAPITest): +class HailBackendDataManagerAPITest(LocalDataManagerAPITest): fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] CLICKHOUSE_HOSTNAME = '' + ES_HOSTNAME = '' - def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): - if self.DAG_NAME != 'LOADING_PIPELINE': - update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' - super()._assert_update_check_airflow_calls(call_count, offset, update_check_path) - - def set_up_one_dag(self, dag_id=None, **kwargs): - if dag_id: - self._dag_url = self._dag_url.replace(self.DAG_NAME, dag_id) - self.DAG_NAME = dag_id - super().set_up_one_dag(**kwargs) - - def _assert_expected_search_data_update(self, response, dag_id, variables): - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), {'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(variables)}']}) - - self.assert_airflow_calls(variables, 5) - - def _setup_trigger_search_data_update(self, *args, dag_id=None, dag_variables=None, **kwargs): - responses.calls.reset() - self.set_up_one_dag(dag_id, variables=dag_variables) - - def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): - super()._test_trigger_search_data_update(url, dag_id, body, dag_variables) - - self.set_dag_trigger_error_response() - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + def _assert_expected_es_status(self, response): self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'errors': [f'{dag_id} DAG is running and cannot be triggered again.'], 'warnings': None, - }) + self.assertEqual(response.json()['error'], 'Elasticsearch is disabled') - # DAG trigger not attempted when airflow is disabled - self.mock_airflow_url.__bool__.return_value = False - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 403) + def _assert_expected_delete_index_response(self, response): + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['error'], 'Deleting indices is disabled without the elasticsearch backend') From cf49c14d4c939344b0324463fa29c6bef574e6a2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Sep 2025 11:30:51 -0400 Subject: [PATCH 24/36] fix pop af values --- clickhouse_search/test_utils.py | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 89df6c0635..3bfe4c220a 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -44,7 +44,7 @@ 'topmed': {'af': 0.07842, 'ac': 20757, 'an': 264690, 'hom': 0, 'het': 20757}, 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.03445, 'ac': 927, 'an': 26912, 'hom': 48, 'hemi': 0, 'filter_af': 0.04028}, + 'gnomad_genomes': {'af': 0.03444932, 'ac': 927, 'an': 26912, 'hom': 48, 'hemi': 0, 'filter_af': 0.04027665}, }, 'predictions': { 'cadd': 4.668, @@ -125,9 +125,9 @@ 'screenRegionType': None, 'populations': { 'seqr': {'ac': 10, 'hom': 3}, - 'topmed': {'af': 0.24615, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, - 'exac': {'af': 0.295, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.4153}, - 'gnomad_exomes': {'af': 0.289, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.41165}, + 'topmed': {'af': 0.246152, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, + 'exac': {'af': 0.29499999, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.41530353}, + 'gnomad_exomes': {'af': 0.28899795, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.4116475}, 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0}, }, 'predictions': { @@ -204,10 +204,10 @@ 'screenRegionType': None, 'populations': { 'seqr': {'ac': 7, 'hom': 0}, - 'topmed': {'af': 0.36268, 'ac': 95998, 'an': 264690, 'hom': 19369, 'het': 57260}, + 'topmed': {'af': 0.362681, 'ac': 95998, 'an': 264690, 'hom': 19369, 'het': 57260}, 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.38041, 'ac': 57757, 'an': 151828, 'hom': 12204, 'hemi': 0, 'filter_af': 0.47978}, + 'gnomad_genomes': {'af': 0.38041073, 'ac': 57757, 'an': 151828, 'hom': 12204, 'hemi': 0, 'filter_af': 0.47977865}, }, 'predictions': { 'cadd': 2.754, @@ -277,7 +277,7 @@ 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00027, 'ac': 39, 'an': 147062, 'hom': 0, 'hemi': 0, 'filter_af': 0.0015}, + 'gnomad_genomes': {'af': 0.00026519, 'ac': 39, 'an': 147062, 'hom': 0, 'hemi': 0, 'filter_af': 0.00150301}, }, 'predictions': { 'cadd': 29.9, @@ -342,7 +342,7 @@ 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00012, 'ac': 2, 'an': 16090, 'hom': 0, 'hemi': 0, 'filter_af': 0.00234}, + 'gnomad_genomes': {'af': 0.0001243, 'ac': 2, 'an': 16090, 'hom': 0, 'hemi': 0, 'filter_af': 0.00234}, }, 'predictions': { 'cadd': 4.653, @@ -394,10 +394,10 @@ }, 'populations': { 'seqr': {'ac': 3, 'hom': 1}, - 'topmed': {'af': 0.52132, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, - 'exac': {'af': 0.63, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.81988}, - 'gnomad_exomes': {'af': 0.63542, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.82261}, - 'gnomad_genomes': {'af': 0.61365, 'ac': 14649, 'an': 23872, 'hom': 4584, 'hemi': 0, 'filter_af': 0.82844}, + 'topmed': {'af': 0.52131897, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, + 'exac': {'af': 0.63, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.81987739}, + 'gnomad_exomes': {'af': 0.63542193, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.82261163}, + 'gnomad_genomes': {'af': 0.61364776, 'ac': 14649, 'an': 23872, 'hom': 4584, 'hemi': 0, 'filter_af': 0.82843894}, }, 'predictions': { 'cadd': 13.02, 'eigen': 3.951, 'primate_ai': 0.49064, @@ -450,7 +450,7 @@ 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 56419}, 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 56419, 'max_hl': 0.0}, 'helix': {'af': 0.0, 'ac': 0, 'an': 195983}, - 'helix_heteroplasmy': {'af': 2e-05, 'ac': 3, 'an': 195983, 'max_hl': 0.20635}, + 'helix_heteroplasmy': {'af': 0.00001531, 'ac': 3, 'an': 195983, 'max_hl': 0.20635}, }, 'predictions': { 'apogee': None, @@ -547,10 +547,10 @@ 'populations': { 'seqr': {'ac': 1}, 'seqr_heteroplasmy': {'ac': 0}, - 'gnomad_mito': {'af': 0.05535, 'ac': 3118, 'an': 56336}, - 'gnomad_mito_heteroplasmy': {'af': 5e-05, 'ac': 3, 'an': 56336, 'max_hl': 1.0}, - 'helix': {'af': 0.04885, 'ac': 9573, 'an': 195983}, - 'helix_heteroplasmy': {'af': 9e-05, 'ac': 18, 'an': 195983, 'max_hl': 0.96269}, + 'gnomad_mito': {'af': 0.05534649, 'ac': 3118, 'an': 56336}, + 'gnomad_mito_heteroplasmy': {'af': 0.00005325, 'ac': 3, 'an': 56336, 'max_hl': 1.0}, + 'helix': {'af': 0.04884607, 'ac': 9573, 'an': 195983}, + 'helix_heteroplasmy': {'af': 0.00009184, 'ac': 18, 'an': 195983, 'max_hl': 0.96269}, }, 'predictions': { 'apogee': None, @@ -652,7 +652,7 @@ }, 'populations': { 'sv_callset': {'ac': 2, 'hom': 0}, - 'gnomad_svs': {'af': 0.00542, 'id': 'gnomAD-SV_v3_INS_1_299', 'hom': 10359, 'het': 35634}, + 'gnomad_svs': {'af': 0.005423, 'id': 'gnomAD-SV_v3_INS_1_299', 'hom': 10359, 'het': 35634}, }, 'predictions': {'strvctvre': None}, 'cpxIntervals': None, @@ -776,7 +776,7 @@ 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], } }, - 'populations': {'sv_callset': {'af': 0.07649, 'ac': 1763, 'an': 23048, 'hom': 0, 'het': 0}}, + 'populations': {'sv_callset': {'af': 0.07649254, 'ac': 1763, 'an': 23048, 'hom': 0, 'het': 0}}, 'predictions': {'strvctvre': 0.181}, 'numExon': 0, 'svType': 'DUP', @@ -812,7 +812,7 @@ 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], } }, - 'populations': {'sv_callset': {'af': 0.01232, 'ac': 284, 'an': 23047, 'hom': 0, 'het': 0}}, + 'populations': {'sv_callset': {'af': 0.01232211, 'ac': 284, 'an': 23047, 'hom': 0, 'het': 0}}, 'predictions': {'strvctvre': 0.548}, 'numExon': 8, 'svType': 'DUP', @@ -852,7 +852,7 @@ 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], }, }, - 'populations': {'sv_callset': {'af': 0.00152, 'ac': 35, 'an': 23048, 'hom': 0, 'het': 0}}, + 'populations': {'sv_callset': {'af': 0.00151857, 'ac': 35, 'an': 23048, 'hom': 0, 'het': 0}}, 'predictions': {'strvctvre': 0.786}, 'numExon': 3, 'svType': 'DUP', @@ -890,7 +890,7 @@ 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], } }, - 'populations': {'sv_callset': {'af': 0.00499, 'ac': 115, 'an': 23048, 'hom': 0, 'het': 0}}, + 'populations': {'sv_callset': {'af': 0.00498959, 'ac': 115, 'an': 23048, 'hom': 0, 'het': 0}}, 'predictions': {'strvctvre': 0.71}, 'numExon': 7, 'svType': 'DEL', From e4b9011395b5bc769f2ef53f28ba589143280786 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Sep 2025 12:02:55 -0400 Subject: [PATCH 25/36] properly handle gcnv dataset type inn delete triggger --- clickhouse_search/search.py | 15 +++++++++++-- .../commands/deactivate_project_search.py | 4 +--- .../transfer_families_to_different_project.py | 6 +++--- ...fer_families_to_different_project_tests.py | 2 +- seqr/views/apis/data_manager_api.py | 21 ++++++++++++------- seqr/views/apis/data_manager_api_tests.py | 4 ++-- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index e4e55c23e3..cdefcf4f57 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -622,7 +622,8 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers return lookup -def delete_clickhouse_project(project, dataset_type=None, **kwargs): +def delete_clickhouse_project(project, dataset_type, sample_type=None): + dataset_type = _clickhouse_dataset_type(dataset_type, sample_type) table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) @@ -635,5 +636,15 @@ def delete_clickhouse_project(project, dataset_type=None, **kwargs): return f'Deleted all {dataset_type} search data for project {project.name}' -def delete_clickhouse_family(project, family_guid, dataset_type=None, **kwargs): +def delete_clickhouse_family(project, family_guid, dataset_type, sample_type=None): + dataset_type = _clickhouse_dataset_type(dataset_type, sample_type) return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}' + + +def _clickhouse_dataset_type(dataset_type, sample_type): + if dataset_type == Sample.DATASET_TYPE_SV_CALLS: + if not sample_type: + raise ValueError('sample_type is required when dataset_type is SV') + if sample_type == Sample.SAMPLE_TYPE_WES: + return 'GCNV' + return dataset_type \ No newline at end of file diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index b0885bc5df..4c1f0c6cff 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -31,7 +31,5 @@ def handle(self, *args, **options): @staticmethod def _delete_clickhouse_project(project, dataset_type, sample_type): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: - dataset_type = 'GCNV' - info = delete_clickhouse_project(project, dataset_type) + info = delete_clickhouse_project(project, dataset_type, sample_type) logger.info(info) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index e4921bb6e1..7744cb905d 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -13,7 +13,7 @@ def _disable_search(families, from_project): updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() - updated_family_dataset_types = list(search_samples.values_list('dataset_type', 'individual__family__guid').distinct()) + updated_family_dataset_types = list(search_samples.values_list('individual__family__guid', 'dataset_type', 'sample_type').distinct()) family_summary = ", ".join(sorted(updated_families)) num_updated = search_samples.update(is_active=False) logger.info( @@ -23,8 +23,8 @@ def _disable_search(families, from_project): def _disable_search_clickhouse(families, from_project): updated_family_dataset_types = _disable_search(families, from_project) - for dataset_type, family_guid in updated_family_dataset_types: - logger.info(delete_clickhouse_family(from_project, family_guid=family_guid, dataset_type=dataset_type)) + for update in updated_family_dataset_types: + logger.info(delete_clickhouse_family(from_project, *update)) class Command(BaseCommand): diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 07f9062a96..5e18dbf721 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -62,7 +62,7 @@ class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTe ('Disabled search for 7 samples in the following 1 families: 2', None), ('Clickhouse does not support deleting individual families from project. Manually delete MITO data for F000002_2 in project R0001_1kg', None), ('Clickhouse does not support deleting individual families from project. Manually delete SNV_INDEL data for F000002_2 in project R0001_1kg', None), - ('Clickhouse does not support deleting individual families from project. Manually delete SV data for F000002_2 in project R0001_1kg', None), + ('Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg', None), ] diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 4c7af6a934..82583479fa 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -428,9 +428,7 @@ def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') project = Project.objects.get(guid=project_guid) - return _trigger_data_update( - project, delete_clickhouse_project, **request_json, - ) + return _trigger_data_update(delete_clickhouse_project, request_json, project) @data_manager_required @@ -438,17 +436,24 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(project, delete_clickhouse_family, family_guid=family_guid, **request_json) + return _trigger_data_update(delete_clickhouse_family, request_json, project, family_guid) def _raise_backend_not_implemented(*args, **kwargs): raise ErrorsWarningsException(['This functionality is not available in the current search backend']) -def _trigger_data_update(project, clickhouse_func, **kwargs): - kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} - info = backend_specific_call(_raise_backend_not_implemented, _raise_backend_not_implemented, clickhouse_func)(project, **kwargs) - return create_json_response({'info': [info]}) +def _trigger_data_update(clickhouse_func, request_json, project, *args): + dataset_type = request_json.get('datasetType') + sample_types = Sample.objects.filter( + individual__family__project=project, dataset_type=dataset_type, is_active=True, + ).values_list('sample_type', flat=True).distinct() if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] + info = [] + for sample_type in sample_types: + info.append(backend_specific_call( + _raise_backend_not_implemented, _raise_backend_not_implemented, clickhouse_func, + )(project, *args, dataset_type=dataset_type, sample_type=sample_type)) + return create_json_response({'info': info}) # Hop-by-hop HTTP response headers shouldn't be forwarded. diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index d31d87ae25..b692d37347 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1532,7 +1532,7 @@ def test_trigger_delete_family(self): self.check_data_manager_login(url) Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') - response = self.client.post(url, content_type='application/json', data=json.dumps({'family': 'F000012_12', 'datasetType': 'MITO'})) + response = self.client.post(url, content_type='application/json', data=json.dumps({'family': 'F000002_2', 'datasetType': 'SV'})) self._assert_expected_delete_family(response) def _assert_expected_delete_project(self, response): @@ -2006,7 +2006,7 @@ def _assert_expected_delete_project(self, response): def _assert_expected_delete_family(self, response): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { - 'info': ['Clickhouse does not support deleting individual families from project. Manually delete MITO data for F000012_12 in project R0003_test'], + 'info': ['Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg'], }) def _assert_expected_airtable_errors(self, url): From 974943cf410539fcfc518b8b8c895e7d41193756 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Sep 2025 16:49:18 -0400 Subject: [PATCH 26/36] get all keys for snv indel saved variants --- clickhouse_search/search.py | 13 +++++++++ seqr/views/apis/summary_data_api.py | 42 ++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index cdefcf4f57..5a938195d0 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -612,6 +612,19 @@ def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): ).values_list('key', flat=True)) +def get_all_clickhouse_keys_for_gene(gene_model, genome_version): + entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] + # TODO helper function + from seqr.utils.xpos_utils import get_xpos + chrom = getattr(gene_model, f'chrom_grch{genome_version}') + xpos_start = get_xpos(chrom, getattr(gene_model, f'start_grch{genome_version}')) + xpos_end = get_xpos(chrom, getattr(gene_model, f'end_grch{genome_version}')) + return list(entry_cls.objects.filter( + geneId_ids__bitmap_has_any=[gene_model.id], + xpos__range=(xpos_start, xpos_end), + ).values_list('key', flat=True).distinct()) + + def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False): key_lookup_class = KEY_LOOKUP_CLASS_MAP[genome_version][dataset_type] lookup = {} diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 8e747c3a02..13a390980b 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -8,12 +8,12 @@ from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf import json -from clickhouse_search.search import get_clickhouse_keys_for_gene +from clickhouse_search.search import get_clickhouse_keys_for_gene, get_all_clickhouse_keys_for_gene from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission -from reference_data.models import HumanPhenotypeOntology -from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy +from reference_data.models import HumanPhenotypeOntology, GeneInfo, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 +from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy, Sample from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream @@ -100,7 +100,8 @@ def success_story(request, success_story_types): @login_and_policies_required def saved_variants_page(request, tag): gene = request.GET.get('gene') - if tag == 'ALL': + is_all_tags = tag == 'ALL' + if is_all_tags: saved_variant_models = SavedVariant.objects.exclude(varianttag=None) else: tags = tag.split(';') @@ -118,7 +119,7 @@ def saved_variants_page(request, tag): lambda *args: Q(), lambda *args: Q(), _saved_variant_with_clickhouse_gene_q, - )(saved_variant_models, gene) + )(saved_variant_models, gene, is_all_tags) saved_variant_models = saved_variant_models.filter(gene_filter) elif saved_variant_models.count() > MAX_SAVED_VARIANTS: return create_json_response({'error': 'Select a gene to filter variants'}, status=400) @@ -131,20 +132,35 @@ def saved_variants_page(request, tag): return create_json_response(response_json) -def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id): - search_type_keys = saved_variant_models.filter(key__isnull=False).values( +def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id, is_all_tags): + saved_variant_models = saved_variant_models.filter(key__isnull=False) + key_qs = [] + + if is_all_tags: + gene_model = GeneInfo.objects.get(gene_id=gene_id) + for genome_version in [GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38]: + gene_keys = get_all_clickhouse_keys_for_gene(gene_model, genome_version) + if gene_keys: + key_qs.append( + Q(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, family__project__genome_version=genome_version, key__in=gene_keys) + ) + saved_variant_models = saved_variant_models.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + + search_type_keys = saved_variant_models.values( 'dataset_type', genome_version=F('family__project__genome_version'), ).annotate(keys=ArrayAgg('key', distinct=True)) - has_key_q = None for agg in search_type_keys: gene_keys = get_clickhouse_keys_for_gene(gene_id, **agg) if gene_keys: q = Q(dataset_type=agg['dataset_type'], family__project__genome_version=agg['genome_version'], key__in=gene_keys) - if has_key_q: - has_key_q |= q - else: - has_key_q = q - return has_key_q or Q() + key_qs.append(q) + + if not key_qs: + return Q() + has_key_q = key_qs[0] + for q in key_qs[1:]: + has_key_q |= q + return has_key_q @login_and_policies_required From 6ee058690eb02b146ee5ba6df24ff42fb0759f02 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Sep 2025 17:00:59 -0400 Subject: [PATCH 27/36] use model manager filter --- clickhouse_search/managers.py | 9 ++++++--- clickhouse_search/search.py | 12 +++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1dc1dee55a..1f8528ed7c 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1043,7 +1043,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, require_gene_filter=False, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, @@ -1076,9 +1076,10 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N if gene_intervals: if hasattr(self.model, 'is_annotated_in_any_gene') and not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) - if (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS: + should_filter_interval = (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS + if should_filter_interval: intervals = list((gene_intervals or {}).values()) + (intervals or []) - else: + if require_gene_filter or (not should_filter_interval): locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) if intervals: @@ -1087,6 +1088,8 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N interval_q |= self._interval_query(*interval) if locus_q is None: locus_q = interval_q + elif require_gene_filter: + locus_q &= interval_q else: locus_q |= interval_q diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 5a938195d0..79d0f301e2 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -614,15 +614,9 @@ def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): def get_all_clickhouse_keys_for_gene(gene_model, genome_version): entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - # TODO helper function - from seqr.utils.xpos_utils import get_xpos - chrom = getattr(gene_model, f'chrom_grch{genome_version}') - xpos_start = get_xpos(chrom, getattr(gene_model, f'start_grch{genome_version}')) - xpos_end = get_xpos(chrom, getattr(gene_model, f'end_grch{genome_version}')) - return list(entry_cls.objects.filter( - geneId_ids__bitmap_has_any=[gene_model.id], - xpos__range=(xpos_start, xpos_end), - ).values_list('key', flat=True).distinct()) + return list(entry_cls.objects.filter_locus(require_gene_filter=True, gene_intervals={ + gene_model.id: [getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']] + }).values_list('key', flat=True).distinct()) def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False): From 0c0ac09235be72d48692302b916b3372386b334a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Sep 2025 17:24:40 -0400 Subject: [PATCH 28/36] fix fixture data --- seqr/fixtures/reference_data.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/seqr/fixtures/reference_data.json b/seqr/fixtures/reference_data.json index 2feea0b571..f0bc4405e2 100644 --- a/seqr/fixtures/reference_data.json +++ b/seqr/fixtures/reference_data.json @@ -946,13 +946,13 @@ "gene_id": "ENSG00000135953", "gene_symbol": "RP11", "chrom_grch37": "1", - "start_grch37": 694412, - "end_grch37": 700305, + "start_grch37": 228367220, + "end_grch37": 268367621, "strand_grch37": "-", "coding_region_size_grch37": 0, - "chrom_grch38": null, - "start_grch38": null, - "end_grch38": null, + "chrom_grch38": "1", + "start_grch38": 694412, + "end_grch38": 700305, "strand_grch38": null, "coding_region_size_grch38": 0, "gencode_gene_type": "lincRNA", From 8696fac692cc05cbfc1edff6d85d6ec42a3359ab Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Sep 2025 17:26:39 -0400 Subject: [PATCH 29/36] clean up --- seqr/views/apis/summary_data_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 13a390980b..7a9e9fdf70 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -133,9 +133,7 @@ def saved_variants_page(request, tag): def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id, is_all_tags): - saved_variant_models = saved_variant_models.filter(key__isnull=False) key_qs = [] - if is_all_tags: gene_model = GeneInfo.objects.get(gene_id=gene_id) for genome_version in [GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38]: @@ -146,7 +144,7 @@ def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id, is_all_ ) saved_variant_models = saved_variant_models.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - search_type_keys = saved_variant_models.values( + search_type_keys = saved_variant_models.filter(key__isnull=False).values( 'dataset_type', genome_version=F('family__project__genome_version'), ).annotate(keys=ArrayAgg('key', distinct=True)) for agg in search_type_keys: From 488f28be384b0c4fb8d9ec9e05d8a8ad23c113ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 10:40:29 -0400 Subject: [PATCH 30/36] only search for x chromosome comp hets in families without males --- clickhouse_search/search.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index cdefcf4f57..58cf3f46b1 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -13,7 +13,7 @@ from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization +from seqr.models import Sample, PhenotypePrioritization, Individual from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE @@ -45,17 +45,20 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] family_guid = sample_data['family_guids'][0] - skip_individual_guid = len(sample_data['project_guids']) > 1 + is_multi_project = len(sample_data['project_guids']) > 1 dataset_results = [] if inheritance_mode != COMPOUND_HET: - result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) + result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=is_multi_project, **search) dataset_results += _evaluate_results(result_q) if has_comp_het: - result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) + comp_het_sample_data = sample_data + if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(**search): + comp_het_sample_data = _no_affected_male_families(sample_data, user) + result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, skip_individual_guid=is_multi_project, **search) dataset_results += _evaluate_results(result_q, is_comp_het=True) - if skip_individual_guid: + if is_multi_project: _add_individual_guids(dataset_results, sample_data) results += dataset_results @@ -372,6 +375,25 @@ def _get_sample_data(samples): return samples_by_dataset_type +def _no_affected_male_families(sample_data, user): + valid_families = { + s['family_guid'] for s in sample_data['samples'] + if s['affected'] == Individual.AFFECTED_STATUS_AFFECTED and s['sex'] != Individual.SEX_MALE + } + logger.info(f'Loading X-chromosome compound het data for {len(valid_families)} families', user) + return { + **sample_data, + 'family_guids': list(valid_families), + 'samples': [s for s in sample_data['samples'] if s['family_guid'] in valid_families], + } + + +def _is_x_chrom_only(parsed_locus=None, **kwargs): + parsed_locus = parsed_locus or {} + all_intervals = list((parsed_locus.get('gene_intervals') or {}).values()) + (parsed_locus.get('intervals') or []) + return bool(all_intervals) and all('X' in interval[0] for interval in all_intervals) + + OMIM_SORT = 'in_omim' GENE_SORTS = { 'constraint': lambda gene_ids, _: { From 8db8f613888f02e5f03d61452de12790fe78c419 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 11:04:46 -0400 Subject: [PATCH 31/36] fix liftover id filtering --- clickhouse_search/search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index cdefcf4f57..fc0b3172c0 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -546,13 +546,13 @@ def _add_liftover_genotypes(variant, data_type, variant_id): lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) if not lifted_entry_cls: return - lifted_id = (variant['liftedOverChrom'], str(variant['liftedOverPos']), *variant_id[2:]) + lifted_id = (variant['liftedOverChrom'], variant['liftedOverPos'], *variant_id[2:]) keys = KEY_LOOKUP_CLASS_MAP[variant['liftedOverGenomeVersion']][data_type].objects.filter( - variant_id='-'.join(lifted_id), + variant_id='-'.join([str(o) for o in lifted_id]), ).values_list('key', flat=True) if not keys: return - lifted_entries = lifted_entry_cls.objects.filter_locus(variant_ids=[variant_id]).filter(key=keys[0]) + lifted_entries = lifted_entry_cls.objects.filter_locus(variant_ids=[lifted_id]).filter(key=keys[0]) lifted_entry_data = lifted_entries.values('key').annotate( familyGenotypes=GroupArrayArray(lifted_entry_cls.objects.genotype_expression()) ) From 7c7bf2d00444a8daecc4f3358b4e8321cb5bbf00 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 11:17:16 -0400 Subject: [PATCH 32/36] restructure to allow lifted lookup --- clickhouse_search/search.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fc0b3172c0..7f9f5cc082 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -527,18 +527,26 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): else: results = results.filter_variant_ids(variant_ids=[variant_id]) - return results.result_values().first() + variant = results.result_values().first() + if variant: + variant = format_clickhouse_results([variant], genome_version)[0] + return variant def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) + if variant: + _add_liftover_genotypes(variant, data_type, variant_id) + else: + lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) + if ENTRY_CLASS_MAP[lifted_genome_version].get(data_type): + lifted_id = _liftover_variant_id(variant_id, lifted_genome_version) + variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type, samples) + if not variant: raise ObjectDoesNotExist('Variant not present in seqr') - variant = format_clickhouse_results([variant], genome_version)[0] - _add_liftover_genotypes(variant, data_type, variant_id) - return variant @@ -572,7 +580,7 @@ def get_clickhouse_variant_by_id(variant_id, samples, genome_version, dataset_ty for data_type in data_types: variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) if variant: - return format_clickhouse_results([variant], genome_version)[0] + return variant return None From 6406e866fa64911790fd78c91b2897c2b8db9120 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 11:46:23 -0400 Subject: [PATCH 33/36] abstract run liftover --- seqr/utils/search/elasticsearch/es_search.py | 28 +++------------- .../search/elasticsearch/es_utils_tests.py | 11 ++++--- seqr/utils/search/utils.py | 32 +++++++++++++++++++ 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index caeefb1895..47adfd192a 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -4,7 +4,6 @@ from elasticsearch_dsl import Search, Q, MultiSearch import hashlib import json -from pyliftover.liftover import LiftOver from sys import maxsize from itertools import combinations @@ -948,16 +947,11 @@ def _add_liftover(result, hit): lifted_over_chrom = grch37_locus['contig'] lifted_over_pos = grch37_locus['position'] else: - # TODO once all projects are lifted in pipeline, remove this code (https://github.com/broadinstitute/seqr/issues/1010) - liftover_grch38_to_grch37 = _liftover_grch38_to_grch37() - if liftover_grch38_to_grch37: - grch37_coord = liftover_grch38_to_grch37.convert_coordinate( - 'chr{}'.format(hit['contig'].lstrip('chr')), int(hit['start']) - ) - if grch37_coord and grch37_coord[0]: - lifted_over_genome_version = GENOME_VERSION_GRCh37 - lifted_over_chrom = grch37_coord[0][0].lstrip('chr') - lifted_over_pos = grch37_coord[0][1] + from seqr.utils.search.utils import run_liftover + grch37_coords = run_liftover(GENOME_VERSION_GRCh37, hit['contig'], hit['start']) + if grch37_coords: + lifted_over_genome_version = GENOME_VERSION_GRCh37 + lifted_over_chrom, lifted_over_pos = grch37_coords result.update({ 'liftedOverGenomeVersion': lifted_over_genome_version, 'liftedOverChrom': lifted_over_chrom, @@ -1329,18 +1323,6 @@ def _get_quality_filters_by_family(self, quality_filter, affected_only): return quality_filters_by_family -# TODO move liftover to hail pipeline once upgraded to 0.2 (https://github.com/broadinstitute/seqr/issues/1010) -LIFTOVER_GRCH38_TO_GRCH37 = None -def _liftover_grch38_to_grch37(): - global LIFTOVER_GRCH38_TO_GRCH37 - if not LIFTOVER_GRCH38_TO_GRCH37: - try: - LIFTOVER_GRCH38_TO_GRCH37 = LiftOver('hg38', 'hg19') - except Exception as e: - logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) - return LIFTOVER_GRCH38_TO_GRCH37 - - def _get_family_affected_status(samples_by_id, inheritance_filter): individual_affected_status = inheritance_filter.get('affected') or {} affected_status = {} diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index a9196c5bb1..787baed991 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -14,8 +14,9 @@ from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import get_single_variant, query_variants, \ get_variant_query_gene_counts, get_variants_for_variant_ids, InvalidSearchException -from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status, _liftover_grch38_to_grch37 +from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException +from seqr.utils.search.utils import _get_liftover from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_SV_WGS_VARIANT,\ PARSED_MITO_VARIANT, TRANSCRIPT_2, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT @@ -3067,8 +3068,8 @@ def test_skip_genotype_filter(self): ], start_index=0, size=2, index=INDEX_NAME) ]) - @mock.patch('seqr.utils.search.elasticsearch.es_search.LIFTOVER_GRCH38_TO_GRCH37', None) - @mock.patch('seqr.utils.search.elasticsearch.es_search.LiftOver') + @mock.patch('seqr.utils.search.utils.LIFTOVERS', {'37': None}) + @mock.patch('seqr.utils.search.utils.LiftOver') @urllib3_responses.activate def test_get_lifted_grch38_variants(self, mock_liftover): setup_responses() @@ -3091,7 +3092,7 @@ def test_get_lifted_grch38_variants(self, mock_liftover): variants, _ = query_variants(results_model, num_results=2) self.assertEqual(len(variants), 2) self.assertListEqual(variants, [PARSED_HG38_VARIANT, expected_no_lift_grch38_variant]) - self.assertIsNone(_liftover_grch38_to_grch37()) + self.assertIsNone(_get_liftover('37')) mock_liftover.assert_called_with('hg38', 'hg19') _set_cache('search_results__{}__xpos'.format(results_model.guid), None) @@ -3100,7 +3101,7 @@ def test_get_lifted_grch38_variants(self, mock_liftover): variants, _ = query_variants(results_model, num_results=2) self.assertEqual(len(variants), 2) self.assertListEqual(variants, [PARSED_HG38_VARIANT, PARSED_HG38_VARIANT]) - self.assertIsNotNone(_liftover_grch38_to_grch37()) + self.assertIsNotNone(_get_liftover('37')) mock_liftover.assert_called_with('hg38', 'hg19') @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_NAME_LENGTH', 30) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 355b098eef..55cb3772c9 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -2,11 +2,13 @@ from copy import deepcopy from datetime import timedelta from django.db.models import Count +from pyliftover.liftover import LiftOver from clickhouse_search.search import clickhouse_backend_enabled, get_clickhouse_variants, format_clickhouse_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project +from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ @@ -19,6 +21,7 @@ from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS +logger = SeqrLogger(__name__) class InvalidSearchException(Exception): pass @@ -657,3 +660,32 @@ def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): end = min(end + offset_pos, MAX_POS) return [chrom, start, end] + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} +PYLIFTOVER_BUILD_LOOKUP = { + GENOME_VERSION_GRCh38: ('hg19', 'hg38'), + GENOME_VERSION_GRCh37: ('hg38', 'hg19'), +} +def _get_liftover(genome_version): + global LIFTOVERS + if not LIFTOVERS[genome_version]: + try: + LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) + except Exception as e: + logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) + return LIFTOVERS[genome_version] + + +def run_liftover(genome_version, chrom, pos): + liftover = _get_liftover(genome_version) + if not liftover: + return None + lifted_coord = liftover.convert_coordinate( + 'chr{}'.format(chrom.lstrip('chr')), int(pos) + ) + if lifted_coord and lifted_coord[0]: + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) + return None From b58845f4104492e0880348d9f517eaa3fefecb41 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 11:53:38 -0400 Subject: [PATCH 34/36] run liftover for missing lookup variant --- clickhouse_search/search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 7f9f5cc082..92842cec48 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -541,8 +541,11 @@ def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, else: lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) if ENTRY_CLASS_MAP[lifted_genome_version].get(data_type): - lifted_id = _liftover_variant_id(variant_id, lifted_genome_version) - variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type, samples) + from seqr.utils.search.utils import run_liftover + liftover_results = run_liftover(lifted_genome_version, variant_id[0], variant_id[1]) + if liftover_results: + lifted_id = (liftover_results[0], liftover_results[1], *variant_id[2:]) + variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type, samples) if not variant: raise ObjectDoesNotExist('Variant not present in seqr') From d16fe3212b55f3ccb86ead3e22a94564ef979de4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 12:11:45 -0400 Subject: [PATCH 35/36] test liftover --- clickhouse_search/search_tests.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 2d5a8b5f93..6ffc0aaaaa 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -659,7 +659,11 @@ def test_invalid_search(self): 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', ) - def test_variant_lookup(self): + @mock.patch('seqr.utils.search.utils.LiftOver') + def test_variant_lookup(self, mock_liftover): + mock_convert_coordinate = mock_liftover.return_value.convert_coordinate + mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] + variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A')) self._assert_expected_variants([variant], [VARIANT_LOOKUP_VARIANT]) @@ -668,12 +672,19 @@ def test_variant_lookup(self): self.assertEqual(str(cm.exception), 'Variant not present in seqr') variant = variant_lookup(self.user, ('7', 143270172, 'A', 'G'), genome_version='37') - self._assert_expected_variants([variant], [{ + grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() ], key=lambda x: x['sampleId'], reverse=True)}, - }]) + } + self._assert_expected_variants([variant], [grch37_lookup_variant]) + + # Lookup works if variant is only present on a different build + variant = variant_lookup(self.user, ('7', 143260172, 'A', 'G')) + self._assert_expected_variants([variant], [grch37_lookup_variant]) + mock_liftover.assert_called_with('hg38', 'hg19') + mock_convert_coordinate.assert_called_with('chr7', 143260172) variant = variant_lookup(self.user, ('M', 4429, 'G', 'A'), genome_version='38') self._assert_expected_variants([variant], [{ @@ -695,7 +706,7 @@ def test_variant_lookup(self): self._assert_expected_variants(variants, [GCNV_VARIANT3]) with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, 'suffix_140608_DEL') + variant_lookup(self.user, ('5', 143270172, 'A', 'G')) self.assertEqual(str(cm.exception), 'Variant not present in seqr') def test_get_single_variant(self): From d497defe3e7430f0f9033bc29709cf386629766b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 5 Sep 2025 12:17:43 -0400 Subject: [PATCH 36/36] remove unneeded global --- seqr/utils/search/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 55cb3772c9..93cee91617 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -670,7 +670,6 @@ def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): GENOME_VERSION_GRCh37: ('hg38', 'hg19'), } def _get_liftover(genome_version): - global LIFTOVERS if not LIFTOVERS[genome_version]: try: LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version])