diff --git a/CHANGELOG.md b/CHANGELOG.md index 643875cc04..60dc4166da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## dev +## 8/7/25 +* Deprecate support for lifting existing projects from GRCH37 to GRCh38 + ## 4/4/25 * Refactor manage commands to update reference data. May require changes to custom cron jobs. (REQUIRES DB MIGRATION) diff --git a/clickhouse_search/backend/fields.py b/clickhouse_search/backend/fields.py index 36c4d87570..a43519218c 100644 --- a/clickhouse_search/backend/fields.py +++ b/clickhouse_search/backend/fields.py @@ -50,7 +50,8 @@ def _from_db_value(self, value, expression, connection, format_item=None): if self.group_by_key: group_value = defaultdict(list) for item in value: - group_value[item[self.group_by_key]].append(item) + group_key = item[self.group_by_key] + group_value['null' if group_key is None else group_key].append(item) if self.flatten_groups: value = {k: v[0] if len(v) == 1 else v for k, v in group_value.items()} else: diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index ca796bf865..85e242665e 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -120,7 +120,7 @@ class DictGet(Func): class If(Func): function = 'if' - template = '%(function)s(%(condition)s, %(expressions)s)' + template = '%(function)s(%(condition)s%(expressions)s)' class MapLookup(Func): diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json new file mode 100644 index 0000000000..8ad858a65f --- /dev/null +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -0,0 +1,333 @@ +[{ + "model": "clickhouse_search.keylookupgrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T" + } +}, { + "model": "clickhouse_search.annotationsgrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "xpos": 1248367227, + "chrom": "1", + "pos": 248367227, + "ref": "TC", + "alt": "T", + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": "CA1501729", + "lifted_over_chrom": null, + "lifted_over_pos": null, + "hgmd": [null, null], + "predictions": [27.2, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [66593, 0.63, 104352, 0.81988, 0, 22269, 22162], + [137532, 0.63542, 216442, 0.82261, 0, 45869], + [14649, 0.61365, 23872, 0.82844, 0, 4584], + [65461, 0.52132, 125568, 33149, 16156] + ], + "sorted_transcript_consequences": [ + [1, [], "ENSG00000240361"], + [1, [], "ENSG00000135953"] + ] + } +}, { + "model": "clickhouse_search.transcriptsgrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "transcripts": [ + [null, "protein_coding", 1, "Gtg/Atg", ["intron_variant"], "ENSG00000240361", "ENST00000262738.3:c.3955G>A", "ENST00000505820.2:c.1586-17C>G", [null, []], "intron_variant", "ENST00000505820", 0], + [null, "protein_coding", 1, null, [], "ENSG00000135953", null, null, [null, []], null, "ENST00000371839", 1] + ] + } +}, { + "model": "clickhouse_search.clinvargrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "allele_id": 12345, + "conflicting_pathogenicities": [], + "gold_stars": null, + "submitters": [], + "conditions": [], + "assertions": [], + "pathogenicity": "Uncertain_significance" + } +}, { + "model": "clickhouse_search.entriesgrch37snvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 1, 99, 0.6530612111091614, 49], + ["HG00731", 2, 48, 1, 16] + ] + } +}, { + "model": "clickhouse_search.keylookupgrch37snvindel", + "pk": 101, + "fields": { + "key": 101, + "variant_id": "2-103343353-GAGA-G" + } +}, { + "model": "clickhouse_search.annotationsgrch37snvindel", + "pk": 101, + "fields": { + "key": 101, + "xpos": 2103343353, + "chrom": "2", + "pos": 103343353, + "ref": "GAGA", + "alt": "G", + "variant_id": "2-103343353-GAGA-G", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "hgmd": [null, null], + "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [4, 0.000044254640670260414, 90386, 0.00011118523252662271, 0, 0], + [13, 0.00004911409996566363, 264690, 13, 0] + ], + "sorted_transcript_consequences": [ + [1, ["inframe_deletion"], "ENSG00000135953"], + [1, ["frameshift_variant"], "ENSG00000228198"] + ] + } +}, { + "model": "clickhouse_search.entriesgrch37snvindel", + "pk": 101, + "fields": { + "key": 101, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2", + "sample_type": "WES", + "xpos": 2103343353, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["HG00731", 2, 99, 0, 67], + ["HG00732", 1, 96, 0, 42], + ["HG00733", 0, 96, 0, 42] + ] + } +}, { + "model": "clickhouse_search.annotationsgrch37snvindel", + "pk": 123, + "fields": { + "key": 123, + "xpos": 261413835, + "chrom": "2", + "pos": 61413835, + "ref": "AAAG", + "alt": "A", + "variant_id": "2-61413835-AAAG-A", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "hgmd": [null, null], + "predictions": [21.9, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0] + ], + "sorted_transcript_consequences": [] + } +}, { + "model": "clickhouse_search.keylookupsnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T" + } +}, { + "model": "clickhouse_search.annotationssnvindel", + "pk": 100, + "fields": { + "key": 100, + "xpos": 1248367227, + "chrom": "1", + "pos": 248367227, + "ref": "TC", + "alt": "T", + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": "CA1501729", + "lifted_over_chrom": "1", + "lifted_over_pos": null, + "hgmd": [null, null], + "screen_region_type": null, + "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [4, 0.000044254640670260414, 90386, 0.00011118523252662271, 0, 0], + [13, 0.00004911409996566363, 264690, 13, 0] + ], + "sorted_transcript_consequences": [], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0004_non_analyst_project", + "family_guid": "F000014_14", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["NA21234", 2, 99, 0, 49] + ] + } +}, { + "model": "clickhouse_search.keylookupsv", + "pk": 101, + "fields": { + "key": 101, + "variant_id": "suffix_19107_DEL" + } +}, { + "model": "clickhouse_search.annotationssv", + "pk": 101, + "fields": { + "key": 101, + "xpos": 1249045487, + "chrom": "1", + "end_chrom": null, + "pos": 249045487, + "end": 249045898, + "variant_id": "suffix_19107_DEL", + "lifted_over_chrom": null, + "lifted_over_pos": null, + "rg37_locus_end": [null, null], + "bothsides_support": false, + "algorithms": "manta", + "predictions": [0.374], + "populations": [ + [0, 0, 0, ""] + ], + "sorted_gene_consequences": [ + ["ENSG00000240361", "LOF"], + ["ENSG00000135953", "LOF"], + ["ENSG00000223972", "LOF"] + ], + "sv_type": "DEL", + "sv_type_detail": null, + "cpx_intervals": [], + "sv_source_detail": [null] + } +},{ + "model": "clickhouse_search.keylookupgcnv", + "pk": 111, + "fields": { + "key": 111, + "variant_id": "prefix_70191_DEL" + } +}, { + "model": "clickhouse_search.annotationsgcnv", + "pk": 111, + "fields": { + "key": 111, + "xpos": 1249045487, + "chrom": "1", + "pos": 249045487, + "end": 249045898, + "variant_id": "prefix_70191_DEL", + "lifted_over_chrom": "2", + "lifted_over_pos": null, + "rg37_locus_end": [null, null], + "num_exon": 2, + "predictions": [0.374], + "populations": [ + [2, 0.063, 32, 0, 0] + ], + "sorted_gene_consequences": [ + ["ENSG00000240361", "LOF"], + ["ENSG00000135953", "LOF"], + ["ENSG00000223972", "LOF"] + ], + "sv_type": "DUP" + } +}, { + "model": "clickhouse_search.annotationsgcnv", + "pk": 123, + "fields": { + "key": 123, + "xpos": 2061413835, + "chrom": "2", + "pos": 61413835, + "end": 61414175, + "variant_id": "batch_123_DUP", + "lifted_over_chrom": "2", + "lifted_over_pos": 61413835, + "rg37_locus_end": [2, 61414175], + "num_exon": 0, + "predictions": [21.9], + "populations": [ + [2, 0.063, 32, 0, 0] + ], + "sorted_gene_consequences": [ + ["ENSG00000240361", "LOF"] + ], + "sv_type": "DUP" + } +}, { + "model": "clickhouse_search.keylookupmito", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "M-14783-T-C" + } +}, { + "model": "clickhouse_search.annotationsmito", + "pk": 100, + "fields": { + "key": 100, + "xpos": 25000014783, + "pos": 14783, + "ref": "T", + "alt": "C", + "variant_id": "M-14783-T-C", + "rsid": "rs193302982", + "lifted_over_pos": 14783, + "common_low_heteroplasmy": true, + "mitomap_pathogenic": true, + "predictions": [null, true, null, null, null, null, 0.7514], + "populations": [ + [3118, 0.05535, 56336], + [3, 0.00005, 56336, 1], + [9573, 0.04885, 195983], + [18, 0.00009, 195983, 0.96269] + ], + "sorted_transcript_consequences": [ + [ + "L", "protein_coding", 1, "Tta/Cta", ["synonymous_variant"], "ENSG00000198727", "ENST00000361789.2:c.37T>C", + "ENSP00000354554.2:p.Leu13=", [null, []], "synonymous_variant", "ENST00000361789", 0 + ] + ] + } +} +] \ No newline at end of file diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 0590b84db5..c2f05942bc 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -191,6 +191,36 @@ "sorted_motif_feature_consequences": [], "sorted_regulatory_feature_consequences": [] } +}, { + "model": "clickhouse_search.annotationssnvindel", + "pk": 22, + "fields": { + "key": 22, + "xpos": 1009310123, + "chrom": "1", + "pos": 9310123, + "ref": "T", + "alt": "C", + "variant_id": "1-9310123-T-C", + "rsid": null, + "caid": null, + "lifted_over_chrom": "1", + "lifted_over_pos": 9310113, + "hgmd": [null, null], + "screen_region_type": null, + "predictions": [1.92299, 2.24799, null, null, null, null, null, null, null, null, null, 0.01, 4, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [4, 0.00004, 90386, 0.0001, 0, 0], + [13, 0.00004, 264690, 0.0002, 0, 13], + [0, 0, 0, 0, 0] + ], + "sorted_transcript_consequences": [ + [null, 1, ["missense_variant"], false, null, "ENSG00000171621"] + ], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } }, { "model": "clickhouse_search.clinvarsnvindel", "pk": 1, @@ -495,6 +525,24 @@ ["HG00731", 1, 58, 0.17241, 29] ] } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 22, + "fields": { + "key": 22, + "project_guid": "R0004_non_analyst_project", + "family_guid": "F000014_14", + "sample_type": "WGS", + "xpos": 1009310123, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["NA21234", 2, 0, 0, 45], + ["NA21987", 1, 58, 0.17241, 29], + ["NA21654", 0, 0, 0, 24] + ] + } }, { "model": "clickhouse_search.annotationsgrch37snvindel", "pk": 10, diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json index aa23fa5449..e7a3352963 100644 --- a/clickhouse_search/fixtures/clickhouse_transcripts.json +++ b/clickhouse_search/fixtures/clickhouse_transcripts.json @@ -159,5 +159,20 @@ ] ] } +}, { + "model": "clickhouse_search.transcriptssnvindel", + "pk": 22, + "fields": { + "key": 22, + "transcripts": [ + [ + [null], "T/I", "protein_coding", 1, "aCc/aTc", ["missense_variant"], [1, 12], "ENSG00000171621", + "ENST00000257261.10:c.131C>T", "ENSP00000257261.6:p.Thr44Ile", [null, null], [null, []], "missense_variant", + null, null, "NM_001281501.1", [false], "ENST00000257261", 0, [null, null, null, [ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null + ], null] + ] + ] + } } ] \ No newline at end of file diff --git a/clickhouse_search/management/__init__.py b/clickhouse_search/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/clickhouse_search/management/commands/__init__.py b/clickhouse_search/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py new file mode 100644 index 0000000000..bc3ea807d2 --- /dev/null +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -0,0 +1,2366 @@ +from collections import defaultdict + +from django.contrib.postgres.aggregates import ArrayAgg +from django.core.management.base import BaseCommand +from django.db.models import F +import logging +import re + +from clickhouse_search.search import get_clickhouse_key_lookup +from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 +from seqr.models import SavedVariant, Sample +from seqr.utils.file_utils import file_iter +from seqr.utils.search.utils import parse_variant_id + +logger = logging.getLogger(__name__) + +GCNV_CALLSET_PATH = 'gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz' + +SV_ID_UPDATE_MAP = { + 'WGS': { + 'CMG.phase1_CMG_DEL_chr10_2038': 'phase2_DEL_chr10_4611', + 'CMG.phase1_CMG_DEL_chr11_2963': 'phase2_DEL_chr11_5789', + 'CMG.phase1_CMG_DEL_chr13_153': 'phase2_DEL_chr13_378', + 'cohort_2911.chr1.final_cleanup_BND_chr1_1167': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1376', + 'cohort_2911.chr1.final_cleanup_BND_chr1_1017': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1208', + 'cohort_2911.chr1.final_cleanup_BND_chr1_2837': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_3326', + 'cohort_2911.chr1.final_cleanup_DEL_chr1_12237': 'phase2_DEL_chr1_9347', + 'cohort_2911.chr1.final_cleanup_DEL_chr1_2953': 'phase2_DEL_chr1_2503', + 'cohort_2911.chrX.final_cleanup_CPX_chrX_20': 'cohort_2911.chrX.final_cleanup_CPX_chrX_19', + 'phase2_DUP_chr1_1164': 'phase4_all_batches.chr1.final_cleanup_DUP_chr1_1666', + 'phase2_CPX_chr1_73': 'phase4_all_batches.chr1.final_cleanup_CPX_chr1_85', + 'cohort_2911.chr1.final_cleanup_DEL_chr1_3844': 'phase2_DEL_chr1_3148', + 'cohort_2911.chr1.final_cleanup_DEL_chr1_4181': 'phase2_DEL_chr1_3389', + 'cohort_2911.chr1.final_cleanup_DEL_chr1_6414': 'phase2_DEL_chr1_5030', + 'cohort_2911.chr10.final_cleanup_DEL_chr10_6275': 'phase2_DEL_chr10_4600', + 'cohort_2911.chr10.final_cleanup_DEL_chr10_7793': 'phase2_DEL_chr10_5695', + 'cohort_2911.chr10.final_cleanup_DUP_chr10_1491': 'phase2_DUP_chr10_1234', + 'cohort_2911.chr10.final_cleanup_DUP_chr10_3659': 'phase2_DUP_chr10_3132', + 'cohort_2911.chr11.final_cleanup_DUP_chr11_3003': 'phase2_DUP_chr11_2757', + 'cohort_2911.chr12.final_cleanup_DEL_chr12_244': 'phase2_DEL_chr12_207', + 'cohort_2911.chr12.final_cleanup_DEL_chr12_5405': 'phase2_DEL_chr12_4072', + 'cohort_2911.chr12.final_cleanup_INS_chr12_42': 'phase4_all_batches.chr12.final_cleanup_INS_chr12_61', + 'cohort_2911.chr14.final_cleanup_DEL_chr14_1513': 'phase2_DEL_chr14_1140', + 'cohort_2911.chr14.final_cleanup_DEL_chr14_2595': 'phase2_DEL_chr14_1998', + 'cohort_2911.chr15.final_cleanup_DEL_chr15_3281': 'phase2_DEL_chr15_2486', + 'cohort_2911.chr16.final_cleanup_DEL_chr16_5308': 'phase2_DEL_chr16_4074', + 'cohort_2911.chr17.final_cleanup_BND_chr17_209': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_250', + 'cohort_2911.chr16.final_cleanup_BND_chr16_635': 'phase4_all_batches.chr16.final_cleanup_BND_chr16_744', + 'cohort_2911.chr17.final_cleanup_CPX_chr17_79': 'phase2_CPX_chr17_46', + 'cohort_2911.chr17.final_cleanup_DEL_chr17_306': 'phase2_DEL_chr17_271', + 'cohort_2911.chr17.final_cleanup_DEL_chr17_3161': 'phase2_DEL_chr17_2627', + 'cohort_2911.chr17.final_cleanup_DEL_chr17_4111': 'phase2_DEL_chr17_3390', + 'cohort_2911.chr17.final_cleanup_DEL_chr17_5157': 'phase2_DEL_chr17_4242', + 'cohort_2911.chr18.final_cleanup_DEL_chr18_3519': 'phase2_DEL_chr18_2585', + 'cohort_2911.chr19.final_cleanup_BND_chr19_162': 'phase4_all_batches.chr19.final_cleanup_BND_chr19_185', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_3395': 'phase2_DEL_chrX_2020', + 'cohort_2911.chr19.final_cleanup_DEL_chr19_1825': 'phase2_DEL_chr19_1567', + 'cohort_2911.chr19.final_cleanup_DEL_chr19_3787': 'phase2_DEL_chr19_3203', + 'cohort_2911.chr19.final_cleanup_DEL_chr19_807': 'phase2_DEL_chr19_692', + 'cohort_2911.chr19.final_cleanup_DEL_chr19_971': 'phase2_DEL_chr19_832', + 'cohort_2911.chr2.final_cleanup_DEL_chr2_7485': 'phase2_DEL_chr2_5601', + 'cohort_2911.chr20.final_cleanup_DEL_chr20_37': 'phase2_DEL_chr20_25', + 'cohort_2911.chr20.final_cleanup_INV_chr20_6': 'phase2_INV_chr20_3', + 'cohort_2911.chr22.final_cleanup_DEL_chr22_1440': 'phase2_DEL_chr22_1210', + 'cohort_2911.chr3.final_cleanup_DUP_chr3_566': 'phase2_DUP_chr3_522', + 'cohort_2911.chr4.final_cleanup_DEL_chr4_5267': 'phase2_DEL_chr4_3794', + 'cohort_2911.chr6.final_cleanup_DEL_chr6_9094': 'phase2_DEL_chr6_6868', + 'cohort_2911.chr7.final_cleanup_DEL_chr7_1540': 'phase2_DEL_chr7_1311', + 'cohort_2911.chr8.final_cleanup_DEL_chr8_75': 'phase2_DEL_chr8_67', + 'cohort_2911.chr8.final_cleanup_DEL_chr8_7934': 'phase2_DEL_chr8_5754', + 'cohort_2911.chr9.final_cleanup_DEL_chr9_6983': 'phase2_DEL_chr9_5130', + 'cohort_2911.chrX.final_cleanup_BND_chrX_166': 'phase4_all_batches.chrX.final_cleanup_BND_chrX_227', + 'cohort_2911.chrX.final_cleanup_CPX_chrX_106': 'cohort_2911.chrX.final_cleanup_CPX_chrX_105', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1861': 'phase2_DEL_chrX_1151', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1922': 'phase2_DEL_chrX_1184', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1936': 'phase2_DEL_chrX_1194', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1974': 'phase2_DEL_chrX_1222', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_2607': 'phase2_DEL_chrX_1550', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_2882': 'phase2_DEL_chrX_1715', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_3450': 'phase2_DEL_chrX_2061', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_6616': 'phase2_DEL_chrX_3880', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_7868': 'phase2_DEL_chrX_4582', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_7872': 'phase2_DEL_chrX_4585', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_7882': 'phase2_DEL_chrX_4594', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_1102': 'phase2_DUP_chrX_1000', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_1791': 'phase2_DUP_chrX_1595', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_2682': 'phase2_DUP_chrX_2394', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_3233': 'phase2_DUP_chrX_2875', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_3564': 'phase2_DUP_chrX_3160', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_3599': 'phase2_DUP_chrX_3192', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_3621': 'phase2_DUP_chrX_3211', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_600': 'phase2_DUP_chrX_554', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_656': 'phase2_DUP_chrX_606', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_818': 'phase2_DUP_chrX_743', + 'phase2_BND_chr17_485': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_710', + 'phase2_BND_chr1_1262': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1669', + 'phase2_BND_chr1_19': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_25', + 'phase2_BND_chr4_1666': 'phase4_all_batches.chr4.final_cleanup_BND_chr4_2264', + 'phase2_BND_chr5_1124': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1476', + 'phase2_DUP_chr11_942': 'phase4_all_batches.chr11.final_cleanup_DUP_chr11_1405', + 'cohort_2911.chr19.final_cleanup_DEL_chr19_3909': 'phase2_DEL_chr19_3310', + 'cohort_2911.chr19.final_cleanup_DUP_chr19_1707': 'phase2_DUP_chr19_1571', + 'cohort_2911.chr2.final_cleanup_DUP_chr2_3291': 'phase2_DUP_chr2_2862', + 'cohort_2911.chr3.final_cleanup_DEL_chr3_8602': 'phase2_DEL_chr3_6419', + 'cohort_2911.chr3.final_cleanup_BND_chr3_2956': 'phase4_all_batches.chr3.final_cleanup_BND_chr3_3398', + 'cohort_2911.chr6.final_cleanup_DEL_chr6_7703': 'phase2_DEL_chr6_5832', + 'cohort_2911.chrX.final_cleanup_CPX_chrX_69': 'phase4_all_batches.chrX.final_cleanup_CPX_chrX_63', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1879': 'phase2_DEL_chrX_1160', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_3396': 'phase2_DEL_chrX_2020', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_6041': 'phase2_DEL_chrX_3547', + 'phase2_CTX_chr3_2': 'phase4_all_batches.chr3.final_cleanup_CTX_chr3_3', + 'phase2_DEL_chr11_3941': 'phase4_all_batches.chr11.final_cleanup_DEL_chr11_6231', + 'phase2_DEL_chr16_453': 'phase2_DEL_chr16_452', + 'phase2_DEL_chr16_516': 'phase2_DEL_chr16_517', + 'phase2_DEL_chr20_2752': 'phase4_all_batches.chr20.final_cleanup_DEL_chr20_4722', + 'phase2_DUP_chr16_1292': 'phase4_all_batches.chr16.final_cleanup_DUP_chr16_1975', + 'cohort_2911.chr2.final_cleanup_DEL_chr2_10322': 'phase2_DEL_chr2_7665', + 'cohort_2911.chr9.final_cleanup_DEL_chr9_2715': 'phase2_DEL_chr9_2002', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1905': 'phase2_DEL_chrX_1175', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_1533': 'phase4_all_batches.chrX.final_cleanup_DUP_chrX_2546', + 'phase2_CPX_chr1_28': 'cohort_2911.chr1.final_cleanup_CPX_chr1_49', + 'phase2_DEL_chr18_2435': 'cohort_2911.chr18.final_cleanup_DEL_chr18_3308', + 'phase2_DUP_chr8_4806': 'phase2_DUP_chr8_4807', + 'cohort_2911.chr2.final_cleanup_BND_chr2_3079': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_3571', + 'cohort_2911.chr2.final_cleanup_BND_chr2_166': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_192', + 'phase2_BND_chr13_369': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_478', + 'phase2_BND_chr7_230': 'phase4_all_batches.chr7.final_cleanup_BND_chr7_300', + 'phase2_BND_chr8_238': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_320', + 'phase2_BND_chr8_53': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_72', + 'phase2_BND_chr9_489': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_663', + 'cohort_2911.chr5.final_cleanup_BND_chr5_1116': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1251', + 'cohort_2911.chr9.final_cleanup_BND_chr9_764': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_849', + 'phase2_BND_chr13_368': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_479', + }, + 'WES': { + 'R4_variant_7334_DUP_08162023': 'R4_variant_7334_DUP', + 'prefix_112949_DEL': 'suffix_210553_DEL', + 'prefix_131670_DUP': 'suffix_251025_DUP', + 'suffix_19443_DEL_2': 'suffix_20030_DEL', + 'prefix_184342_DEL': 'suffix_194439_DEL', + 'prefix_255018_DEL': 'suffix_155939_DEL', + 'prefix_188042_DUP': 'suffix_107531_DUP', + 'suffix_104367_DUP_2': 'suffix_107531_DUP', + 'prefix_59865_DEL': 'suffix_124465_DEL', + 'suffix_120814_DEL_2': 'suffix_124465_DEL', + 'prefix_152595_DEL': 'suffix_286580_DEL', + 'suffix_277504_DEL_2': 'suffix_286581_DEL', + 'prefix_185630_DEL': 'suffix_217490_DEL', + 'suffix_210888_DEL_2': 'suffix_217490_DEL', + 'prefix_126071_DUP': 'suffix_48694_DUP', + 'suffix_47226_DUP_2': 'suffix_48694_DUP', + 'prefix_177696_DUP': 'suffix_23_DUP', + 'prefix_33049_DEL': 'suffix_123373_DEL', + 'suffix_119753_DEL_2': 'suffix_123373_DEL', + 'prefix_6968_DUP': 'suffix_8158_DUP', + 'prefix_111128_DEL': 'suffix_124250_DEL', + 'prefix_265455_DEL': 'suffix_124250_DEL', + 'prefix_133233_DEL': 'suffix_149757_DEL', + 'suffix_145373_DEL_2': 'suffix_149757_DEL', + 'prefix_104962_DEL': 'suffix_262703_DEL', + 'prefix_230669_DUP': 'suffix_37212_DUP', + 'prefix_195634_DUP': 'suffix_337680_DUP', + 'suffix_326904_DUP_2': 'suffix_337680_DUP', + 'prefix_252896_DUP': 'suffix_154760_DUP', + 'prefix_252895_DUP': 'suffix_150484_DUP', + 'prefix_192808_DUP': 'suffix_182662_DUP', + 'prefix_31312_DUP': 'suffix_343278_DUP', + 'suffix_332369_DUP_2': 'suffix_343278_DUP', + 'prefix_201559_DEL': 'suffix_228780_DEL', + 'prefix_173086_DEL': 'suffix_195404_DEL', + 'prefix_230567_DEL': 'suffix_261336_DEL', + 'prefix_25206_DEL': 'suffix_27995_DEL', + 'prefix_72517_DEL': 'suffix_27995_DEL', + 'prefix_194936_DUP': 'suffix_220746_DUP', + 'prefix_117065_DEL': 'suffix_131048_DEL', + 'suffix_127235_DEL_2': 'suffix_131048_DEL', + 'prefix_168621_DEL': 'suffix_336993_DEL', + 'prefix_297236_DEL': 'suffix_336993_DEL', + 'prefix_198998_DUP': 'suffix_225862_DUP', + 'prefix_239300_DUP': 'suffix_271302_DUP', + 'prefix_236836_DEL': 'suffix_268435_DEL', + 'prefix_31131_DUP': 'suffix_34872_DUP', + 'prefix_238314_DEL': 'suffix_336979_DEL', + 'prefix_75798_DUP': 'suffix_84697_DUP', + 'prefix_261715_DEL': 'suffix_337689_DEL', + 'prefix_183989_DEL': 'suffix_4946_DEL', + 'prefix_26448_DEL': 'suffix_43066_DEL', + 'suffix_41786_DEL_2': 'suffix_43066_DEL', + 'prefix_103176_DEL': 'suffix_114811_DEL', + 'suffix_111479_DEL_2': 'suffix_114811_DEL', + 'prefix_120528_DEL': 'suffix_135375_DEL', + 'prefix_296302_DEL': 'suffix_335637_DEL', + 'prefix_108274_DEL': 'suffix_121089_DEL', + 'prefix_46615_DEL': 'suffix_118481_DEL', + 'prefix_185022_DUP': 'suffix_12708_DUP', + 'suffix_12284_DUP_2': 'suffix_12708_DUP', + 'prefix_252585_DUP': 'suffix_309117_DUP', + 'suffix_299264_DUP_2': 'suffix_309117_DUP', + 'prefix_80031_DUP': 'suffix_241944_DUP', + 'suffix_234357_DUP_2': 'suffix_241944_DUP', + 'prefix_117179_DEL': 'suffix_131305_DEL', + 'prefix_71499_DUP': 'suffix_330986_DUP', + 'suffix_320433_DUP_2': 'suffix_330986_DUP', + 'prefix_24788_DUP': 'suffix_336670_DUP', + 'suffix_325911_DUP_2': 'suffix_336670_DUP', + 'prefix_192365_DUP': 'suffix_139090_DUP', + 'prefix_255360_DUP': 'suffix_337082_DUP', + 'prefix_237614_DEL': 'suffix_269379_DEL', + 'prefix_284003_DEL': 'suffix_321174_DEL', + 'suffix_12426_DEL_2': 'suffix_12852_DEL', + 'prefix_9852_DEL': 'suffix_12852_DEL', + 'suffix_50042_DEL_2': 'suffix_51609_DEL', + 'prefix_223675_DEL': 'suffix_51609_DEL', + 'prefix_175156_DEL': 'suffix_54122_DEL', + 'suffix_52499_DEL_2': 'suffix_54122_DEL', + 'prefix_3138_DEL': 'suffix_109381_DEL', + 'suffix_106163_DEL_2': 'suffix_109381_DEL', + 'prefix_255276_DEL': 'suffix_93464_DEL', + 'prefix_152594_DEL': 'suffix_286542_DEL', + 'prefix_77396_DEL': 'suffix_287070_DEL', + 'prefix_152599_DEL': 'suffix_287163_DEL', + 'prefix_152600_DEL': 'suffix_287381_DEL', + 'prefix_108317_DEL': 'suffix_161791_DEL', + 'prefix_201562_DEL': 'suffix_168210_DEL', + 'suffix_163193_DEL_2': 'suffix_168210_DEL', + 'prefix_246635_DEL': 'suffix_92687_DEL', + 'suffix_89974_DEL_2': 'suffix_92687_DEL', + 'prefix_246642_DEL': 'suffix_93299_DEL', + 'suffix_90573_DEL_2': 'suffix_93299_DEL', + 'prefix_246674_DUP': 'suffix_126676_DUP', + 'suffix_122983_DUP_2': 'suffix_126676_DUP', + 'prefix_199389_DEL': 'suffix_94004_DEL', + 'prefix_3593_DEL': 'suffix_27094_DEL', + 'prefix_100725_DEL': 'suffix_336955_DEL', + 'prefix_297227_DEL': 'suffix_336980_DEL', + 'prefix_179648_DEL': 'suffix_202719_DEL', + 'prefix_143182_DEL': 'suffix_100756_DEL', + 'prefix_34890_DEL': 'suffix_38919_DEL', + 'prefix_36703_DEL': 'suffix_39815_DEL', + 'prefix_164553_DEL': 'suffix_185929_DEL', + 'prefix_132206_DEL': 'suffix_282692_DEL', + 'suffix_273714_DEL_2': 'suffix_282692_DEL', + 'prefix_144442_DEL': 'suffix_282716_DEL', + 'suffix_273738_DEL_2': 'suffix_282716_DEL', + 'suffix_273732_DEL_2': 'suffix_282710_DEL', + 'prefix_6326_DEL': 'suffix_282710_DEL', + 'prefix_178200_DEL': 'suffix_118870_DEL', + 'suffix_115409_DEL_2': 'suffix_118870_DEL', + 'prefix_254507_DUP': 'suffix_283948_DUP', + 'prefix_254510_DUP': 'suffix_39298_DUP', + 'prefix_13260_DEL': 'suffix_38202_DEL', + 'suffix_37073_DEL_2': 'suffix_38202_DEL', + 'prefix_278497_DEL': 'suffix_338533_DEL', + 'suffix_327734_DEL_2': 'suffix_338533_DEL', + 'prefix_242131_DEL': 'suffix_176483_DEL', + 'suffix_171236_DEL_2': 'suffix_176483_DEL', + 'suffix_32210_DEL_2': 'suffix_33192_DEL', + 'prefix_37286_DEL': 'suffix_33192_DEL', + 'prefix_113302_DUP': 'suffix_236396_DUP', + 'suffix_228996_DUP_2': 'suffix_236396_DUP', + 'prefix_154396_DEL': 'suffix_174270_DEL', + 'prefix_71769_DEL': 'suffix_227830_DEL', + 'suffix_220764_DEL_2': 'suffix_227830_DEL', + 'prefix_44455_DEL': 'suffix_50096_DEL', + 'prefix_135473_DEL': 'suffix_152259_DEL', + 'prefix_83881_DUP': 'suffix_93771_DUP', + 'prefix_185180_DEL': 'suffix_190807_DEL', + 'prefix_230340_DEL': 'suffix_217490_DEL', + 'prefix_98781_DUP': 'suffix_118080_DUP', + 'suffix_333515_DEL_2': 'suffix_344457_DEL', + 'prefix_50567_DEL': 'suffix_344457_DEL', + 'suffix_243997_DEL_2': 'suffix_251931_DEL', + 'prefix_185688_DEL': 'suffix_251931_DEL', + 'prefix_185697_DEL': 'suffix_252985_DEL', + 'suffix_245019_DEL_2': 'suffix_252985_DEL', + 'prefix_185700_DEL': 'suffix_253165_DEL', + 'suffix_245194_DEL_2': 'suffix_253165_DEL', + 'prefix_164600_DUP': 'suffix_33272_DUP', + 'suffix_32287_DUP_2': 'suffix_33272_DUP', + 'prefix_239826_DUP': 'suffix_339672_DUP', + 'suffix_328837_DUP_2': 'suffix_339672_DUP', + 'prefix_239847_DUP': 'suffix_341812_DUP', + 'suffix_330933_DUP_2': 'suffix_341812_DUP', + 'suffix_217083_DUP_2': 'suffix_223945_DUP', + 'prefix_78332_DUP': 'suffix_223945_DUP', + 'prefix_81687_DUP': 'suffix_132894_DUP', + 'suffix_129007_DUP_2': 'suffix_132894_DUP', + 'suffix_150605_DEL_2': 'suffix_155170_DEL', + 'prefix_47681_DEL': 'suffix_155170_DEL', + 'suffix_119752_DUP_2': 'suffix_123374_DUP', + 'prefix_75741_DUP': 'suffix_123374_DUP', + 'prefix_17370_DUP': 'suffix_19297_DUP', + 'prefix_54006_DUP': 'suffix_19297_DUP', + 'prefix_137115_DUP': 'suffix_67761_DUP', + 'suffix_65769_DUP_2': 'suffix_67761_DUP', + 'prefix_127018_DUP': 'suffix_183995_DUP', + 'prefix_179697_DUP': 'suffix_183995_DUP', + 'prefix_65335_DEL': 'suffix_334692_DEL', + 'prefix_26523_DEL': 'suffix_92931_DEL', + 'suffix_90209_DEL_2': 'suffix_92931_DEL', + 'prefix_121355_DEL': 'suffix_310958_DEL', + 'prefix_3970_DEL': 'suffix_311681_DEL', + 'prefix_16643_DEL': 'suffix_337172_DEL', + 'suffix_326410_DEL_2': 'suffix_337172_DEL', + 'prefix_98251_DEL': 'suffix_179586_DEL', + 'suffix_174257_DEL_2': 'suffix_179586_DEL', + 'prefix_302982_DUP': 'suffix_344928_DUP', + 'prefix_78630_DUP': 'suffix_344928_DUP', + 'prefix_248115_DEL': 'suffix_344177_DEL', + 'prefix_60051_DUP': 'suffix_338928_DUP', + 'suffix_328124_DUP_2': 'suffix_338928_DUP', + 'prefix_202120_DUP': 'suffix_10024_DUP', + 'prefix_137202_DEL': 'suffix_105069_DEL', + 'suffix_101981_DEL_2': 'suffix_105069_DEL', + 'suffix_312453_DEL_2': 'suffix_322732_DEL', + 'prefix_101760_DEL': 'suffix_322732_DEL', + 'prefix_164827_DUP': 'suffix_67293_DUP', + 'prefix_164831_DEL': 'suffix_223792_DEL', + 'suffix_216934_DEL_2': 'suffix_223792_DEL', + 'prefix_47704_DEL': 'suffix_24051_DEL', + 'prefix_108297_DEL': 'suffix_155176_DEL', + 'prefix_284632_DEL': 'suffix_155176_DEL', + 'prefix_37083_DUP': 'suffix_246174_DUP', + 'prefix_37090_DUP': 'suffix_246928_DUP', + 'prefix_37105_DUP': 'suffix_247737_DUP', + 'prefix_164627_DEL': 'suffix_308855_DEL', + 'prefix_164630_DUP': 'suffix_309488_DUP', + 'prefix_247841_DUP': 'suffix_310746_DUP', + 'prefix_164644_DUP': 'suffix_311682_DUP', + 'suffix_301772_DUP_2': 'suffix_311682_DUP', + 'prefix_137780_DEL': 'suffix_223761_DEL', + 'prefix_12281_DEL': 'suffix_30168_DEL', + 'suffix_29274_DEL_2': 'suffix_30168_DEL', + 'prefix_202150_DEL': 'suffix_5892_DEL', + 'prefix_249235_DUP': 'suffix_178512_DUP', + 'suffix_173214_DUP_2': 'suffix_178512_DUP', + 'prefix_286336_DUP': 'suffix_137695_DUP', + 'suffix_133674_DUP_2': 'suffix_137695_DUP', + 'prefix_202153_DUP': 'suffix_217142_DUP', + 'suffix_210547_DUP_2': 'suffix_217142_DUP', + 'suffix_101293_DUP_2': 'suffix_104364_DUP', + 'prefix_240446_DUP': 'suffix_104364_DUP', + 'suffix_326211_DEL_2': 'suffix_336972_DEL', + 'suffix_214414_DUP_2': 'suffix_221162_DUP', + 'suffix_264403_DUP_2': 'suffix_273062_DUP', + 'suffix_92903_DEL_2': 'suffix_95683_DEL', + 'suffix_326915_DEL_2': 'suffix_337695_DEL', + 'suffix_432_DEL_2': 'suffix_442_DEL', + 'suffix_96995_DEL_2': 'suffix_99914_DEL', + 'suffix_16343_DEL_2': 'suffix_16857_DEL', + 'suffix_239134_DUP_2': 'suffix_246928_DUP', + 'suffix_239928_DUP_2': 'suffix_247737_DUP', + 'suffix_299627_DUP_2': 'suffix_309488_DUP', + 'suffix_300851_DUP_2': 'suffix_310746_DUP', + 'prefix_100003_DEL': 'suffix_183787_DEL', + 'prefix_101443_DEL': 'suffix_308476_DEL', + 'prefix_10035_DEL': 'suffix_191074_DEL', + 'prefix_102085_DUP': 'suffix_258879_DUP', + 'prefix_102166_DUP': 'suffix_308496_DUP', + 'prefix_102435_DEL': 'suffix_4265_DEL', + 'prefix_102446_DEL': 'suffix_8242_DEL', + 'prefix_102604_DEL': 'suffix_114190_DEL', + 'prefix_10313_DEL': 'suffix_282612_DEL', + 'prefix_10397_DEL': 'suffix_299122_DEL', + 'prefix_104124_DEL': 'suffix_115920_DEL', + 'prefix_104792_DEL': 'suffix_116623_DEL', + 'prefix_105204_DEL': 'suffix_321184_DEL', + 'prefix_10520_DEL': 'suffix_12114_DEL', + 'prefix_105408_DEL': 'suffix_124299_DEL', + 'prefix_105567_DEL': 'suffix_137041_DEL', + 'prefix_105575_DEL': 'suffix_132895_DEL', + 'prefix_105957_DEL': 'suffix_117939_DEL', + 'prefix_106328_DUP': 'suffix_118372_DUP', + 'prefix_10674_DEL': 'suffix_62051_DEL', + 'prefix_106904_DUP': 'suffix_119175_DUP', + 'prefix_10706_DUP': 'suffix_12310_DUP', + 'prefix_107471_DEL': 'suffix_119959_DEL', + 'prefix_107474_DEL': 'suffix_107226_DEL', + 'prefix_107640_DEL': 'suffix_42123_DEL', + 'prefix_107682_DEL': 'suffix_328436_DEL', + 'prefix_107713_DEL': 'suffix_295713_DEL', + 'prefix_107739_DEL': 'suffix_120359_DEL', + 'prefix_108245_DUP': 'suffix_345422_DUP', + 'prefix_108246_DUP': 'suffix_345434_DUP', + 'prefix_108445_DEL': 'suffix_193406_DEL', + 'prefix_108449_DUP': 'suffix_306110_DUP', + 'prefix_108682_DEL': 'suffix_121584_DEL', + 'prefix_108787_DUP': 'suffix_121697_DUP', + 'prefix_109461_DUP': 'suffix_69248_DUP', + 'prefix_109954_DEL': 'suffix_136917_DEL', + 'prefix_11087_DEL': 'suffix_12711_DEL', + 'prefix_111170_DUP': 'suffix_124298_DUP', + 'prefix_111287_DUP': 'suffix_124448_DUP', + 'prefix_111457_DEL': 'suffix_325622_DEL', + 'prefix_112147_DEL': 'suffix_176498_DEL', + 'prefix_112170_DEL': 'suffix_189247_DEL', + 'prefix_112473_DEL': 'suffix_189253_DEL', + 'prefix_112579_DUP': 'suffix_218784_DUP', + 'prefix_112737_DEL': 'suffix_322979_DEL', + 'prefix_113007_DEL': 'suffix_189763_DEL', + 'prefix_113045_DEL': 'suffix_182363_DEL', + 'prefix_113061_DUP': 'suffix_258764_DUP', + 'prefix_113221_DUP': 'suffix_345607_DUP', + 'prefix_113248_DEL': 'suffix_220751_DEL', + 'prefix_113388_DEL': 'suffix_49218_DEL', + 'prefix_114137_DUP': 'suffix_181479_DUP', + 'prefix_114155_DEL': 'suffix_203405_DEL', + 'prefix_114307_DUP': 'suffix_127847_DUP', + 'prefix_114524_DEL': 'suffix_236802_DEL', + 'prefix_115161_DEL': 'suffix_337104_DEL', + 'prefix_115528_DEL': 'suffix_345416_DEL', + 'prefix_115594_DEL': 'suffix_185881_DEL', + 'prefix_1164_DEL': 'suffix_1403_DEL', + 'prefix_116701_DEL': 'suffix_130655_DEL', + 'prefix_11784_DEL': 'suffix_193515_DEL', + 'prefix_11793_DEL': 'suffix_261170_DEL', + 'prefix_118448_DUP': 'suffix_132890_DUP', + 'prefix_118567_DUP': 'suffix_257299_DUP', + 'prefix_118926_DEL': 'suffix_272807_DEL', + 'prefix_119331_DEL': 'suffix_133934_DEL', + 'prefix_119619_DUP': 'suffix_251043_DUP', + 'prefix_12014_DEL': 'suffix_171415_DEL', + 'prefix_120661_DUP': 'suffix_253757_DUP', + 'prefix_120683_DUP': 'suffix_299274_DUP', + 'prefix_121292_DUP': 'suffix_136324_DUP', + 'prefix_122017_DUP': 'suffix_169951_DUP', + 'prefix_122819_DEL': 'suffix_137963_DEL', + 'prefix_122974_DUP': 'suffix_138130_DUP', + 'prefix_123007_DUP': 'suffix_138159_DUP', + 'prefix_123041_DEL': 'suffix_335303_DEL', + 'prefix_123977_DEL': 'suffix_139305_DEL', + 'prefix_12470_DEL': 'suffix_226823_DEL', + 'prefix_124928_DEL': 'suffix_140340_DEL', + 'prefix_12605_DEL': 'suffix_241864_DEL', + 'prefix_127020_DUP': 'suffix_202584_DUP', + 'prefix_127505_DUP': 'suffix_252433_DUP', + 'prefix_12830_DEL': 'suffix_282628_DEL', + 'prefix_13020_DEL': 'suffix_14730_DEL', + 'prefix_13052_DEL': 'suffix_311669_DEL', + 'prefix_131669_DEL': 'suffix_33717_DEL', + 'prefix_131746_DEL': 'suffix_147895_DEL', + 'prefix_131753_DEL': 'suffix_111130_DEL', + 'prefix_13269_DUP': 'suffix_14995_DUP', + 'prefix_133114_DUP': 'suffix_217164_DUP', + 'prefix_133204_DUP': 'suffix_49743_DUP', + 'prefix_133405_DEL': 'suffix_342958_DEL', + 'prefix_13359_DEL': 'suffix_15085_DEL', + 'prefix_13369_DEL': 'suffix_52450_DEL', + 'prefix_133839_DUP': 'suffix_82443_DUP', + 'prefix_134071_DEL': 'suffix_307896_DEL', + 'prefix_134074_DEL': 'suffix_308281_DEL', + 'prefix_134075_DEL': 'suffix_308321_DEL', + 'prefix_134411_DEL': 'suffix_196304_DEL', + 'prefix_134571_DEL': 'suffix_150896_DEL', + 'prefix_134858_DEL': 'suffix_151606_DEL', + 'prefix_135555_DEL': 'suffix_152343_DEL', + 'prefix_136351_DUP': 'suffix_153160_DUP', + 'prefix_137771_DUP': 'suffix_290375_DUP', + 'prefix_138469_DUP': 'suffix_211854_DUP', + 'prefix_138595_DUP': 'suffix_97099_DUP', + 'prefix_138886_DUP': 'suffix_118224_DUP', + 'prefix_139433_DEL': 'suffix_21321_DEL', + 'prefix_139627_DEL': 'suffix_228579_DEL', + 'prefix_141117_DEL': 'suffix_75812_DEL', + 'prefix_141394_DEL': 'suffix_212333_DEL', + 'prefix_141485_DEL': 'suffix_158734_DEL', + 'prefix_141766_DUP': 'suffix_250308_DUP', + 'prefix_142256_DEL': 'suffix_151736_DEL', + 'prefix_142808_DEL': 'suffix_50260_DEL', + 'prefix_143266_DEL': 'suffix_283626_DEL', + 'prefix_143618_DEL': 'suffix_336978_DEL', + 'prefix_144319_DEL': 'suffix_12781_DEL', + 'prefix_144571_DEL': 'suffix_225727_DEL', + 'prefix_144878_DEL': 'suffix_162955_DEL', + 'prefix_144935_DEL': 'suffix_88144_DEL', + 'prefix_14542_DEL': 'suffix_29900_DEL', + 'prefix_145614_DEL': 'suffix_64370_DEL', + 'prefix_145631_DUP': 'suffix_213097_DUP', + 'prefix_145739_DUP': 'suffix_29512_DUP', + 'prefix_145751_DUP': 'suffix_142338_DUP', + 'prefix_145859_DEL': 'suffix_236767_DEL', + 'prefix_146138_DEL': 'suffix_341905_DEL', + 'prefix_146698_DUP': 'suffix_75558_DUP', + 'prefix_146725_DEL': 'suffix_92679_DEL', + 'prefix_146819_DEL': 'suffix_204036_DEL', + 'prefix_146982_DUP': 'suffix_80598_DUP', + 'prefix_147832_DUP': 'suffix_36481_DUP', + 'prefix_147905_DEL': 'suffix_116249_DEL', + 'prefix_14586_DEL': 'suffix_189167_DEL', + 'prefix_150663_DUP': 'suffix_169723_DUP', + 'prefix_150988_DEL': 'suffix_170148_DEL', + 'prefix_151487_DEL': 'suffix_87868_DEL', + 'prefix_151896_DUP': 'suffix_298612_DUP', + 'prefix_151897_DUP': 'suffix_67741_DUP', + 'prefix_151958_DEL': 'suffix_72228_DEL', + 'prefix_152188_DEL': 'suffix_258374_DEL', + 'prefix_152243_DEL': 'suffix_171706_DEL', + 'prefix_152598_DEL': 'suffix_287152_DEL', + 'prefix_153441_DUP': 'suffix_181703_DUP', + 'prefix_15373_DEL': 'suffix_17189_DEL', + 'prefix_155426_DEL': 'suffix_175579_DEL', + 'prefix_156182_DEL': 'suffix_12082_DEL', + 'prefix_157522_DEL': 'suffix_222475_DEL', + 'prefix_157813_DUP': 'suffix_178342_DUP', + 'prefix_158523_DUP': 'suffix_313465_DUP', + 'prefix_158661_DEL': 'suffix_88654_DEL', + 'prefix_160612_DEL': 'suffix_248116_DEL', + 'prefix_163949_DUP': 'suffix_344732_DUP', + 'prefix_164522_DEL': 'suffix_185883_DEL', + 'prefix_164622_DEL': 'suffix_186016_DEL', + 'prefix_164629_DUP': 'suffix_309441_DUP', + 'prefix_164642_DUP': 'suffix_311548_DUP', + 'prefix_164777_DEL': 'suffix_186196_DEL', + 'prefix_165076_DEL': 'suffix_328506_DEL', + 'prefix_165078_DEL': 'suffix_328786_DEL', + 'prefix_1657_DEL': 'suffix_2017_DEL', + 'prefix_167202_DEL': 'suffix_336972_DEL', + 'prefix_168614_DUP': 'suffix_190522_DUP', + 'prefix_168800_DUP': 'suffix_122588_DUP', + 'prefix_169366_DEL': 'suffix_191327_DEL', + 'prefix_170220_DUP': 'suffix_122478_DUP', + 'prefix_170275_DEL': 'suffix_329992_DEL', + 'prefix_170760_DEL': 'suffix_192854_DEL', + 'prefix_171486_DEL': 'suffix_193608_DEL', + 'prefix_17150_DEL': 'suffix_98009_DEL', + 'prefix_17230_DEL': 'suffix_336965_DEL', + 'prefix_17231_DEL': 'suffix_337022_DEL', + 'prefix_172982_DUP': 'suffix_195278_DUP', + 'prefix_17317_DEL': 'suffix_275059_DEL', + 'prefix_173203_DEL': 'suffix_195570_DEL', + 'prefix_173318_DEL': 'suffix_173502_DEL', + 'prefix_173699_DUP': 'suffix_127421_DUP', + 'prefix_173819_DEL': 'suffix_344967_DEL', + 'prefix_174099_DEL': 'suffix_151801_DEL', + 'prefix_174146_DEL': 'suffix_193980_DEL', + 'prefix_174221_DUP': 'suffix_125584_DUP', + 'prefix_174279_DEL': 'suffix_157330_DEL', + 'prefix_174372_DEL': 'suffix_262729_DEL', + 'prefix_174562_DEL': 'suffix_174111_DEL', + 'prefix_174564_DEL': 'suffix_1324_DEL', + 'prefix_175140_DEL': 'suffix_272786_DEL', + 'prefix_175243_DUP': 'suffix_322125_DUP', + 'prefix_175281_DUP': 'suffix_323384_DUP', + 'prefix_175291_DUP': 'suffix_323971_DUP', + 'prefix_176490_DUP': 'suffix_337504_DUP', + 'prefix_176785_DEL': 'suffix_256937_DEL', + 'prefix_177587_DUP': 'suffix_56526_DUP', + 'prefix_177834_DUP': 'suffix_276088_DUP', + 'prefix_178234_DEL': 'suffix_170832_DEL', + 'prefix_178374_DUP': 'suffix_173115_DUP', + 'prefix_178921_DEL': 'suffix_201891_DEL', + 'prefix_179719_DEL': 'suffix_326378_DEL', + 'prefix_179787_DEL': 'suffix_337951_DEL', + 'prefix_179804_DEL': 'suffix_289797_DEL', + 'prefix_180023_DEL': 'suffix_298661_DEL', + 'prefix_180149_DUP': 'suffix_203322_DUP', + 'prefix_180338_DEL': 'suffix_45778_DEL', + 'prefix_180351_DEL': 'suffix_107299_DEL', + 'prefix_180464_DEL': 'suffix_342917_DEL', + 'prefix_180500_DEL': 'suffix_183250_DEL', + 'prefix_180680_DUP': 'suffix_82979_DUP', + 'prefix_180721_DEL': 'suffix_4948_DEL', + 'prefix_181685_DEL': 'suffix_42286_DEL', + 'prefix_181724_DUP': 'suffix_343954_DUP', + 'prefix_18172_DUP': 'suffix_94217_DUP', + 'prefix_182085_DUP': 'suffix_248675_DUP', + 'prefix_182185_DUP': 'suffix_185453_DUP', + 'prefix_182381_DEL': 'suffix_65928_DEL', + 'prefix_182507_DEL': 'suffix_162956_DEL', + 'prefix_182_DEL': 'suffix_106425_DEL', + 'prefix_183723_DUP': 'suffix_207297_DUP', + 'prefix_184282_DUP': 'suffix_301766_DUP', + 'prefix_184370_DEL': 'suffix_156979_DEL', + 'prefix_184996_DUP': 'suffix_328901_DUP', + 'prefix_185023_DEL': 'suffix_15090_DEL', + 'prefix_185054_DEL': 'suffix_280761_DEL', + 'prefix_185055_DEL': 'suffix_132024_DEL', + 'prefix_185138_DEL': 'suffix_127756_DEL', + 'prefix_185141_DEL': 'suffix_134624_DEL', + 'prefix_185531_DEL': 'suffix_132027_DEL', + 'prefix_185615_DEL': 'suffix_217486_DEL', + 'prefix_185627_DEL': 'suffix_224393_DEL', + 'prefix_185677_DEL': 'suffix_173146_DEL', + 'prefix_185750_DUP': 'suffix_102606_DUP', + 'prefix_185860_DEL': 'suffix_241831_DEL', + 'prefix_185877_DEL': 'suffix_302969_DEL', + 'prefix_185935_DEL': 'suffix_173921_DEL', + 'prefix_186018_DEL': 'suffix_217498_DEL', + 'prefix_186031_DEL': 'suffix_44013_DEL', + 'prefix_156507_DEL': 'suffix_176830_DEL', + 'prefix_186032_DUP': 'suffix_2023_DUP', + 'prefix_186063_DEL': 'suffix_209934_DEL', + 'prefix_186624_DUP': 'suffix_210575_DUP', + 'prefix_186854_DUP': 'suffix_210807_DUP', + 'prefix_187881_DEL': 'suffix_92683_DEL', + 'prefix_190360_DEL': 'suffix_214876_DEL', + 'prefix_191004_DUP': 'suffix_215642_DUP', + 'prefix_191257_DEL': 'suffix_228609_DEL', + 'prefix_191292_DUP': 'suffix_234879_DUP', + 'prefix_191354_DUP': 'suffix_187230_DUP', + 'prefix_191471_DEL': 'suffix_208014_DEL', + 'prefix_191474_DEL': 'suffix_280450_DEL', + 'prefix_191494_DUP': 'suffix_202343_DUP', + 'prefix_191709_DEL': 'suffix_171576_DEL', + 'prefix_191829_DUP': 'suffix_217491_DUP', + 'prefix_191939_DEL': 'suffix_62014_DEL', + 'prefix_192002_DUP': 'suffix_271568_DUP', + 'prefix_192090_DEL': 'suffix_132549_DEL', + 'prefix_192468_DUP': 'suffix_217487_DUP', + 'prefix_192692_DEL': 'suffix_217799_DEL', + 'prefix_192750_DEL': 'suffix_217878_DEL', + 'prefix_192794_DEL': 'suffix_103053_DEL', + 'prefix_192809_DEL': 'suffix_242803_DEL', + 'prefix_192811_DUP': 'suffix_288571_DUP', + 'prefix_192812_DUP': 'suffix_288581_DUP', + 'prefix_192881_DEL': 'suffix_203645_DEL', + 'prefix_193000_DEL': 'suffix_327418_DEL', + 'prefix_193117_DEL': 'suffix_105935_DEL', + 'prefix_193570_DEL': 'suffix_218935_DEL', + 'prefix_193750_DEL': 'suffix_171289_DEL', + 'prefix_193822_DEL': 'suffix_219267_DEL', + 'prefix_193980_DEL': 'suffix_173513_DEL', + 'prefix_193998_DEL': 'suffix_342920_DEL', + 'prefix_193_DEL': 'suffix_175579_DEL', + 'prefix_195154_DUP': 'suffix_262756_DUP', + 'prefix_195332_DUP': 'suffix_301004_DUP', + 'prefix_195356_DEL': 'suffix_67397_DEL', + 'prefix_195394_DUP': 'suffix_32943_DUP', + 'prefix_195409_DEL': 'suffix_345343_DEL', + 'prefix_195446_DUP': 'suffix_218272_DUP', + 'prefix_195686_DEL': 'suffix_36573_DEL', + 'prefix_195893_DEL': 'suffix_216802_DEL', + 'prefix_196343_DEL': 'suffix_187682_DEL', + 'prefix_196653_DEL': 'suffix_287837_DEL', + 'prefix_197018_DEL': 'suffix_176475_DEL', + 'prefix_197343_DEL': 'suffix_329991_DEL', + 'prefix_197345_DUP': 'suffix_217953_DUP', + 'prefix_197514_DEL': 'suffix_224017_DEL', + 'prefix_197601_DUP': 'suffix_92304_DUP', + 'prefix_197689_DEL': 'suffix_4035_DEL', + 'prefix_19776_DUP': 'suffix_22049_DUP', + 'prefix_197839_DEL': 'suffix_37637_DEL', + 'prefix_197846_DEL': 'suffix_201717_DEL', + 'prefix_197848_DUP': 'suffix_285533_DUP', + 'prefix_197857_DEL': 'suffix_33261_DEL', + 'prefix_197858_DEL': 'suffix_33273_DEL', + 'prefix_1981_DEL': 'suffix_2420_DEL', + 'prefix_198681_DEL': 'suffix_225482_DEL', + 'prefix_198960_DEL': 'suffix_214208_DEL', + 'prefix_198962_DUP': 'suffix_298013_DUP', + 'prefix_199230_DUP': 'suffix_219340_DUP', + 'prefix_199526_DEL': 'suffix_244411_DEL', + 'prefix_199572_DEL': 'suffix_273734_DEL', + 'prefix_199578_DEL': 'suffix_132651_DEL', + 'prefix_201088_DEL': 'suffix_343155_DEL', + 'prefix_201199_DUP': 'suffix_136419_DUP', + 'prefix_201221_DEL': 'suffix_79200_DEL', + 'prefix_201288_DEL': 'suffix_228487_DEL', + 'prefix_201337_DEL': 'suffix_228541_DEL', + 'prefix_201565_DEL': 'suffix_168578_DEL', + 'prefix_202110_DEL': 'suffix_177304_DEL', + 'prefix_202154_DUP': 'suffix_217485_DUP', + 'prefix_202256_DEL': 'suffix_331004_DEL', + 'prefix_202259_DUP': 'suffix_163215_DUP', + 'prefix_203581_DEL': 'suffix_231189_DEL', + 'prefix_20401_DEL': 'suffix_22739_DEL', + 'prefix_204188_DUP': 'suffix_1547_DUP', + 'prefix_204826_DUP': 'suffix_117758_DUP', + 'prefix_20504_DUP': 'suffix_61796_DUP', + 'prefix_205313_DEL': 'suffix_233124_DEL', + 'prefix_205330_DEL': 'suffix_233146_DEL', + 'prefix_20552_DEL': 'suffix_162107_DEL', + 'prefix_20587_DEL': 'suffix_177607_DEL', + 'prefix_206642_DUP': 'suffix_191476_DUP', + 'prefix_20842_DUP': 'suffix_247208_DUP', + 'prefix_208997_DUP': 'suffix_68391_DUP', + 'prefix_209842_DEL': 'suffix_238144_DEL', + 'prefix_210581_DEL': 'suffix_282703_DEL', + 'prefix_210630_DEL': 'suffix_126646_DEL', + 'prefix_210904_DUP': 'suffix_239309_DUP', + 'prefix_211888_DEL': 'suffix_337006_DEL', + 'prefix_212443_DEL': 'suffix_19298_DEL', + 'prefix_215700_DUP': 'suffix_244607_DUP', + 'prefix_216096_DEL': 'suffix_271336_DEL', + 'prefix_21663_DUP': 'suffix_30036_DUP', + 'prefix_21679_DUP': 'suffix_343012_DUP', + 'prefix_21680_DEL': 'suffix_344961_DEL', + 'prefix_218702_DUP': 'suffix_248113_DUP', + 'prefix_219145_DEL': 'suffix_198229_DEL', + 'prefix_221051_DEL': 'suffix_32473_DEL', + 'prefix_22118_DEL': 'suffix_291073_DEL', + 'prefix_222234_DEL': 'suffix_176476_DEL', + 'prefix_22347_DUP': 'suffix_336271_DUP', + 'prefix_223818_DEL': 'suffix_253681_DEL', + 'prefix_223972_DEL': 'suffix_253871_DEL', + 'prefix_224087_DEL': 'suffix_254000_DEL', + 'prefix_22411_DEL': 'suffix_329985_DEL', + 'prefix_22607_DEL': 'suffix_107271_DEL', + 'prefix_22608_DEL': 'suffix_107295_DEL', + 'prefix_22727_DEL': 'suffix_93545_DEL', + 'prefix_227942_DUP': 'suffix_258345_DUP', + 'prefix_228561_DEL': 'suffix_247820_DEL', + 'prefix_228798_DEL': 'suffix_224733_DEL', + 'prefix_229038_DUP': 'suffix_40155_DUP', + 'prefix_229254_DUP': 'suffix_345612_DUP', + 'prefix_229345_DUP': 'suffix_178578_DUP', + 'prefix_229543_DUP': 'suffix_260172_DUP', + 'prefix_229784_DEL': 'suffix_4824_DEL', + 'prefix_229791_DUP': 'suffix_188332_DUP', + 'prefix_229805_DUP': 'suffix_301762_DUP', + 'prefix_229839_DEL': 'suffix_39713_DEL', + 'prefix_229851_DEL': 'suffix_134949_DEL', + 'prefix_229879_DUP': 'suffix_12928_DUP', + 'prefix_230003_DEL': 'suffix_107348_DEL', + 'prefix_230020_DUP': 'suffix_118182_DUP', + 'prefix_230025_DUP': 'suffix_223544_DUP', + 'prefix_230090_DUP': 'suffix_332572_DUP', + 'prefix_230178_DUP': 'suffix_302641_DUP', + 'prefix_230183_DUP': 'suffix_74958_DUP', + 'prefix_230251_DEL': 'suffix_133150_DEL', + 'prefix_230271_DEL': 'suffix_161946_DEL', + 'prefix_230308_DEL': 'suffix_244610_DEL', + 'prefix_230350_DEL': 'suffix_252059_DEL', + 'prefix_230445_DEL': 'suffix_19341_DEL', + 'prefix_230454_DUP': 'suffix_313052_DUP', + 'prefix_230603_DEL': 'suffix_77668_DEL', + 'prefix_230616_DEL': 'suffix_316424_DEL', + 'prefix_230738_DEL': 'suffix_261522_DEL', + 'prefix_231752_DEL': 'suffix_262711_DEL', + 'prefix_231770_DEL': 'suffix_262731_DEL', + 'prefix_231952_DUP': 'suffix_322130_DUP', + 'prefix_232113_DUP': 'suffix_85765_DUP', + 'prefix_232394_DEL': 'suffix_308668_DEL', + 'prefix_232438_DEL': 'suffix_263531_DEL', + 'prefix_232444_DEL': 'suffix_263537_DEL', + 'prefix_232452_DUP': 'suffix_80443_DUP', + 'prefix_232540_DUP': 'suffix_134297_DUP', + 'prefix_232595_DEL': 'suffix_189246_DEL', + 'prefix_232637_DEL': 'suffix_182330_DEL', + 'prefix_232677_DEL': 'suffix_55000_DEL', + 'prefix_232776_DUP': 'suffix_262197_DUP', + 'prefix_232899_DUP': 'suffix_329033_DUP', + 'prefix_232900_DUP': 'suffix_329084_DUP', + 'prefix_233045_DEL': 'suffix_150737_DEL', + 'prefix_233119_DEL': 'suffix_264248_DEL', + 'prefix_233311_DEL': 'suffix_241927_DEL', + 'prefix_233321_DEL': 'suffix_344725_DEL', + 'prefix_233717_DUP': 'suffix_201544_DUP', + 'prefix_234863_DEL': 'suffix_185898_DEL', + 'prefix_23489_DEL': 'suffix_331839_DEL', + 'prefix_234953_DEL': 'suffix_109390_DEL', + 'prefix_234990_DEL': 'suffix_200659_DEL', + 'prefix_236431_DUP': 'suffix_267998_DUP', + 'prefix_237073_DUP': 'suffix_319276_DUP', + 'prefix_237178_DEL': 'suffix_93427_DEL', + 'prefix_237197_DEL': 'suffix_65316_DEL', + 'prefix_237515_DUP': 'suffix_269265_DUP', + 'prefix_237761_DEL': 'suffix_218136_DEL', + 'prefix_237780_DUP': 'suffix_237634_DUP', + 'prefix_237791_DEL': 'suffix_37586_DEL', + 'prefix_237797_DUP': 'suffix_279247_DUP', + 'prefix_237833_DUP': 'suffix_118074_DUP', + 'prefix_237857_DEL': 'suffix_217856_DEL', + 'prefix_237872_DUP': 'suffix_54996_DUP', + 'prefix_237874_DEL': 'suffix_172470_DEL', + 'prefix_237991_DUP': 'suffix_138522_DUP', + 'prefix_238021_DUP': 'suffix_136966_DUP', + 'prefix_238147_DUP': 'suffix_270013_DUP', + 'prefix_238938_DEL': 'suffix_270899_DEL', + 'prefix_238945_DUP': 'suffix_308856_DUP', + 'prefix_238961_DEL': 'suffix_310956_DEL', + 'prefix_239339_DEL': 'suffix_255507_DEL', + 'prefix_240008_DEL': 'suffix_272099_DEL', + 'prefix_240186_DEL': 'suffix_264944_DEL', + 'prefix_240251_DUP': 'suffix_151050_DUP', + 'prefix_240270_DEL': 'suffix_39288_DEL', + 'prefix_240483_DEL': 'suffix_39678_DEL', + 'prefix_241524_DEL': 'suffix_121573_DEL', + 'prefix_241665_DUP': 'suffix_274132_DUP', + 'prefix_241760_DEL': 'suffix_274237_DEL', + 'prefix_242143_DEL': 'suffix_178407_DEL', + 'prefix_242192_DUP': 'suffix_317053_DUP', + 'prefix_242247_DUP': 'suffix_345761_DUP', + 'prefix_242331_DEL': 'suffix_169106_DEL', + 'prefix_24255_DEL': 'suffix_212769_DEL', + 'prefix_242646_DEL': 'suffix_212305_DEL', + 'prefix_242678_DEL': 'suffix_259663_DEL', + 'prefix_242679_DEL': 'suffix_259723_DEL', + 'prefix_242681_DEL': 'suffix_259864_DEL', + 'prefix_242685_DEL': 'suffix_260161_DEL', + 'prefix_244399_DEL': 'suffix_277206_DEL', + 'prefix_244573_DUP': 'suffix_277418_DUP', + 'prefix_24479_DUP': 'suffix_119594_DUP', + 'prefix_245216_DEL': 'suffix_270267_DEL', + 'prefix_245217_DEL': 'suffix_89519_DEL', + 'prefix_245531_DEL': 'suffix_21740_DEL', + 'prefix_245561_DEL': 'suffix_56540_DEL', + 'prefix_245634_DUP': 'suffix_70271_DUP', + 'prefix_245655_DEL': 'suffix_38769_DEL', + 'prefix_245659_DUP': 'suffix_175204_DUP', + 'prefix_245721_DUP': 'suffix_199804_DUP', + 'prefix_245813_DUP': 'suffix_40862_DUP', + 'prefix_24581_DUP': 'suffix_141949_DUP', + 'prefix_24583_DEL': 'suffix_217839_DEL', + 'prefix_245903_DUP': 'suffix_127338_DUP', + 'prefix_24592_DEL': 'suffix_46695_DEL', + 'prefix_245933_DUP': 'suffix_279006_DUP', + 'prefix_24593_DEL': 'suffix_46762_DEL', + 'prefix_245959_DUP': 'suffix_121405_DUP', + 'prefix_24595_DEL': 'suffix_46839_DEL', + 'prefix_245986_DUP': 'suffix_239962_DUP', + 'prefix_246310_DEL': 'suffix_139843_DEL', + 'prefix_24637_DEL': 'suffix_185437_DEL', + 'prefix_24662_DEL': 'suffix_302153_DEL', + 'prefix_247373_DEL': 'suffix_280718_DEL', + 'prefix_247836_DUP': 'suffix_309310_DUP', + 'prefix_247847_DUP': 'suffix_311619_DUP', + 'prefix_247848_DUP': 'suffix_311683_DUP', + 'prefix_248168_DEL': 'suffix_132850_DEL', + 'prefix_24841_DUP': 'suffix_2568_DUP', + 'prefix_248446_DUP': 'suffix_90044_DUP', + 'prefix_24870_DEL': 'suffix_337649_DEL', + 'prefix_248998_DUP': 'suffix_282433_DUP', + 'prefix_24904_DEL': 'suffix_298012_DEL', + 'prefix_249320_DUP': 'suffix_227500_DUP', + 'prefix_249361_DEL': 'suffix_220755_DEL', + 'prefix_24943_DEL': 'suffix_157892_DEL', + 'prefix_249451_DUP': 'suffix_6339_DUP', + 'prefix_250214_DEL': 'suffix_293400_DEL', + 'prefix_250459_DEL': 'suffix_284644_DEL', + 'prefix_250460_DEL': 'suffix_69510_DEL', + 'prefix_250461_DEL': 'suffix_134254_DEL', + 'prefix_250483_DUP': 'suffix_277241_DUP', + 'prefix_250509_DEL': 'suffix_218771_DEL', + 'prefix_250600_DEL': 'suffix_52226_DEL', + 'prefix_250616_DEL': 'suffix_322013_DEL', + 'prefix_250674_DUP': 'suffix_304173_DUP', + 'prefix_250694_DUP': 'suffix_313136_DUP', + 'prefix_250818_DEL': 'suffix_27205_DEL', + 'prefix_250819_DEL': 'suffix_27210_DEL', + 'prefix_250889_DUP': 'suffix_111556_DUP', + 'prefix_250932_DUP': 'suffix_133045_DUP', + 'prefix_250981_DEL': 'suffix_91074_DEL', + 'prefix_251102_DEL': 'suffix_146078_DEL', + 'prefix_251291_DUP': 'suffix_284913_DUP', + 'prefix_251847_DEL': 'suffix_224352_DEL', + 'prefix_251893_DEL': 'suffix_212162_DEL', + 'prefix_252103_DEL': 'suffix_306862_DEL', + 'prefix_252152_DEL': 'suffix_252976_DEL', + 'prefix_252262_DEL': 'suffix_171030_DEL', + 'prefix_252266_DUP': 'suffix_253194_DUP', + 'prefix_252268_DEL': 'suffix_127157_DEL', + 'prefix_25237_DEL': 'suffix_344078_DEL', + 'prefix_252465_DEL': 'suffix_255499_DEL', + 'prefix_252468_DUP': 'suffix_61453_DUP', + 'prefix_252588_DEL': 'suffix_179491_DEL', + 'prefix_252589_DEL': 'suffix_158777_DEL', + 'prefix_252657_DEL': 'suffix_332312_DEL', + 'prefix_252672_DUP': 'suffix_8208_DUP', + 'prefix_252677_DEL': 'suffix_297350_DEL', + 'prefix_252719_DEL': 'suffix_118143_DEL', + 'prefix_253047_DUP': 'suffix_211078_DUP', + 'prefix_254257_DEL': 'suffix_258839_DEL', + 'prefix_254295_DEL': 'suffix_37577_DEL', + 'prefix_254321_DUP': 'suffix_102639_DUP', + 'prefix_254471_DUP': 'suffix_210560_DUP', + 'prefix_254486_DEL': 'suffix_61825_DEL', + 'prefix_254504_DEL': 'suffix_89660_DEL', + 'prefix_254514_DEL': 'suffix_100226_DEL', + 'prefix_254902_DUP': 'suffix_165508_DUP', + 'prefix_254973_DEL': 'suffix_166161_DEL', + 'prefix_255104_DEL': 'suffix_4167_DEL', + 'prefix_255420_DUP': 'suffix_72092_DUP', + 'prefix_255620_DEL': 'suffix_258860_DEL', + 'prefix_255624_DEL': 'suffix_260164_DEL', + 'prefix_256004_DEL': 'suffix_211645_DEL', + 'prefix_257014_DUP': 'suffix_170805_DUP', + 'prefix_257015_DUP': 'suffix_170872_DUP', + 'prefix_257016_DUP': 'suffix_170906_DUP', + 'prefix_257671_DUP': 'suffix_291855_DUP', + 'prefix_258103_DEL': 'suffix_292360_DEL', + 'prefix_25895_DEL': 'suffix_105875_DEL', + 'prefix_259285_DEL': 'suffix_176497_DEL', + 'prefix_260110_DEL': 'suffix_8146_DEL', + 'prefix_260118_DEL': 'suffix_106667_DEL', + 'prefix_260165_DUP': 'suffix_241615_DUP', + 'prefix_260287_DUP': 'suffix_130663_DUP', + 'prefix_260388_DUP': 'suffix_68171_DUP', + 'prefix_260444_DUP': 'suffix_295009_DUP', + 'prefix_260683_DEL': 'suffix_169109_DEL', + 'prefix_260715_DEL': 'suffix_100707_DEL', + 'prefix_260762_DEL': 'suffix_117930_DEL', + 'prefix_260790_DUP': 'suffix_172476_DUP', + 'prefix_260802_DEL': 'suffix_323217_DEL', + 'prefix_260879_DEL': 'suffix_27705_DEL', + 'prefix_260902_DEL': 'suffix_67106_DEL', + 'prefix_260908_DEL': 'suffix_129735_DEL', + 'prefix_260_DEL': 'suffix_55007_DEL', + 'prefix_261548_DEL': 'suffix_16881_DEL', + 'prefix_26254_DEL': 'suffix_29316_DEL', + 'prefix_262735_DEL': 'suffix_297605_DEL', + 'prefix_263583_DUP': 'suffix_318930_DUP', + 'prefix_263614_DEL': 'suffix_309554_DEL', + 'prefix_263688_DUP': 'suffix_287109_DUP', + 'prefix_263713_DEL': 'suffix_315234_DEL', + 'prefix_263717_DEL': 'suffix_206974_DEL', + 'prefix_263844_DEL': 'suffix_93604_DEL', + 'prefix_264229_DEL': 'suffix_299315_DEL', + 'prefix_264544_DEL': 'suffix_159878_DEL', + 'prefix_26468_DEL': 'suffix_29565_DEL', + 'prefix_264705_DUP': 'suffix_122589_DUP', + 'prefix_264818_DEL': 'suffix_165079_DEL', + 'prefix_264891_DUP': 'suffix_113524_DUP', + 'prefix_265301_DEL': 'suffix_300475_DEL', + 'prefix_265379_DEL': 'suffix_246254_DEL', + 'prefix_266533_DEL': 'suffix_301845_DEL', + 'prefix_266544_DEL': 'suffix_301853_DEL', + 'prefix_266545_DEL': 'suffix_301855_DEL', + 'prefix_267_DEL': 'suffix_122539_DEL', + 'prefix_269805_DEL': 'suffix_305404_DEL', + 'prefix_2705_DEL': 'suffix_139305_DEL', + 'prefix_270760_DEL': 'suffix_306509_DEL', + 'prefix_27114_DEL': 'suffix_150454_DEL', + 'prefix_271645_DEL': 'suffix_30695_DEL', + 'prefix_272591_DEL': 'suffix_308628_DEL', + 'prefix_272819_DEL': 'suffix_308867_DEL', + 'prefix_276988_DEL': 'suffix_174302_DEL', + 'prefix_277163_DUP': 'suffix_29468_DUP', + 'prefix_277765_DEL': 'suffix_190995_DEL', + 'prefix_277834_DEL': 'suffix_237788_DEL', + 'prefix_277849_DUP': 'suffix_187676_DUP', + 'prefix_278532_DEL': 'suffix_224674_DEL', + 'prefix_278576_DEL': 'suffix_225982_DEL', + 'prefix_27857_DEL': 'suffix_31144_DEL', + 'prefix_278620_DEL': 'suffix_61479_DEL', + 'prefix_278624_DEL': 'suffix_62018_DEL', + 'prefix_278768_DEL': 'suffix_260377_DEL', + 'prefix_280607_DUP': 'suffix_49717_DUP', + 'prefix_280679_DEL': 'suffix_88375_DEL', + 'prefix_280785_DUP': 'suffix_317588_DUP', + 'prefix_282546_DUP': 'suffix_188434_DUP', + 'prefix_282581_DUP': 'suffix_232495_DUP', + 'prefix_283354_DEL': 'suffix_183799_DEL', + 'prefix_283359_DEL': 'suffix_139093_DEL', + 'prefix_283381_DUP': 'suffix_271584_DUP', + 'prefix_283498_DUP': 'suffix_184079_DUP', + 'prefix_284364_DEL': 'suffix_16928_DEL', + 'prefix_2843_DUP': 'suffix_3431_DUP', + 'prefix_284546_DUP': 'suffix_234318_DUP', + 'prefix_284653_DEL': 'suffix_29356_DEL', + 'prefix_284718_DEL': 'suffix_233901_DEL', + 'prefix_284720_DEL': 'suffix_233993_DEL', + 'prefix_285131_DEL': 'suffix_322576_DEL', + 'prefix_285518_DUP': 'suffix_219653_DUP', + 'prefix_285910_DUP': 'suffix_131091_DUP', + 'prefix_285938_DEL': 'suffix_114389_DEL', + 'prefix_285942_DUP': 'suffix_261140_DUP', + 'prefix_285998_DUP': 'suffix_191729_DUP', + 'prefix_286350_DUP': 'suffix_151353_DUP', + 'prefix_286450_DUP': 'suffix_77791_DUP', + 'prefix_287805_DEL': 'suffix_325622_DEL', + 'prefix_295024_DUP': 'suffix_333861_DUP', + 'prefix_295802_DEL': 'suffix_334964_DEL', + 'prefix_296006_DUP': 'suffix_335221_DUP', + 'prefix_296764_DUP': 'suffix_336317_DUP', + 'prefix_297208_DEL': 'suffix_336953_DEL', + 'prefix_297213_DEL': 'suffix_336958_DEL', + 'prefix_297223_DEL': 'suffix_336975_DEL', + 'prefix_191716_DEL': 'suffix_107536_DEL', + 'prefix_297224_DEL': 'suffix_336978_DEL', + 'prefix_297279_DEL': 'suffix_337045_DEL', + 'prefix_297283_DEL': 'suffix_337053_DEL', + 'prefix_297309_DUP': 'suffix_337088_DUP', + 'prefix_29997_DEL': 'suffix_223825_DEL', + 'prefix_301818_DEL': 'suffix_343282_DEL', + 'prefix_30205_DEL': 'suffix_253777_DEL', + 'prefix_30217_DEL': 'suffix_264896_DEL', + 'prefix_302357_DEL': 'suffix_344078_DEL', + 'prefix_303466_DUP': 'suffix_345668_DUP', + 'prefix_32087_DEL': 'suffix_9607_DEL', + 'prefix_32332_DUP': 'suffix_88460_DUP', + 'prefix_32367_DEL': 'suffix_119716_DEL', + 'prefix_32706_DEL': 'suffix_36588_DEL', + 'prefix_33082_DUP': 'suffix_248523_DUP', + 'prefix_3322_DEL': 'suffix_336956_DEL', + 'prefix_33597_DUP': 'suffix_37540_DUP', + 'prefix_33614_DUP': 'suffix_37570_DUP', + 'prefix_34068_DEL': 'suffix_38077_DEL', + 'prefix_34213_DUP': 'suffix_92153_DUP', + 'prefix_34545_DEL': 'suffix_38571_DEL', + 'prefix_34597_DEL': 'suffix_109532_DEL', + 'prefix_34869_DEL': 'suffix_87992_DEL', + 'prefix_35040_DEL': 'suffix_27009_DEL', + 'prefix_35106_DEL': 'suffix_237735_DEL', + 'prefix_35180_DUP': 'suffix_190057_DUP', + 'prefix_35775_DEL': 'suffix_39951_DEL', + 'prefix_36042_DUP': 'suffix_42_DUP', + 'prefix_36108_DUP': 'suffix_339711_DUP', + 'prefix_36111_DEL': 'suffix_297767_DEL', + 'prefix_3617_DEL': 'suffix_183767_DEL', + 'prefix_36425_DEL': 'suffix_109536_DEL', + 'prefix_36528_DEL': 'suffix_338271_DEL', + 'prefix_36589_DUP': 'suffix_225864_DUP', + 'prefix_36614_DEL': 'suffix_36479_DEL', + 'prefix_36746_DUP': 'suffix_72227_DUP', + 'prefix_36846_DUP': 'suffix_107654_DUP', + 'prefix_37079_DUP': 'suffix_245784_DUP', + 'prefix_37080_DUP': 'suffix_245791_DUP', + 'prefix_37081_DUP': 'suffix_245798_DUP', + 'prefix_37082_DUP': 'suffix_246118_DUP', + 'prefix_37094_DUP': 'suffix_247108_DUP', + 'prefix_37095_DUP': 'suffix_247270_DUP', + 'prefix_37096_DUP': 'suffix_247352_DUP', + 'prefix_37098_DUP': 'suffix_247400_DUP', + 'prefix_37100_DUP': 'suffix_247574_DUP', + 'prefix_37108_DUP': 'suffix_248111_DUP', + 'prefix_37112_DUP': 'suffix_248141_DUP', + 'prefix_37282_DEL': 'suffix_32945_DEL', + 'prefix_37834_DEL': 'suffix_42257_DEL', + 'prefix_3816_DEL': 'suffix_265546_DEL', + 'prefix_3852_DEL': 'suffix_282714_DEL', + 'prefix_39151_DEL': 'suffix_336958_DEL', + 'prefix_39601_DEL': 'suffix_337053_DEL', + 'prefix_39656_DEL': 'suffix_221906_DEL', + 'prefix_3991_DEL': 'suffix_317434_DEL', + 'prefix_40087_DUP': 'suffix_198027_DUP', + 'prefix_41276_DUP': 'suffix_46123_DUP', + 'prefix_41357_DEL': 'suffix_121015_DEL', + 'prefix_42493_DUP': 'suffix_118615_DUP', + 'prefix_42547_DUP': 'suffix_47670_DUP', + 'prefix_42549_DEL': 'suffix_47672_DEL', + 'prefix_43054_DEL': 'suffix_253999_DEL', + 'prefix_43130_DEL': 'suffix_48422_DEL', + 'prefix_43183_DEL': 'suffix_48494_DEL', + 'prefix_43482_DEL': 'suffix_29927_DEL', + 'prefix_43657_DEL': 'suffix_49127_DEL', + 'prefix_44112_DEL': 'suffix_12769_DEL', + 'prefix_44349_DEL': 'suffix_234536_DEL', + 'prefix_44363_DEL': 'suffix_235902_DEL', + 'prefix_45800_DEL': 'suffix_182593_DEL', + 'prefix_45862_DEL': 'suffix_51654_DEL', + 'prefix_45965_DUP': 'suffix_140827_DUP', + 'prefix_46144_DEL': 'suffix_299340_DEL', + 'prefix_46154_DUP': 'suffix_308320_DUP', + 'prefix_46234_DEL': 'suffix_329984_DEL', + 'prefix_46275_DEL': 'suffix_292777_DEL', + 'prefix_46406_DUP': 'suffix_52251_DUP', + 'prefix_46447_DEL': 'suffix_22508_DEL', + 'prefix_4670_DEL': 'suffix_52153_DEL', + 'prefix_46723_DEL': 'suffix_297314_DEL', + 'prefix_4687_DEL': 'suffix_337096_DEL', + 'prefix_46883_DEL': 'suffix_322347_DEL', + 'prefix_46902_DEL': 'suffix_16522_DEL', + 'prefix_46949_DEL': 'suffix_221643_DEL', + 'prefix_47442_DEL': 'suffix_32709_DEL', + 'prefix_47443_DEL': 'suffix_32942_DEL', + 'prefix_47446_DEL': 'suffix_33262_DEL', + 'prefix_47486_DUP': 'suffix_341592_DUP', + 'prefix_47629_DUP': 'suffix_309398_DUP', + 'prefix_47726_DEL': 'suffix_290347_DEL', + 'prefix_47796_DEL': 'suffix_1449_DEL', + 'prefix_477_DUP': 'suffix_59131_DUP', + 'prefix_4783_DUP': 'suffix_335171_DUP', + 'prefix_47920_DUP': 'suffix_69500_DUP', + 'prefix_48061_DUP': 'suffix_297897_DUP', + 'prefix_48220_DEL': 'suffix_234836_DEL', + 'prefix_48353_DUP': 'suffix_84392_DUP', + 'prefix_49235_DEL': 'suffix_39714_DEL', + 'prefix_4948_DEL': 'suffix_5834_DEL', + 'prefix_49781_DEL': 'suffix_56043_DEL', + 'prefix_49980_DUP': 'suffix_56297_DUP', + 'prefix_50167_DEL': 'suffix_56527_DEL', + 'prefix_51403_DEL': 'suffix_13373_DEL', + 'prefix_51897_DEL': 'suffix_336964_DEL', + 'prefix_52447_DEL': 'suffix_179521_DEL', + 'prefix_53019_DEL': 'suffix_249573_DEL', + 'prefix_53360_DEL': 'suffix_28019_DEL', + 'prefix_53373_DEL': 'suffix_169978_DEL', + 'prefix_54115_DEL': 'suffix_124254_DEL', + 'prefix_5415_DEL': 'suffix_274987_DEL', + 'prefix_55334_DEL': 'suffix_182678_DEL', + 'prefix_55463_DUP': 'suffix_62593_DUP', + 'prefix_5580_DEL': 'suffix_106522_DEL', + 'prefix_56296_DUP': 'suffix_20205_DUP', + 'prefix_56375_DEL': 'suffix_122348_DEL', + 'prefix_56758_DEL': 'suffix_133934_DEL', + 'prefix_5719_DUP': 'suffix_46123_DUP', + 'prefix_57494_DEL': 'suffix_64715_DEL', + 'prefix_5756_DEL': 'suffix_195683_DEL', + 'prefix_58444_DUP': 'suffix_65759_DUP', + 'prefix_59270_DEL': 'suffix_88382_DEL', + 'prefix_598_DEL': 'suffix_729_DEL', + 'prefix_59944_DEL': 'suffix_105066_DEL', + 'prefix_60675_DEL': 'suffix_215556_DEL', + 'prefix_61714_DUP': 'suffix_69571_DUP', + 'prefix_62048_DEL': 'suffix_220763_DEL', + 'prefix_63074_DEL': 'suffix_262731_DEL', + 'prefix_63258_DEL': 'suffix_200662_DEL', + 'prefix_6325_DEL': 'suffix_282673_DEL', + 'prefix_63634_DEL': 'suffix_192720_DEL', + 'prefix_6643_DEL': 'suffix_131071_DEL', + 'prefix_67724_DEL': 'suffix_75807_DEL', + 'prefix_67761_DEL': 'suffix_300474_DEL', + 'prefix_68258_DEL': 'suffix_183179_DEL', + 'prefix_68433_DEL': 'suffix_241887_DEL', + 'prefix_69473_DEL': 'suffix_213975_DEL', + 'prefix_70643_DEL': 'suffix_173247_DEL', + 'prefix_70766_DEL': 'suffix_255510_DEL', + 'prefix_70923_DUP': 'suffix_64371_DUP', + 'prefix_71010_DEL': 'suffix_298994_DEL', + 'prefix_71023_DEL': 'suffix_102982_DEL', + 'prefix_71045_DEL': 'suffix_262680_DEL', + 'prefix_71163_DEL': 'suffix_79602_DEL', + 'prefix_7128_DEL': 'suffix_340564_DEL', + 'prefix_71349_DUP': 'suffix_310205_DUP', + 'prefix_71400_DUP': 'suffix_88019_DUP', + 'prefix_71606_DUP': 'suffix_220541_DUP', + 'prefix_71676_DUP': 'suffix_327952_DUP', + 'prefix_71691_DEL': 'suffix_131549_DEL', + 'prefix_718_DEL': 'suffix_263531_DEL', + 'prefix_71936_DUP': 'suffix_116525_DUP', + 'prefix_71961_DEL': 'suffix_140125_DEL', + 'prefix_71969_DUP': 'suffix_157089_DUP', + 'prefix_72062_DUP': 'suffix_253345_DUP', + 'prefix_72143_DEL': 'suffix_172606_DEL', + 'prefix_72208_DEL': 'suffix_197482_DEL', + 'prefix_72369_DEL': 'suffix_115813_DEL', + 'prefix_73088_DEL': 'suffix_81641_DEL', + 'prefix_73094_DUP': 'suffix_185884_DUP', + 'prefix_73451_DEL': 'suffix_307441_DEL', + 'prefix_7358_DEL': 'suffix_337112_DEL', + 'prefix_74031_DUP': 'suffix_134917_DUP', + 'prefix_74343_DUP': 'suffix_83006_DUP', + 'prefix_74628_DEL': 'suffix_208625_DEL', + 'prefix_74629_DUP': 'suffix_5338_DUP', + 'prefix_74896_DEL': 'suffix_226397_DEL', + 'prefix_75369_DEL': 'suffix_136965_DEL', + 'prefix_75422_DEL': 'suffix_84245_DEL', + 'prefix_75547_DEL': 'suffix_126315_DEL', + 'prefix_75551_DUP': 'suffix_155780_DUP', + 'prefix_75625_DUP': 'suffix_310853_DUP', + 'prefix_7613_DEL': 'suffix_247991_DEL', + 'prefix_7837_DUP': 'suffix_9141_DUP', + 'prefix_78409_DUP': 'suffix_159950_DUP', + 'prefix_78419_DEL': 'suffix_87809_DEL', + 'prefix_78493_DUP': 'suffix_87885_DUP', + 'prefix_78575_DEL': 'suffix_310958_DEL', + 'prefix_78580_DEL': 'suffix_312313_DEL', + 'prefix_78692_DEL': 'suffix_332978_DEL', + 'prefix_78775_DUP': 'suffix_178999_DUP', + 'prefix_78808_DUP': 'suffix_88248_DUP', + 'prefix_78890_DEL': 'suffix_203497_DEL', + 'prefix_78923_DUP': 'suffix_305735_DUP', + 'prefix_78942_DUP': 'suffix_307917_DUP', + 'prefix_78949_DEL': 'suffix_308466_DEL', + 'prefix_78964_DUP': 'suffix_244587_DUP', + 'prefix_79085_DUP': 'suffix_319372_DUP', + 'prefix_79632_DUP': 'suffix_260100_DUP', + 'prefix_80034_DUP': 'suffix_298803_DUP', + 'prefix_8058_DUP': 'suffix_177412_DUP', + 'prefix_80933_DEL': 'suffix_90592_DEL', + 'prefix_8121_DUP': 'suffix_9447_DUP', + 'prefix_816_DUP': 'suffix_986_DUP', + 'prefix_81889_DEL': 'suffix_91604_DEL', + 'prefix_8206_DUP': 'suffix_170741_DUP', + 'prefix_7357_DEL': 'suffix_337078_DEL', + 'prefix_82314_DUP': 'suffix_89153_DUP', + 'prefix_82745_DEL': 'suffix_95957_DEL', + 'prefix_83209_DUP': 'suffix_256121_DUP', + 'prefix_83230_DUP': 'suffix_57589_DUP', + 'prefix_83245_DUP': 'suffix_49612_DUP', + 'prefix_83313_DEL': 'suffix_31491_DEL', + 'prefix_83315_DEL': 'suffix_93129_DEL', + 'prefix_83649_DEL': 'suffix_105765_DEL', + 'prefix_83650_DEL': 'suffix_105792_DEL', + 'prefix_83651_DEL': 'suffix_105804_DEL', + 'prefix_83935_DEL': 'suffix_336309_DEL', + 'prefix_83975_DUP': 'suffix_134029_DUP', + 'prefix_83980_DEL': 'suffix_242921_DEL', + 'prefix_83984_DUP': 'suffix_32384_DUP', + 'prefix_84015_DUP': 'suffix_271757_DUP', + 'prefix_84044_DEL': 'suffix_263835_DEL', + 'prefix_8420_DEL': 'suffix_75588_DEL', + 'prefix_84288_DUP': 'suffix_338248_DUP', + 'prefix_84403_DUP': 'suffix_345428_DUP', + 'prefix_84717_DEL': 'suffix_94529_DEL', + 'prefix_85642_DEL': 'suffix_236439_DEL', + 'prefix_86089_DEL': 'suffix_109334_DEL', + 'prefix_86353_DEL': 'suffix_12713_DEL', + 'prefix_86387_DEL': 'suffix_24837_DEL', + 'prefix_87733_DEL': 'suffix_147400_DEL', + 'prefix_87760_DEL': 'suffix_337119_DEL', + 'prefix_88833_DUP': 'suffix_37585_DUP', + 'prefix_88886_DEL': 'suffix_98841_DEL', + 'prefix_889_DUP': 'suffix_1064_DUP', + 'prefix_8950_DEL': 'suffix_314461_DEL', + 'prefix_9017_DEL': 'suffix_344075_DEL', + 'prefix_90242_DUP': 'suffix_340596_DUP', + 'prefix_9027_DEL': 'suffix_10504_DEL', + 'prefix_90299_DEL': 'suffix_170093_DEL', + 'prefix_92044_DUP': 'suffix_229729_DUP', + 'prefix_92416_DEL': 'suffix_41456_DEL', + 'prefix_92442_DUP': 'suffix_140124_DUP', + 'prefix_92506_DUP': 'suffix_244044_DUP', + 'prefix_92527_DUP': 'suffix_308955_DUP', + 'prefix_92550_DEL': 'suffix_336957_DEL', + 'prefix_92558_DUP': 'suffix_309329_DUP', + 'prefix_92559_DUP': 'suffix_309345_DUP', + 'prefix_92627_DUP': 'suffix_342164_DUP', + 'prefix_92640_DEL': 'suffix_256704_DEL', + 'prefix_92708_DUP': 'suffix_259795_DUP', + 'prefix_92724_DEL': 'suffix_252290_DEL', + 'prefix_99990_DUP': 'suffix_306887_DUP', + 'prefix_99998_DUP': 'suffix_308100_DUP', + 'suffix_102137_DEL_2': 'suffix_105232_DEL', + 'suffix_102188_DEL_2': 'suffix_105284_DEL', + 'suffix_102279_DEL_2': 'suffix_105380_DEL', + 'suffix_10229_DUP_2': 'suffix_10602_DUP', + 'suffix_102741_DUP_2': 'suffix_105859_DUP', + 'suffix_103521_DUP_2': 'suffix_106670_DUP', + 'suffix_103609_DEL_2': 'suffix_106761_DEL', + 'suffix_106849_DUP_2': 'suffix_110077_DUP', + 'suffix_107254_DUP_2': 'suffix_110487_DUP', + 'suffix_107284_DEL_2': 'suffix_110517_DEL', + 'suffix_108892_DUP_2': 'suffix_112158_DUP', + 'suffix_112876_DUP_2': 'suffix_116262_DUP', + 'suffix_115899_DUP_2': 'suffix_119375_DUP', + 'suffix_116678_DUP_2': 'suffix_120179_DUP', + 'suffix_116680_DEL_2': 'suffix_120181_DEL', + 'suffix_1179_DEL_2': 'suffix_1216_DEL', + 'suffix_118089_DEL_2': 'suffix_121646_DEL', + 'suffix_118753_DUP_2': 'suffix_122349_DUP', + 'suffix_118957_DUP_2': 'suffix_122563_DUP', + 'suffix_121705_DUP_2': 'suffix_125377_DUP', + 'suffix_121728_DUP_2': 'suffix_125401_DUP', + 'suffix_123193_DEL_2': 'suffix_126890_DEL', + 'suffix_12347_DEL_2': 'suffix_12771_DEL', + 'suffix_123507_DEL_2': 'suffix_127211_DEL', + 'suffix_123859_DUP_2': 'suffix_127567_DUP', + 'suffix_126255_DEL_2': 'suffix_130031_DEL', + 'suffix_126378_DEL_2': 'suffix_130158_DEL', + 'suffix_126871_DEL_2': 'suffix_130662_DEL', + 'suffix_127257_DEL_2': 'suffix_131070_DEL', + 'suffix_127603_DUP_2': 'suffix_131424_DUP', + 'suffix_127635_DUP_2': 'suffix_131456_DUP', + 'suffix_128475_DEL_2': 'suffix_132338_DEL', + 'suffix_130874_DUP_2': 'suffix_134810_DUP', + 'suffix_133323_DUP_2': 'suffix_137335_DUP', + 'suffix_137071_DUP_2': 'suffix_141198_DUP', + 'suffix_1371_DUP_2': 'suffix_1425_DUP', + 'suffix_137756_DUP_2': 'suffix_141889_DUP', + 'suffix_138407_DUP_2': 'suffix_142558_DUP', + 'suffix_139177_DUP_2': 'suffix_143347_DUP', + 'suffix_139287_DEL_2': 'suffix_143460_DEL', + 'suffix_140067_DUP_2': 'suffix_144262_DUP', + 'suffix_140211_DEL_2': 'suffix_144411_DEL', + 'suffix_140216_DUP_2': 'suffix_144417_DUP', + 'suffix_140998_DUP_2': 'suffix_145233_DUP', + 'suffix_143072_DUP_2': 'suffix_147367_DUP', + 'suffix_144703_DUP_2': 'suffix_149060_DUP', + 'suffix_146079_DEL_2': 'suffix_150480_DEL', + 'suffix_146082_DEL_2': 'suffix_150483_DEL', + 'suffix_146309_DEL_2': 'suffix_150716_DEL', + 'suffix_146679_DEL_2': 'suffix_151094_DEL', + 'suffix_147023_DUP_2': 'suffix_151455_DUP', + 'suffix_147062_DUP_2': 'suffix_151496_DUP', + 'suffix_147065_DEL_2': 'suffix_151498_DEL', + 'suffix_147755_DUP_2': 'suffix_152214_DUP', + 'suffix_148894_DUP_2': 'suffix_153392_DUP', + 'suffix_150694_DUP_2': 'suffix_155261_DUP', + 'suffix_152328_DUP_2': 'suffix_156958_DUP', + 'suffix_152423_DUP_2': 'suffix_157059_DUP', + 'suffix_152437_DUP_2': 'suffix_157073_DUP', + 'suffix_152439_DEL_2': 'suffix_157075_DEL', + 'suffix_153155_DUP_2': 'suffix_157821_DUP', + 'suffix_153923_DEL_2': 'suffix_158633_DEL', + 'suffix_15410_DEL_2': 'suffix_15899_DEL', + 'suffix_15414_DEL_2': 'suffix_15903_DEL', + 'suffix_154873_DEL_2': 'suffix_159630_DEL', + 'suffix_156350_DUP_2': 'suffix_161149_DUP', + 'suffix_156426_DUP_2': 'suffix_161225_DUP', + 'suffix_157651_DUP_2': 'suffix_162486_DUP', + 'suffix_161505_DUP_2': 'suffix_166461_DUP', + 'suffix_162350_DUP_2': 'suffix_167337_DUP', + 'suffix_16449_DEL_2': 'suffix_16965_DEL', + 'suffix_16476_DEL_2': 'suffix_16992_DEL', + 'suffix_164891_DUP_2': 'suffix_169964_DUP', + 'suffix_165535_DEL_2': 'suffix_170615_DEL', + 'suffix_167366_DUP_2': 'suffix_172492_DUP', + 'suffix_167382_DUP_2': 'suffix_172508_DUP', + 'suffix_169697_DUP_2': 'suffix_174892_DUP', + 'suffix_170359_DEL_2': 'suffix_175579_DEL', + 'suffix_171232_DEL_2': 'suffix_176479_DEL', + 'suffix_171251_DEL_2': 'suffix_176499_DEL', + 'suffix_171281_DEL_2': 'suffix_176529_DEL', + 'suffix_172345_DUP_2': 'suffix_177610_DUP', + 'suffix_173330_DUP_2': 'suffix_178634_DUP', + 'suffix_173374_DUP_2': 'suffix_178679_DUP', + 'suffix_173898_DEL_2': 'suffix_179214_DEL', + 'suffix_176174_DUP_2': 'suffix_181566_DUP', + 'suffix_176223_DUP_2': 'suffix_181617_DUP', + 'suffix_177584_DEL_2': 'suffix_183019_DEL', + 'suffix_177830_DUP_2': 'suffix_183274_DUP', + 'suffix_178098_DUP_2': 'suffix_183558_DUP', + 'suffix_178276_DEL_2': 'suffix_183739_DEL', + 'suffix_178301_DEL_2': 'suffix_183764_DEL', + 'suffix_179460_DEL_2': 'suffix_184960_DEL', + 'suffix_180373_DEL_2': 'suffix_185895_DEL', + 'suffix_180638_DEL_2': 'suffix_186170_DEL', + 'suffix_180645_DUP_2': 'suffix_186177_DUP', + 'suffix_180670_DUP_2': 'suffix_186202_DUP', + 'suffix_182096_DEL_2': 'suffix_187682_DEL', + 'suffix_182350_DUP_2': 'suffix_187944_DUP', + 'suffix_182381_DEL_2': 'suffix_187975_DEL', + 'suffix_183117_DUP_2': 'suffix_188748_DUP', + 'suffix_183487_DUP_2': 'suffix_189139_DUP', + 'suffix_183693_DUP_2': 'suffix_189354_DUP', + 'suffix_184465_DUP_2': 'suffix_190174_DUP', + 'suffix_186206_DUP_2': 'suffix_191986_DUP', + 'suffix_186235_DEL_2': 'suffix_192019_DEL', + 'suffix_186644_DUP_2': 'suffix_192445_DUP', + 'suffix_186658_DEL_2': 'suffix_192460_DEL', + 'suffix_187547_DEL_2': 'suffix_193377_DEL', + 'suffix_188120_DEL_2': 'suffix_193959_DEL', + 'suffix_188699_DEL_2': 'suffix_194556_DEL', + 'suffix_189196_DUP_2': 'suffix_195072_DUP', + 'suffix_189361_DEL_2': 'suffix_195245_DEL', + 'suffix_189787_DEL_2': 'suffix_195683_DEL', + 'suffix_189988_DUP_2': 'suffix_195887_DUP', + 'suffix_192255_DEL_2': 'suffix_198235_DEL', + 'suffix_194657_DUP_2': 'suffix_200737_DUP', + 'suffix_195491_DUP_2': 'suffix_201616_DUP', + 'suffix_196079_DUP_2': 'suffix_202227_DUP', + 'suffix_197227_DUP_2': 'suffix_203421_DUP', + 'suffix_197905_DUP_2': 'suffix_204137_DUP', + 'suffix_199236_DUP_2': 'suffix_205495_DUP', + 'suffix_199351_DUP_2': 'suffix_205610_DUP', + 'suffix_199613_DUP_2': 'suffix_205874_DUP', + 'suffix_199853_DEL_2': 'suffix_206119_DEL', + 'suffix_200568_DEL_2': 'suffix_206849_DEL', + 'suffix_200934_DUP_2': 'suffix_207220_DUP', + 'suffix_2015_DUP_2': 'suffix_2089_DUP', + 'suffix_201955_DUP_2': 'suffix_208285_DUP', + 'suffix_202641_DUP_2': 'suffix_208994_DUP', + 'suffix_205028_DUP_2': 'suffix_211453_DUP', + 'suffix_205123_DEL_2': 'suffix_211548_DEL', + 'suffix_205138_DEL_2': 'suffix_211564_DEL', + 'suffix_206613_DEL_2': 'suffix_213095_DEL', + 'suffix_206780_DEL_2': 'suffix_213264_DEL', + 'suffix_207068_DEL_2': 'suffix_213553_DEL', + 'suffix_207138_DEL_2': 'suffix_213623_DEL', + 'suffix_207672_DEL_2': 'suffix_214167_DEL', + 'suffix_207715_DEL_2': 'suffix_214213_DEL', + 'suffix_208370_DEL_2': 'suffix_214895_DEL', + 'suffix_208573_DEL_2': 'suffix_215104_DEL', + 'suffix_208972_DEL_2': 'suffix_215523_DEL', + 'suffix_209068_DEL_2': 'suffix_215620_DEL', + 'suffix_209199_DUP_2': 'suffix_215756_DUP', + 'suffix_210259_DUP_2': 'suffix_216845_DUP', + 'suffix_210283_DUP_2': 'suffix_216871_DUP', + 'suffix_210613_DEL_2': 'suffix_217209_DEL', + 'suffix_212296_DEL_2': 'suffix_218955_DEL', + 'suffix_212578_DEL_2': 'suffix_219252_DEL', + 'suffix_212637_DEL_2': 'suffix_219317_DEL', + 'suffix_212756_DEL_2': 'suffix_219444_DEL', + 'suffix_213708_DEL_2': 'suffix_220434_DEL', + 'suffix_213875_DEL_2': 'suffix_220605_DEL', + 'suffix_214152_DEL_2': 'suffix_220894_DEL', + 'suffix_214429_DEL_2': 'suffix_221177_DEL', + 'suffix_214857_DEL_2': 'suffix_221627_DEL', + 'suffix_216188_DEL_2': 'suffix_223018_DEL', + 'suffix_216198_DEL_2': 'suffix_223028_DEL', + 'suffix_216958_DUP_2': 'suffix_223818_DUP', + 'suffix_217583_DEL_2': 'suffix_224470_DEL', + 'suffix_217868_DEL_2': 'suffix_224762_DEL', + 'suffix_217871_DEL_2': 'suffix_224765_DEL', + 'suffix_218073_DUP_2': 'suffix_224973_DUP', + 'suffix_218439_DUP_2': 'suffix_225366_DUP', + 'suffix_218904_DUP_2': 'suffix_225864_DUP', + 'suffix_219658_DUP_2': 'suffix_226657_DUP', + 'suffix_220370_DEL_2': 'suffix_227413_DEL', + 'suffix_223567_DUP_2': 'suffix_230759_DUP', + 'suffix_223841_DUP_2': 'suffix_231044_DUP', + 'suffix_22451_DEL_2': 'suffix_23110_DEL', + 'suffix_224608_DEL_2': 'suffix_231845_DEL', + 'suffix_226755_DEL_2': 'suffix_234078_DEL', + 'suffix_227182_DEL_2': 'suffix_234514_DEL', + 'suffix_227203_DEL_2': 'suffix_234535_DEL', + 'suffix_227204_DEL_2': 'suffix_234536_DEL', + 'suffix_228485_DEL_2': 'suffix_235865_DEL', + 'suffix_2306_DEL_2': 'suffix_2387_DEL', + 'suffix_230714_DUP_2': 'suffix_238195_DUP', + 'suffix_230932_DEL_2': 'suffix_238422_DEL', + 'suffix_231136_DEL_2': 'suffix_238630_DEL', + 'suffix_234299_DUP_2': 'suffix_241888_DUP', + 'suffix_236708_DEL_2': 'suffix_244400_DEL', + 'suffix_236887_DEL_2': 'suffix_244582_DEL', + 'suffix_237084_DEL_2': 'suffix_244791_DEL', + 'suffix_237399_DEL_2': 'suffix_245122_DEL', + 'suffix_238815_DEL_2': 'suffix_246599_DEL', + 'suffix_238984_DUP_2': 'suffix_246775_DUP', + 'suffix_239194_DEL_2': 'suffix_246989_DEL', + 'suffix_239311_DUP_2': 'suffix_247109_DUP', + 'suffix_240516_DUP_2': 'suffix_248341_DUP', + 'suffix_24067_DEL_2': 'suffix_24774_DEL', + 'suffix_24112_DEL_2': 'suffix_24819_DEL', + 'suffix_245147_DUP_2': 'suffix_253116_DUP', + 'suffix_245631_DUP_2': 'suffix_253615_DUP', + 'suffix_246007_DEL_2': 'suffix_254014_DEL', + 'suffix_246832_DUP_2': 'suffix_254855_DUP', + 'suffix_249752_DEL_2': 'suffix_257879_DEL', + 'suffix_249760_DUP_2': 'suffix_257887_DUP', + 'suffix_250396_DUP_2': 'suffix_258549_DUP', + 'suffix_251212_DEL_2': 'suffix_259389_DEL', + 'suffix_251719_DUP_2': 'suffix_259912_DUP', + 'suffix_252163_DEL_2': 'suffix_260375_DEL', + 'suffix_254429_DEL_2': 'suffix_262732_DEL', + 'suffix_254575_DEL_2': 'suffix_262881_DEL', + 'suffix_255204_DEL_2': 'suffix_263531_DEL', + 'suffix_25533_DUP_2': 'suffix_26296_DUP', + 'suffix_256655_DEL_2': 'suffix_265034_DEL', + 'suffix_256744_DEL_2': 'suffix_265124_DEL', + 'suffix_256775_DEL_2': 'suffix_265157_DEL', + 'suffix_25949_DEL_2': 'suffix_26718_DEL', + 'suffix_260357_DEL_2': 'suffix_268858_DEL', + 'suffix_260853_DUP_2': 'suffix_269376_DUP', + 'suffix_260855_DEL_2': 'suffix_269379_DEL', + 'suffix_261034_DEL_2': 'suffix_269563_DEL', + 'suffix_262533_DEL_2': 'suffix_271118_DEL', + 'suffix_262539_DEL_2': 'suffix_271125_DEL', + 'suffix_26417_DUP_2': 'suffix_27197_DUP', + 'suffix_265005_DEL_2': 'suffix_273689_DEL', + 'suffix_266230_DEL_2': 'suffix_274984_DEL', + 'suffix_268706_DUP_2': 'suffix_277538_DUP', + 'suffix_269299_DEL_2': 'suffix_278150_DEL', + 'suffix_27000_DUP_2': 'suffix_27795_DUP', + 'suffix_27116_DEL_2': 'suffix_27917_DEL', + 'suffix_272983_DEL_2': 'suffix_281956_DEL', + 'suffix_273392_DEL_2': 'suffix_282368_DEL', + 'suffix_273424_DUP_2': 'suffix_282400_DUP', + 'suffix_273728_DEL_2': 'suffix_282706_DEL', + 'suffix_273736_DEL_2': 'suffix_282714_DEL', + 'suffix_274940_DEL_2': 'suffix_283943_DEL', + 'suffix_276413_DEL_2': 'suffix_285456_DEL', + 'suffix_27644_DEL_2': 'suffix_28486_DEL', + 'suffix_277451_DEL_2': 'suffix_286527_DEL', + 'suffix_278695_DEL_2': 'suffix_287788_DEL', + 'suffix_279450_DEL_2': 'suffix_288554_DEL', + 'suffix_279452_DEL_2': 'suffix_288556_DEL', + 'suffix_280776_DEL_2': 'suffix_289942_DEL', + 'suffix_280836_DUP_2': 'suffix_290002_DUP', + 'suffix_281014_DEL_2': 'suffix_290187_DEL', + 'suffix_281363_DUP_2': 'suffix_290550_DUP', + 'suffix_281846_DEL_2': 'suffix_291053_DEL', + 'suffix_282575_DUP_2': 'suffix_291822_DUP', + 'suffix_283188_DEL_2': 'suffix_292459_DEL', + 'suffix_283756_DEL_2': 'suffix_293051_DEL', + 'suffix_284360_DEL_2': 'suffix_293684_DEL', + 'suffix_284797_DEL_2': 'suffix_294141_DEL', + 'suffix_286080_DUP_2': 'suffix_295482_DUP', + 'suffix_286258_DUP_2': 'suffix_295667_DUP', + 'suffix_286295_DEL_2': 'suffix_295706_DEL', + 'suffix_286314_DUP_2': 'suffix_295727_DUP', + 'suffix_286315_DEL_2': 'suffix_295726_DEL', + 'suffix_288518_DEL_2': 'suffix_298011_DEL', + 'suffix_288856_DUP_2': 'suffix_298358_DUP', + 'suffix_288968_DEL_2': 'suffix_298482_DEL', + 'suffix_28909_DEL_2': 'suffix_29798_DEL', + 'suffix_289586_DEL_2': 'suffix_299115_DEL', + 'suffix_290193_DEL_2': 'suffix_299736_DEL', + 'suffix_29032_DEL_2': 'suffix_29921_DEL', + 'suffix_292554_DEL_2': 'suffix_302162_DEL', + 'suffix_293764_DEL_2': 'suffix_303412_DEL', + 'suffix_295098_DUP_2': 'suffix_304770_DUP', + 'suffix_295112_DUP_2': 'suffix_304785_DUP', + 'suffix_297809_DUP_2': 'suffix_307588_DUP', + 'suffix_297979_DEL_2': 'suffix_307770_DEL', + 'suffix_298560_DUP_2': 'suffix_308385_DUP', + 'suffix_299334_DEL_2': 'suffix_309189_DEL', + 'suffix_299424_DUP_2': 'suffix_309280_DUP', + 'suffix_299426_DUP_2': 'suffix_309282_DUP', + 'suffix_299461_DUP_2': 'suffix_309319_DUP', + 'suffix_299537_DUP_2': 'suffix_309398_DUP', + 'suffix_299973_DEL_2': 'suffix_309849_DEL', + 'suffix_300029_DEL_2': 'suffix_309906_DEL', + 'suffix_300096_DUP_2': 'suffix_309975_DUP', + 'suffix_300178_DEL_2': 'suffix_310060_DEL', + 'suffix_300705_DUP_2': 'suffix_310598_DUP', + 'suffix_300835_DUP_2': 'suffix_310730_DUP', + 'suffix_300947_DUP_2': 'suffix_310843_DUP', + 'suffix_301053_DUP_2': 'suffix_310954_DUP', + 'suffix_301313_DEL_2': 'suffix_311215_DEL', + 'suffix_302352_DEL_2': 'suffix_312295_DEL', + 'suffix_302614_DUP_2': 'suffix_312565_DUP', + 'suffix_305489_DEL_2': 'suffix_315530_DEL', + 'suffix_305497_DEL_2': 'suffix_315539_DEL', + 'suffix_305995_DEL_2': 'suffix_316050_DEL', + 'suffix_30805_DEL_2': 'suffix_31751_DEL', + 'suffix_310213_DUP_2': 'suffix_320398_DUP', + 'suffix_310801_DUP_2': 'suffix_321018_DUP', + 'suffix_312831_DUP_2': 'suffix_323123_DUP', + 'suffix_313834_DUP_2': 'suffix_324164_DUP', + 'suffix_314680_DUP_2': 'suffix_325048_DUP', + 'suffix_318209_DUP_2': 'suffix_328707_DUP', + 'suffix_319248_DUP_2': 'suffix_329772_DUP', + 'suffix_319459_DEL_2': 'suffix_329991_DEL', + 'suffix_320939_DUP_2': 'suffix_331503_DUP', + 'suffix_321448_DEL_2': 'suffix_332033_DEL', + 'suffix_321475_DEL_2': 'suffix_332061_DEL', + 'suffix_323409_DUP_2': 'suffix_334076_DUP', + 'suffix_323934_DUP_2': 'suffix_334633_DUP', + 'suffix_324816_DUP_2': 'suffix_335549_DUP', + 'suffix_32507_DUP_2': 'suffix_33497_DUP', + 'suffix_325590_DUP_2': 'suffix_336339_DUP', + 'suffix_325871_DUP_2': 'suffix_336630_DUP', + 'suffix_326049_DUP_2': 'suffix_336809_DUP', + 'suffix_326068_DUP_2': 'suffix_336828_DUP', + 'suffix_326213_DUP_2': 'suffix_336974_DUP', + 'suffix_326219_DEL_2': 'suffix_336980_DEL', + 'suffix_326226_DEL_2': 'suffix_336987_DEL', + 'suffix_326275_DUP_2': 'suffix_337036_DUP', + 'suffix_326360_DEL_2': 'suffix_337122_DEL', + 'suffix_326662_DEL_2': 'suffix_337431_DEL', + 'suffix_326846_DEL_2': 'suffix_337620_DEL', + 'suffix_327283_DEL_2': 'suffix_338071_DEL', + 'suffix_327813_DEL_2': 'suffix_338612_DEL', + 'suffix_328030_DUP_2': 'suffix_338831_DUP', + 'suffix_328354_DUP_2': 'suffix_339162_DUP', + 'suffix_328867_DUP_2': 'suffix_339705_DUP', + 'suffix_32906_DUP_2': 'suffix_33904_DUP', + 'suffix_32996_DUP_2': 'suffix_34000_DUP', + 'suffix_332611_DUP_2': 'suffix_343530_DUP', + 'suffix_332820_DEL_2': 'suffix_343748_DEL', + 'suffix_332823_DEL_2': 'suffix_343751_DEL', + 'suffix_333026_DEL_2': 'suffix_343960_DEL', + 'suffix_333055_DUP_2': 'suffix_343990_DUP', + 'suffix_333717_DUP_2': 'suffix_344664_DUP', + 'suffix_334188_DEL_2': 'suffix_345154_DEL', + 'suffix_334626_DEL_2': 'suffix_345601_DEL', + 'suffix_34446_DEL_2': 'suffix_35494_DEL', + 'suffix_35726_DEL_2': 'suffix_36817_DEL', + 'suffix_35799_DEL_2': 'suffix_36894_DEL', + 'suffix_3651_DEL_2': 'suffix_3783_DEL', + 'suffix_3691_DEL_2': 'suffix_3824_DEL', + 'suffix_37781_DUP_2': 'suffix_38929_DUP', + 'suffix_39085_DUP_2': 'suffix_40276_DUP', + 'suffix_42000_DUP_2': 'suffix_43285_DUP', + 'suffix_44539_DUP_2': 'suffix_45913_DUP', + 'suffix_44736_DUP_2': 'suffix_46123_DUP', + 'suffix_4505_DEL_2': 'suffix_4670_DEL', + 'suffix_46580_DUP_2': 'suffix_48024_DUP', + 'suffix_46882_DEL_2': 'suffix_48334_DEL', + 'suffix_4995_DUP_2': 'suffix_5177_DUP', + 'suffix_50873_DEL_2': 'suffix_52450_DEL', + 'suffix_51969_DUP_2': 'suffix_53573_DUP', + 'suffix_5276_DUP_2': 'suffix_5474_DUP', + 'suffix_57692_DUP_2': 'suffix_59439_DUP', + 'suffix_58774_DUP_2': 'suffix_60544_DUP', + 'suffix_58893_DEL_2': 'suffix_60667_DEL', + 'suffix_59352_DEL_2': 'suffix_61133_DEL', + 'suffix_59736_DUP_2': 'suffix_61529_DUP', + 'suffix_59942_DEL_2': 'suffix_61742_DEL', + 'suffix_6172_DUP_2': 'suffix_6414_DUP', + 'suffix_64006_DUP_2': 'suffix_65935_DUP', + 'suffix_65035_DUP_2': 'suffix_67010_DUP', + 'suffix_65938_DUP_2': 'suffix_67937_DUP', + 'suffix_6612_DEL_2': 'suffix_6868_DEL', + 'suffix_66192_DUP_2': 'suffix_68205_DUP', + 'suffix_66563_DEL_2': 'suffix_68588_DEL', + 'suffix_66964_DUP_2': 'suffix_68997_DUP', + 'suffix_70658_DEL_2': 'suffix_72772_DEL', + 'suffix_71031_DEL_2': 'suffix_73155_DEL', + 'suffix_72679_DUP_2': 'suffix_74850_DUP', + 'suffix_72691_DUP_2': 'suffix_74863_DUP', + 'suffix_72959_DUP_2': 'suffix_75144_DUP', + 'suffix_73683_DUP_2': 'suffix_75884_DUP', + 'suffix_76491_DEL_2': 'suffix_78773_DEL', + 'suffix_7806_DUP_2': 'suffix_8099_DUP', + 'suffix_81587_DUP_2': 'suffix_83995_DUP', + 'suffix_8201_DUP_2': 'suffix_8510_DUP', + 'suffix_82039_DUP_2': 'suffix_84458_DUP', + 'suffix_82642_DUP_2': 'suffix_85071_DUP', + 'suffix_837_DUP_2': 'suffix_862_DUP', + 'suffix_84346_DEL_2': 'suffix_86829_DEL', + 'suffix_84718_DUP_2': 'suffix_87225_DUP', + 'suffix_85814_DUP_2': 'suffix_88367_DUP', + 'suffix_86689_DUP_2': 'suffix_89280_DUP', + 'suffix_87664_DUP_2': 'suffix_90289_DUP', + 'suffix_88351_DEL_2': 'suffix_91000_DEL', + 'suffix_89655_DEL_2': 'suffix_92358_DEL', + 'suffix_90187_DEL_2': 'suffix_92909_DEL', + 'suffix_90707_DEL_2': 'suffix_93436_DEL', + 'suffix_90995_DEL_2': 'suffix_93733_DEL', + 'suffix_91277_DEL_2': 'suffix_94018_DEL', + 'suffix_91760_DUP_2': 'suffix_94518_DUP', + 'suffix_91769_DEL_2': 'suffix_94529_DEL', + 'suffix_94202_DEL_2': 'suffix_97018_DEL', + 'prefix_109408_DUP': 'suffix_220966_DUP', + 'prefix_109696_DUP': 'suffix_9533_DUP', + 'prefix_118254_DUP': 'suffix_132626_DUP', + 'prefix_120819_DUP': 'suffix_135755_DUP', + 'prefix_135048_DUP': 'suffix_151788_DUP', + 'prefix_136173_DEL': 'suffix_54386_DEL', + 'prefix_148570_DEL': 'suffix_107482_DEL', + 'prefix_150603_DUP': 'suffix_169662_DUP', + 'prefix_150646_DUP': 'suffix_343955_DUP', + 'prefix_175273_DUP': 'suffix_322926_DUP', + 'prefix_178590_DUP': 'suffix_201523_DUP', + 'prefix_18102_DUP': 'suffix_47262_DUP', + 'prefix_184122_DUP': 'suffix_118450_DUP', + 'prefix_191470_DUP': 'suffix_117959_DUP', + 'prefix_19177_DUP': 'suffix_21314_DUP', + 'prefix_193273_DUP': 'suffix_218537_DUP', + 'prefix_195055_DUP': 'suffix_277123_DUP', + 'prefix_195635_DUP': 'suffix_337960_DUP', + 'prefix_196043_DUP': 'suffix_223818_DUP', + 'prefix_196756_DUP': 'suffix_95953_DUP', + 'prefix_200666_DEL': 'suffix_4258_DEL', + 'prefix_212037_DEL': 'suffix_130018_DEL', + 'prefix_2145_DUP': 'suffix_2610_DUP', + 'prefix_22446_DUP': 'suffix_312822_DUP', + 'prefix_229750_DUP': 'suffix_190694_DUP', + 'prefix_237049_DUP': 'suffix_268708_DUP', + 'prefix_237750_DUP': 'suffix_42133_DUP', + 'prefix_239745_DEL': 'suffix_308552_DEL', + 'prefix_240072_DUP': 'suffix_108801_DUP', + 'prefix_240150_DEL': 'suffix_5244_DEL', + 'prefix_242658_DUP': 'suffix_84380_DUP', + 'prefix_246643_DEL': 'suffix_93326_DEL', + 'prefix_24840_DUP': 'suffix_882_DUP', + 'prefix_252587_DUP': 'suffix_158624_DUP', + 'prefix_254761_DUP': 'suffix_57665_DUP', + 'prefix_255040_DUP': 'suffix_106670_DUP', + 'prefix_261005_DEL': 'suffix_178167_DEL', + 'prefix_263591_DUP': 'suffix_319425_DUP', + 'prefix_263818_DEL': 'suffix_201219_DEL', + 'prefix_285913_DEL': 'suffix_74_DEL', + 'prefix_297312_DUP': 'suffix_337092_DUP', + 'prefix_30992_DEL': 'suffix_331860_DEL', + 'prefix_3358_DEL': 'suffix_71259_DEL', + 'prefix_33658_DUP': 'suffix_268605_DUP', + 'prefix_37118_DUP': 'suffix_248738_DUP', + 'prefix_39859_DUP': 'suffix_122375_DUP', + 'prefix_42735_DUP': 'suffix_155097_DUP', + 'prefix_48349_DUP': 'suffix_150163_DUP', + 'prefix_52511_DUP': 'suffix_234367_DUP', + 'prefix_54966_DEL': 'suffix_62104_DEL', + 'prefix_7639_DUP': 'suffix_307081_DUP', + 'prefix_74657_DUP': 'suffix_83373_DUP', + 'prefix_77210_DUP': 'suffix_241886_DUP', + 'prefix_7935_DUP': 'suffix_124466_DUP', + 'prefix_80037_DUP': 'suffix_332875_DUP', + 'prefix_82121_DUP': 'suffix_269588_DUP', + 'prefix_83942_DUP': 'suffix_322084_DUP', + 'prefix_84486_DUP': 'suffix_94510_DUP', + 'prefix_92979_DEL': 'suffix_103592_DEL', + 'prefix_92985_DEL': 'suffix_148088_DEL', + 'prefix_93168_DEL': 'suffix_184994_DEL', + 'prefix_932_DEL': 'suffix_334231_DEL', + 'prefix_9387_DUP': 'suffix_10911_DUP', + 'prefix_94035_DEL': 'suffix_262718_DEL', + 'prefix_94077_DEL': 'suffix_103402_DEL', + 'prefix_94079_DUP': 'suffix_115968_DUP', + 'prefix_94158_DEL': 'suffix_8045_DEL', + 'prefix_9428_DEL': 'suffix_10965_DEL', + 'prefix_94589_DEL': 'suffix_232242_DEL', + 'prefix_94688_DEL': 'suffix_329994_DEL', + 'prefix_94693_DUP': 'suffix_133009_DUP', + 'prefix_94714_DEL': 'suffix_86407_DEL', + 'prefix_94949_DEL': 'suffix_3640_DEL', + 'prefix_95467_DEL': 'suffix_106269_DEL', + 'prefix_95753_DEL': 'suffix_106601_DEL', + 'prefix_96081_DEL': 'suffix_39335_DEL', + 'prefix_96409_DEL': 'suffix_107290_DEL', + 'prefix_96417_DEL': 'suffix_107299_DEL', + 'prefix_96497_DUP': 'suffix_107390_DUP', + 'prefix_96678_DEL': 'suffix_107578_DEL', + 'prefix_96701_DUP': 'suffix_107599_DUP', + 'prefix_97034_DUP': 'suffix_107976_DUP', + 'prefix_97265_DUP': 'suffix_296214_DUP', + 'prefix_97383_DEL': 'suffix_269445_DEL', + 'prefix_97410_DEL': 'suffix_21697_DEL', + 'prefix_97624_DEL': 'suffix_19282_DEL', + 'prefix_98159_DUP': 'suffix_339672_DUP', + 'prefix_98246_DEL': 'suffix_343159_DEL', + 'prefix_98359_DUP': 'suffix_253054_DUP', + 'prefix_98374_DEL': 'suffix_303813_DEL', + 'prefix_98492_DEL': 'suffix_256422_DEL', + 'prefix_99210_DUP': 'suffix_110452_DUP', + 'prefix_99385_DEL': 'suffix_130483_DEL', + 'prefix_99429_DEL': 'suffix_316828_DEL', + 'prefix_99941_DEL': 'suffix_245189_DEL', + 'prefix_99962_DEL': 'suffix_11652_DEL', + 'prefix_99964_DUP': 'suffix_184974_DUP', + 'prefix_99980_DEL': 'suffix_291827_DEL', + 'prefix_99987_DUP': 'suffix_306785_DUP', + 'suffix_115271_DUP_2': 'suffix_118725_DUP', + 'suffix_147347_DUP_2': 'suffix_151788_DUP', + 'suffix_152903_DUP_2': 'suffix_157559_DUP', + 'suffix_16434_DEL_2': 'suffix_16950_DEL', + 'suffix_17973_DEL_2': 'suffix_18512_DEL', + 'suffix_199564_DUP_2': 'suffix_205825_DUP', + 'suffix_208055_DUP_2': 'suffix_214569_DUP', + 'suffix_208666_DEL_2': 'suffix_215204_DEL', + 'suffix_215449_DUP_2': 'suffix_222247_DUP', + 'suffix_24241_DUP_2': 'suffix_24952_DUP', + 'suffix_282215_DUP_2': 'suffix_291438_DUP', + 'suffix_78257_DEL_2': 'suffix_80567_DEL', + 'prefix_105041_DUP': 'suffix_157907_DUP', + 'prefix_105835_DUP': 'suffix_139575_DUP', + 'prefix_108273_DEL': 'suffix_302962_DEL', + 'prefix_110759_DUP': 'suffix_121735_DUP', + 'prefix_112991_DEL': 'suffix_19457_DEL', + 'prefix_116608_DUP': 'suffix_132952_DUP', + 'prefix_123515_DEL': 'suffix_138768_DEL', + 'prefix_133566_DUP': 'suffix_150246_DUP', + 'prefix_141809_DEL': 'suffix_300943_DEL', + 'prefix_142858_DEL': 'suffix_128588_DEL', + 'prefix_146871_DUP': 'suffix_133106_DUP', + 'prefix_149568_DEL': 'suffix_118635_DEL', + 'prefix_167368_DEL': 'suffix_189133_DEL', + 'prefix_176845_DUP': 'suffix_211875_DUP', + 'prefix_177271_DUP': 'suffix_312822_DUP', + 'prefix_181179_DUP': 'suffix_234798_DUP', + 'prefix_185619_DEL': 'suffix_217950_DEL', + 'prefix_185910_DEL': 'suffix_291085_DEL', + 'prefix_192960_DEL': 'suffix_290358_DEL', + 'prefix_219036_DUP': 'suffix_341481_DUP', + 'prefix_21518_DUP': 'suffix_218342_DUP', + 'prefix_22434_DEL': 'suffix_189133_DEL', + 'prefix_22509_DUP': 'suffix_261218_DUP', + 'prefix_230654_DEL': 'suffix_106308_DEL', + 'prefix_237348_DUP': 'suffix_155009_DUP', + 'prefix_239011_DUP': 'suffix_107426_DUP', + 'prefix_240074_DUP': 'suffix_109378_DUP', + 'prefix_248113_DUP': 'suffix_291388_DUP', + 'prefix_252320_DUP': 'suffix_286050_DUP', + 'prefix_252636_DUP': 'suffix_337479_DUP', + 'prefix_29923_DEL': 'suffix_157806_DEL', + 'prefix_30391_DEL': 'suffix_230937_DEL', + 'prefix_30532_DUP': 'suffix_86319_DUP', + 'prefix_32634_DUP': 'suffix_95910_DUP', + 'prefix_32918_DEL': 'suffix_36811_DEL', + 'prefix_37021_DUP': 'suffix_123374_DUP', + 'prefix_39925_DEL': 'suffix_107137_DEL', + 'prefix_45770_DUP': 'suffix_329746_DUP', + 'prefix_45776_DEL': 'suffix_167381_DEL', + 'prefix_46296_DUP': 'suffix_255601_DUP', + 'prefix_47514_DUP': 'suffix_53429_DUP', + 'prefix_47557_DUP': 'suffix_247215_DUP', + 'prefix_47647_DUP': 'suffix_311519_DUP', + 'prefix_49053_DEL': 'suffix_33001_DEL', + 'prefix_53399_DUP': 'suffix_335231_DUP', + 'prefix_72156_DUP': 'suffix_122944_DUP', + 'prefix_7322_DUP': 'suffix_189132_DUP', + 'prefix_75522_DUP': 'suffix_84376_DUP', + 'prefix_78549_DEL': 'suffix_50090_DEL', + 'prefix_80929_DEL': 'suffix_90588_DEL', + 'prefix_81289_DUP': 'suffix_90965_DUP', + 'prefix_8811_DUP': 'suffix_148957_DUP', + 'prefix_91635_DEL': 'suffix_293519_DEL', + 'prefix_98234_DEL': 'suffix_109388_DEL', + 'prefix_99986_DUP': 'suffix_306740_DUP', + 'suffix_10162_DEL_2': 'suffix_10535_DEL', + 'suffix_119700_DUP_2': 'suffix_123318_DUP', + 'suffix_146970_DUP_2': 'suffix_151397_DUP', + 'suffix_189219_DEL_2': 'suffix_195097_DEL', + 'suffix_200704_DUP_2': 'suffix_206987_DUP', + 'suffix_208083_DEL_2': 'suffix_214598_DEL', + 'suffix_208097_DEL_2': 'suffix_214612_DEL', + 'suffix_212079_DUP_2': 'suffix_218732_DUP', + 'suffix_24668_DUP_2': 'suffix_25398_DUP', + 'suffix_247327_DUP_2': 'suffix_255366_DUP', + 'suffix_248140_DUP_2': 'suffix_256203_DUP', + 'suffix_26120_DUP_2': 'suffix_26896_DUP', + 'suffix_26153_DUP_2': 'suffix_26929_DUP', + 'suffix_28933_DEL_2': 'suffix_29823_DEL', + 'suffix_83854_DUP_2': 'suffix_86319_DUP', + 'prefix_113219_DUP': 'suffix_345434_DUP', + 'prefix_173077_DEL': 'suffix_195402_DEL', + 'prefix_11978_DEL': 'suffix_283553_DEL', + 'prefix_284589_DUP': 'suffix_289594_DUP', + 'prefix_4812_DUP': 'suffix_5671_DUP', + 'suffix_326634_DEL_2': 'suffix_337403_DEL', + 'suffix_333975_DUP_2': 'suffix_344929_DUP', + 'prefix_102605_DEL': 'suffix_265004_DEL', + 'prefix_103754_DEL': 'suffix_308671_DEL', + 'prefix_105176_DEL': 'suffix_617_DEL', + 'prefix_105255_DEL': 'suffix_212161_DEL', + 'prefix_105413_DUP': 'suffix_16872_DUP', + 'prefix_108429_DEL': 'suffix_276230_DEL', + 'prefix_11302_DEL': 'suffix_211819_DEL', + 'prefix_115700_DUP': 'suffix_201547_DUP', + 'prefix_117580_DUP': 'suffix_181916_DUP', + 'prefix_119226_DEL': 'suffix_223666_DEL', + 'prefix_119993_DUP': 'suffix_62592_DUP', + 'prefix_121412_DEL': 'suffix_136441_DEL', + 'prefix_12154_DEL': 'suffix_16787_DEL', + 'prefix_12156_DEL': 'suffix_16888_DEL', + 'prefix_12407_DEL': 'suffix_193951_DEL', + 'prefix_126083_DUP': 'suffix_93868_DUP', + 'prefix_132933_DUP': 'suffix_178504_DUP', + 'prefix_132951_DUP': 'suffix_201246_DUP', + 'prefix_138735_DUP': 'suffix_155567_DUP', + 'prefix_140549_DUP': 'suffix_157451_DUP', + 'prefix_141948_DEL': 'suffix_159338_DEL', + 'prefix_146744_DEL': 'suffix_31511_DEL', + 'prefix_147468_DUP': 'suffix_166040_DUP', + 'prefix_152051_DUP': 'suffix_136974_DUP', + 'prefix_15323_DUP': 'suffix_62805_DUP', + 'prefix_164074_DEL': 'suffix_2290_DEL', + 'prefix_167_DEL': 'suffix_46122_DEL', + 'prefix_168590_DUP': 'suffix_288536_DUP', + 'prefix_168606_DUP': 'suffix_157956_DUP', + 'prefix_171981_DUP': 'suffix_35868_DUP', + 'prefix_173688_DUP': 'suffix_54837_DUP', + 'prefix_174135_DEL': 'suffix_16791_DEL', + 'prefix_175272_DUP': 'suffix_322830_DUP', + 'prefix_175274_DUP': 'suffix_322963_DUP', + 'prefix_183_DEL': 'suffix_106660_DEL', + 'prefix_184814_DEL': 'suffix_195404_DEL', + 'prefix_18485_DEL': 'suffix_29789_DEL', + 'prefix_185069_DEL': 'suffix_132891_DEL', + 'prefix_185206_DEL': 'suffix_1868_DEL', + 'prefix_185338_DEL': 'suffix_290358_DEL', + 'prefix_191249_DEL': 'suffix_150483_DEL', + 'prefix_191318_DEL': 'suffix_20510_DEL', + 'prefix_191655_DUP': 'suffix_327391_DUP', + 'prefix_191946_DEL': 'suffix_216855_DEL', + 'prefix_194528_DUP': 'suffix_220210_DUP', + 'prefix_195246_DUP': 'suffix_293034_DUP', + 'prefix_198967_DUP': 'suffix_244057_DUP', + 'prefix_198991_DUP': 'suffix_244754_DUP', + 'prefix_206418_DUP': 'suffix_234368_DUP', + 'prefix_20_DUP': 'suffix_268896_DUP', + 'prefix_216861_DEL': 'suffix_246030_DEL', + 'prefix_22176_DUP': 'suffix_47721_DUP', + 'prefix_22336_DUP': 'suffix_159422_DUP', + 'prefix_22604_DEL': 'suffix_106759_DEL', + 'prefix_2317_DEL': 'suffix_176841_DEL', + 'prefix_233367_DUP': 'suffix_100639_DUP', + 'prefix_23494_DEL': 'suffix_333136_DEL', + 'prefix_236689_DUP': 'suffix_268271_DUP', + 'prefix_236881_DUP': 'suffix_251030_DUP', + 'prefix_23854_DEL': 'suffix_120387_DEL', + 'prefix_240210_DUP': 'suffix_184198_DUP', + 'prefix_241766_DEL': 'suffix_274245_DEL', + 'prefix_242193_DUP': 'suffix_317066_DUP', + 'prefix_242258_DUP': 'suffix_257305_DUP', + 'prefix_24253_DEL': 'suffix_212318_DEL', + 'prefix_243424_DUP': 'suffix_276101_DUP', + 'prefix_243699_DUP': 'suffix_276379_DUP', + 'prefix_24375_DUP': 'suffix_258696_DUP', + 'prefix_24557_DUP': 'suffix_47694_DUP', + 'prefix_250938_DUP': 'suffix_105404_DUP', + 'prefix_250978_DEL': 'suffix_114000_DEL', + 'prefix_251024_DEL': 'suffix_292979_DEL', + 'prefix_252358_DUP': 'suffix_69555_DUP', + 'prefix_252448_DUP': 'suffix_210958_DUP', + 'prefix_253050_DUP': 'suffix_11042_DUP', + 'prefix_255274_DEL': 'suffix_74443_DEL', + 'prefix_255883_DEL': 'suffix_105498_DEL', + 'prefix_257310_DUP': 'suffix_291433_DUP', + 'prefix_260697_DUP': 'suffix_101357_DUP', + 'prefix_262045_DUP': 'suffix_125836_DUP', + 'prefix_266484_DUP': 'suffix_301782_DUP', + 'prefix_26953_DEL': 'suffix_307435_DEL', + 'prefix_272536_DUP': 'suffix_308571_DUP', + 'prefix_273661_DEL': 'suffix_309764_DEL', + 'prefix_277674_DEL': 'suffix_126886_DEL', + 'prefix_27955_DEL': 'suffix_118689_DEL', + 'prefix_280210_DEL': 'suffix_63525_DEL', + 'prefix_23608_DEL': 'suffix_141141_DEL', + 'prefix_284014_DUP': 'suffix_217043_DUP', + 'prefix_297304_DEL': 'suffix_337079_DEL', + 'prefix_298624_DUP': 'suffix_338928_DUP', + 'prefix_29886_DEL': 'suffix_122545_DEL', + 'prefix_30066_DEL': 'suffix_105693_DEL', + 'prefix_30121_DUP': 'suffix_82107_DUP', + 'prefix_31125_DUP': 'suffix_34864_DUP', + 'prefix_31175_DEL': 'suffix_334478_DEL', + 'prefix_32814_DEL': 'suffix_290329_DEL', + 'prefix_3353_DUP': 'suffix_4034_DUP', + 'prefix_3417_DUP': 'suffix_106659_DUP', + 'prefix_34842_DUP': 'suffix_110095_DUP', + 'prefix_352_DEL': 'suffix_195699_DEL', + 'prefix_36954_DUP': 'suffix_322053_DUP', + 'prefix_36966_DUP': 'suffix_137770_DUP', + 'prefix_37088_DUP': 'suffix_246551_DUP', + 'prefix_37312_DEL': 'suffix_269442_DEL', + 'prefix_37395_DEL': 'suffix_268115_DEL', + 'prefix_37462_DUP': 'suffix_155684_DUP', + 'prefix_37511_DUP': 'suffix_88367_DUP', + 'prefix_38100_DUP': 'suffix_280153_DUP', + 'prefix_41266_DEL': 'suffix_46106_DEL', + 'prefix_44220_DEL': 'suffix_29831_DEL', + 'prefix_45074_DEL': 'suffix_102910_DEL', + 'prefix_4522_DEL': 'suffix_246030_DEL', + 'prefix_463_DUP': 'suffix_291433_DUP', + 'prefix_46468_DUP': 'suffix_273617_DUP', + 'prefix_46530_DEL': 'suffix_258864_DEL', + 'prefix_47637_DUP': 'suffix_309976_DUP', + 'prefix_51641_DEL': 'suffix_58293_DEL', + 'prefix_53677_DUP': 'suffix_34864_DUP', + 'prefix_54008_DUP': 'suffix_285632_DUP', + 'prefix_54421_DUP': 'suffix_47721_DUP', + 'prefix_55677_DUP': 'suffix_62795_DUP', + 'prefix_65173_DUP': 'suffix_290375_DUP', + 'prefix_56142_DUP': 'suffix_147501_DUP', + 'prefix_58202_DUP': 'suffix_116683_DUP', + 'prefix_5881_DUP': 'suffix_230728_DUP', + 'prefix_62733_DUP': 'suffix_228856_DUP', + 'prefix_628_DEL': 'suffix_195672_DEL', + 'prefix_6327_DEL': 'suffix_283625_DEL', + 'prefix_63755_DUP': 'suffix_258929_DUP', + 'prefix_6487_DEL': 'suffix_337048_DEL', + 'prefix_6596_DEL': 'suffix_35678_DEL', + 'prefix_66180_DEL': 'suffix_42022_DEL', + 'prefix_6683_DUP': 'suffix_189168_DUP', + 'prefix_69844_DEL': 'suffix_139225_DEL', + 'prefix_71567_DUP': 'suffix_297455_DUP', + 'prefix_71575_DUP': 'suffix_292697_DUP', + 'prefix_71627_DUP': 'suffix_101357_DUP', + 'prefix_72141_DUP': 'suffix_334049_DUP', + 'prefix_74496_DUP': 'suffix_37213_DUP', + 'prefix_74947_DEL': 'suffix_1698_DEL', + 'prefix_77626_DEL': 'suffix_35767_DEL', + 'prefix_778_DUP': 'suffix_166040_DUP', + 'prefix_78301_DEL': 'suffix_336999_DEL', + 'prefix_78948_DUP': 'suffix_308282_DUP', + 'prefix_79561_DUP': 'suffix_216297_DUP', + 'prefix_79966_DUP': 'suffix_202096_DUP', + 'prefix_8308_DUP': 'suffix_62795_DUP', + 'prefix_83433_DEL': 'suffix_307445_DEL', + 'prefix_83930_DUP': 'suffix_32721_DUP', + 'prefix_86900_DUP': 'suffix_47150_DUP', + 'prefix_89846_DEL': 'suffix_234997_DEL', + 'prefix_92729_DEL': 'suffix_287108_DEL', + 'prefix_93407_DEL': 'suffix_336228_DEL', + 'prefix_93530_DUP': 'suffix_324867_DUP', + 'prefix_93992_DUP': 'suffix_181474_DUP', + 'prefix_95800_DUP': 'suffix_106659_DUP', + 'prefix_96646_DUP': 'suffix_212303_DUP', + 'prefix_97428_DEL': 'suffix_335378_DEL', + 'prefix_9785_DEL': 'suffix_336447_DEL', + 'prefix_9871_DEL': 'suffix_13825_DEL', + 'suffix_106521_DEL_2': 'suffix_109744_DEL', + 'suffix_111044_DUP_2': 'suffix_114370_DUP', + 'suffix_115236_DEL_2': 'suffix_118689_DEL', + 'suffix_118984_DUP_2': 'suffix_122590_DUP', + 'suffix_123189_DEL_2': 'suffix_126886_DEL', + 'suffix_128305_DUP_2': 'suffix_132156_DUP', + 'suffix_128994_DEL_2': 'suffix_132877_DEL', + 'suffix_143611_DEL_2': 'suffix_147926_DEL', + 'suffix_145008_DUP_2': 'suffix_149379_DUP', + 'suffix_145792_DUP_2': 'suffix_150187_DUP', + 'suffix_146879_DUP_2': 'suffix_151303_DUP', + 'suffix_152658_DUP_2': 'suffix_157304_DUP', + 'suffix_15417_DEL_2': 'suffix_15906_DEL', + 'suffix_1560_DEL_2': 'suffix_1619_DEL', + 'suffix_159007_DUP_2': 'suffix_163879_DUP', + 'suffix_16278_DUP_2': 'suffix_16792_DUP', + 'suffix_16287_DUP_2': 'suffix_16801_DUP', + 'suffix_16320_DUP_2': 'suffix_16834_DUP', + 'suffix_179450_DEL_2': 'suffix_184949_DEL', + 'suffix_188093_DEL_2': 'suffix_193931_DEL', + 'suffix_189764_DUP_2': 'suffix_195660_DUP', + 'suffix_189797_DEL_2': 'suffix_195693_DEL', + 'suffix_189963_DEL_2': 'suffix_195862_DEL', + 'suffix_190204_DEL_2': 'suffix_196111_DEL', + 'suffix_190596_DEL_2': 'suffix_196510_DEL', + 'suffix_198910_DUP_2': 'suffix_205165_DUP', + 'suffix_198945_DUP_2': 'suffix_205200_DUP', + 'suffix_205378_DUP_2': 'suffix_211816_DUP', + 'suffix_205382_DEL_2': 'suffix_211819_DEL', + 'suffix_206291_DEL_2': 'suffix_212760_DEL', + 'suffix_208232_DEL_2': 'suffix_214749_DEL', + 'suffix_208718_DUP_2': 'suffix_215260_DUP', + 'suffix_210692_DEL_2': 'suffix_217291_DEL', + 'suffix_210883_DUP_2': 'suffix_217485_DUP', + 'suffix_211544_DUP_2': 'suffix_218173_DUP', + 'suffix_216501_DEL_2': 'suffix_223347_DEL', + 'suffix_216580_DEL_2': 'suffix_223427_DEL', + 'suffix_233162_DEL_2': 'suffix_240724_DEL', + 'suffix_238770_DUP_2': 'suffix_246551_DUP', + 'suffix_239364_DEL_2': 'suffix_247162_DEL', + 'suffix_253024_DUP_2': 'suffix_261270_DUP', + 'suffix_26252_DEL_2': 'suffix_27028_DEL', + 'suffix_26422_DEL_2': 'suffix_27202_DEL', + 'suffix_273614_DEL_2': 'suffix_282591_DEL', + 'suffix_274216_DEL_2': 'suffix_283203_DEL', + 'suffix_285251_DEL_2': 'suffix_294612_DEL', + 'suffix_285526_DUP_2': 'suffix_294904_DUP', + 'suffix_286293_DEL_2': 'suffix_295704_DEL', + 'suffix_28918_DEL_2': 'suffix_29808_DEL', + 'suffix_290688_DEL_2': 'suffix_300251_DEL', + 'suffix_291524_DUP_2': 'suffix_301110_DUP', + 'suffix_291969_DUP_2': 'suffix_301562_DUP', + 'suffix_299400_DUP_2': 'suffix_309256_DUP', + 'suffix_303591_DEL_2': 'suffix_313583_DEL', + 'suffix_30447_DUP_2': 'suffix_31375_DUP', + 'suffix_304499_DEL_2': 'suffix_314515_DEL', + 'suffix_3148_DEL_2': 'suffix_3268_DEL', + 'suffix_324233_DUP_2': 'suffix_334946_DUP', + 'suffix_32624_DUP_2': 'suffix_33615_DUP', + 'suffix_35346_DEL_2': 'suffix_36419_DEL', + 'suffix_35506_DEL_2': 'suffix_36591_DEL', + 'suffix_38424_DUP_2': 'suffix_39596_DUP', + 'suffix_38845_DEL_2': 'suffix_40030_DEL', + 'suffix_39353_DUP_2': 'suffix_40557_DUP', + 'suffix_41150_DUP_2': 'suffix_42404_DUP', + 'suffix_47113_DUP_2': 'suffix_48575_DUP', + 'suffix_48026_DUP_2': 'suffix_49547_DUP', + 'suffix_49445_DUP_2': 'suffix_50997_DUP', + 'suffix_61720_DUP_2': 'suffix_63567_DUP', + 'suffix_62316_DUP_2': 'suffix_64183_DUP', + 'suffix_9094_DUP_2': 'suffix_9431_DUP', + 'suffix_914_DEL_2': 'suffix_925_DEL', + 'suffix_92273_DUP_2': 'suffix_95045_DUP', + 'suffix_93367_DUP_2': 'suffix_96167_DUP', + 'prefix_156333_DUP': 'suffix_176635_DUP', + 'prefix_1723_DUP': 'suffix_176578_DUP', + 'prefix_172949_DEL': 'suffix_195223_DUP', + 'prefix_18_DUP': 'suffix_20_DUP', + 'prefix_192882_DUP': 'suffix_225192_DUP', + 'prefix_230001_DEL': 'suffix_106669_DEL', + 'prefix_241273_DUP': 'suffix_273630_DUP', + 'prefix_260458_DUP': 'suffix_295039_DUP', + 'prefix_26552_DEL': 'suffix_118629_DEL', + 'prefix_268993_DUP': 'suffix_304512_DUP', + 'prefix_34289_DUP': 'suffix_122919_DUP', + 'prefix_42135_DUP': 'suffix_47150_DUP', + 'prefix_447_DUP': 'suffix_195223_DUP', + 'prefix_5926_DUP': 'suffix_277846_DUP', + 'prefix_6582_DEL': 'suffix_212509_DEL', + 'prefix_65866_DEL': 'suffix_282591_DEL', + 'prefix_92476_DUP': 'suffix_110991_DUP', + 'prefix_10078_DEL': 'suffix_195635_DEL', + 'prefix_106132_DEL': 'suffix_195692_DEL', + 'prefix_120338_DUP': 'suffix_135104_DUP', + 'prefix_143062_DUP': 'suffix_160819_DUP', + 'prefix_167275_DUP': 'suffix_308395_DUP', + 'prefix_191895_DEL': 'suffix_216785_DEL', + 'prefix_191896_DEL': 'suffix_216785_DEL', + 'prefix_23369_DEL': 'suffix_296578_DEL', + 'prefix_244929_DUP': 'suffix_277868_DUP', + 'prefix_42634_DUP': 'suffix_166401_DUP', + 'prefix_91448_DUP': 'suffix_148093_DUP', + }, +} +SV_DROPPED_IDS = { + 'cluster_6_last_call_cnv_17479_DUP', 'cluster_1_last_call_cnv_30127_DEL', 'cluster_19_COHORT_cnv_23176_DEL', + 'phase2_DEL_chrX_1149', 'prefix_121357_DUP', 'prefix_73945_DEL', 'phase2_DUP_chr9_1663', 'phase2_CPX_chr20_4', + 'phase2_INV_chr19_1', 'cohort_2911.chr2.final_cleanup_BND_chr2_3805', 'prefix_136453_DEL', 'prefix_283065_DEL', + 'phase2_CPX_chr1_27', 'cohort_2911.chr1.final_cleanup_BND_chr1_3716', 'cohort_2911.chr1.final_cleanup_DEL_chr1_8598', + 'cohort_2911.chr10.final_cleanup_BND_chr10_174', 'cohort_2911.chr10.final_cleanup_DUP_chr10_3475','prefix_100035_DEL', + 'cohort_2911.chr11.final_cleanup_DEL_chr11_885', 'cohort_2911.chr11.final_cleanup_DUP_chr11_2292', + 'cohort_2911.chr12.final_cleanup_DEL_chr12_4527', 'cohort_2911.chr13.final_cleanup_DEL_chr13_2215', 'prefix_100069_DEL', + 'cohort_2911.chr11.final_cleanup_DEL_chr11_8856', 'cohort_2911.chr14.final_cleanup_DEL_chr14_5833', + 'cohort_2911.chr16.final_cleanup_DEL_chr16_3', 'cohort_2911.chr2.final_cleanup_BND_chr2_197', + 'cohort_2911.chr2.final_cleanup_BND_chr2_88', 'cohort_2911.chr4.final_cleanup_BND_chr4_1988', 'phase2_BND_chr4_1503', + 'prefix_187707_DUP', 'phase2_DEL_chrX_1156', 'phase2_INV_chr13_1', 'prefix_104037_DUP', 'prefix_101570_DEL', + 'prefix_107442_DEL', 'prefix_110663_DUP', 'prefix_116075_DEL', 'prefix_116353_DEL', 'prefix_131840_DEL', + 'prefix_158993_DUP', 'prefix_15755_DEL', 'prefix_176502_DUP', 'prefix_186412_DUP', 'prefix_186413_DUP', + 'prefix_232873_DUP', 'prefix_232874_DUP', 'prefix_232877_DUP', 'prefix_28289_DUP', 'prefix_31304_DUP', + 'prefix_51597_DUP', 'prefix_190102_DEL', 'prefix_51608_DUP', 'prefix_221407_DEL', 'prefix_221408_DEL', + 'prefix_255688_DUP', 'prefix_37290_DEL', 'prefix_260433_DEL', 'prefix_72862_DEL', 'prefix_98188_DEL', + 'prefix_90352_DUP', 'prefix_107497_DUP', 'phase2_DEL_chr1_6031', 'prefix_121381_DEL', 'prefix_202121_DUP', + 'cohort_2911.chr2.final_cleanup_BND_chr2_889', 'cohort_2911.chr2.final_cleanup_DUP_chr2_3932', + 'cohort_2911.chr2.final_cleanup_DUP_chr2_7490', 'cohort_2911.chr2.final_cleanup_INS_chr2_1130', + 'cohort_2911.chr22.final_cleanup_DUP_chr22_1511', 'cohort_2911.chr3.final_cleanup_BND_chr3_268', + 'cohort_2911.chr3.final_cleanup_DEL_chr3_10060', 'cohort_2911.chr3.final_cleanup_DUP_chr3_2896', + 'cohort_2911.chr5.final_cleanup_BND_chr5_884', 'cohort_2911.chr5.final_cleanup_DEL_chr5_5718', + 'cohort_2911.chr6.final_cleanup_BND_chr6_2309', 'cohort_2911.chr6.final_cleanup_CPX_chr6_38', + 'cohort_2911.chr7.final_cleanup_BND_chr7_1589', 'cohort_2911.chr7.final_cleanup_BND_chr7_1731', + 'cohort_2911.chr7.final_cleanup_BND_chr7_1751', 'cohort_2911.chr7.final_cleanup_DEL_chr7_9736', + 'cohort_2911.chr7.final_cleanup_DUP_chr7_5963', 'cohort_2911.chr9.final_cleanup_BND_chr9_1042', + 'cohort_2911.chrX.final_cleanup_BND_chrX_236', 'cohort_2911.chrX.final_cleanup_BND_chrX_370', + 'cohort_2911.chrX.final_cleanup_BND_chrX_482', 'cohort_2911.chrX.final_cleanup_BND_chrX_501', + 'cohort_2911.chrX.final_cleanup_BND_chrX_569', 'cohort_2911.chrX.final_cleanup_DEL_chrX_1911', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1912', 'cohort_2911.chrX.final_cleanup_DEL_chrX_1916', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_1958', 'cohort_2911.chrX.final_cleanup_DEL_chrX_4003', + 'cohort_2911.chrX.final_cleanup_DEL_chrX_4372', 'cohort_2911.chrX.final_cleanup_DUP_chrX_2709', + 'cohort_2911.chrX.final_cleanup_DUP_chrX_631', 'phase2_BND_chr11_47', 'phase2_BND_chr11_810', 'phase2_BND_chr1_1064', + 'phase2_BND_chr1_143', 'phase2_BND_chr1_28', 'phase2_BND_chr2_2776', 'phase2_BND_chr3_2305', 'phase2_BND_chr4_1271', + 'phase2_BND_chr7_987', 'phase2_BND_chrX_310', 'phase2_CPX_chr11_67', 'phase2_CPX_chr1_191', 'phase2_CPX_chr20_50', + 'phase2_CPX_chr2_178', 'phase2_CPX_chr3_12', 'phase2_CPX_chrX_49', 'phase2_CTX_chr3_1', 'phase2_DEL_chr11_6013', + 'phase2_DEL_chr11_61', 'phase2_DEL_chr11_64', 'phase2_DEL_chr11_6470', 'phase2_DEL_chr11_70', 'phase2_DEL_chr11_72', + 'phase2_DEL_chr12_1092', 'phase2_DEL_chr12_5745', 'phase2_DEL_chr14_304', 'phase2_DEL_chr14_4456', + 'phase2_DEL_chr16_3480', 'phase2_DEL_chr17_2166', 'phase2_DEL_chr19_3', 'phase2_DEL_chr19_3801', 'phase2_DEL_chr19_4089', + 'phase2_DEL_chr1_10289', 'phase2_DEL_chr1_1859', 'phase2_DEL_chr1_564', 'phase2_DEL_chr1_6442', 'phase2_DEL_chr1_702', + 'phase2_DEL_chr1_825', 'phase2_DEL_chr1_8985', 'phase2_DEL_chr20_3055', 'phase2_DEL_chr2_11622', 'phase2_DEL_chr2_246', + 'phase2_DEL_chr2_6560', 'phase2_DEL_chr2_8969', 'phase2_DEL_chr3_3865', 'phase2_DEL_chr5_3479', 'phase2_DEL_chr5_3841', + 'phase2_DEL_chr5_5773', 'phase2_DEL_chr5_837', 'phase2_DEL_chr5_8866', 'phase2_DEL_chr5_9284', 'phase2_DEL_chr5_940', + 'phase2_DEL_chr6_3310', 'phase2_DEL_chr6_3783', 'phase2_DEL_chr7_3994', 'phase2_DEL_chr7_7308', 'phase2_DEL_chr7_8285', + 'phase2_DEL_chr8_7050', 'phase2_DEL_chrX_1692', 'phase2_DEL_chrX_1711', 'phase2_DEL_chrX_2006', 'phase2_DEL_chrX_2570', + 'phase2_DEL_chrX_4225', 'phase2_DUP_chr11_2031', 'phase2_DUP_chr13_2894', 'phase2_DUP_chr15_1044', 'phase2_DUP_chr16_56', + 'phase2_DUP_chr1_4', 'phase2_DUP_chr1_4156', 'phase2_DUP_chr2_8878', 'phase2_DUP_chr4_2375', 'phase2_DUP_chr4_486', + 'phase2_DUP_chr5_1272', 'phase2_DUP_chr5_6239', 'phase2_DUP_chr6_12', 'phase2_DUP_chr7_235', 'phase2_DUP_chr7_2872', + 'phase2_DUP_chr7_6068', 'phase2_DUP_chr7_6226', 'phase2_DUP_chrX_1009', 'phase2_DUP_chrX_1597', 'phase2_DUP_chrX_3449', + 'phase2_INS_chr11_1083', 'phase2_INS_chr14_995', 'phase2_INS_chr15_462', 'phase2_INS_chr17_887', 'phase2_INS_chr18_112', + 'phase2_INS_chr4_8', 'phase2_INS_chr5_14', 'phase2_INS_chr7_8', 'phase2_INS_chr7_985', 'phase2_INS_chrX_634', + 'prefix_101656_DUP', 'prefix_103772_DUP', 'prefix_104848_DUP', 'prefix_104955_DEL', 'prefix_105106_DEL', + 'prefix_10566_DEL', 'prefix_105762_DEL', 'prefix_105767_DUP', 'prefix_106133_DEL', 'prefix_106350_DUP', + 'prefix_107375_DUP', 'prefix_107449_DEL', 'prefix_107998_DEL', 'prefix_108257_DEL', 'prefix_109835_DEL', + 'prefix_11007_DEL', 'prefix_110195_DUP', 'prefix_112228_DEL', 'prefix_113169_DUP', 'prefix_114542_DEL', + 'prefix_116109_DUP', 'prefix_116684_DEL', 'prefix_11729_DUP', 'prefix_118876_DUP', 'prefix_120303_DEL', + 'prefix_120655_DEL', 'prefix_120715_DUP', 'prefix_120732_DUP', 'prefix_120770_DEL', 'prefix_121356_DEL', + 'prefix_122104_DUP', 'prefix_122891_DEL', 'prefix_123043_DEL', 'prefix_123043_DEL', 'prefix_123043_DEL', + 'prefix_12347_DEL', 'prefix_1243_DUP', 'prefix_124627_DEL', 'prefix_1250_DUP', 'prefix_125972_DEL', 'prefix_127583_DEL', + 'prefix_127702_DEL', 'prefix_13002_DEL', 'prefix_131133_DUP', 'prefix_131619_DUP', 'prefix_131832_DEL', + 'prefix_132292_DUP', 'prefix_132327_DEL', 'prefix_132903_DUP', 'prefix_133314_DEL', 'prefix_133771_DEL', + 'prefix_133965_DEL', 'prefix_133974_DEL', 'prefix_134970_DUP', 'prefix_134975_DUP', 'prefix_134982_DUP', + 'prefix_137070_DEL', 'prefix_137352_DUP', 'prefix_137515_DUP', 'prefix_137824_DEL', 'prefix_141384_DUP', + 'prefix_141673_DEL', 'prefix_142235_DUP', 'prefix_14236_DUP', 'prefix_142_DUP', 'prefix_143105_DUP', 'prefix_144640_DEL', + 'prefix_145910_DEL', 'prefix_146365_DEL', 'prefix_146774_DUP', 'prefix_146964_DUP', 'prefix_147003_DUP', + 'prefix_150029_DUP', 'prefix_15016_DEL', 'prefix_151274_DEL', 'prefix_153605_DEL', 'prefix_155212_DUP', + 'prefix_156514_DUP', 'prefix_158384_DUP', 'prefix_158650_DEL', 'prefix_158664_DEL', 'prefix_160277_DUP', + 'prefix_161748_DEL', 'prefix_162169_DEL', 'prefix_162239_DEL', 'prefix_162270_DEL', 'prefix_162738_DEL', + 'prefix_162951_DUP', 'prefix_163088_DEL', 'prefix_163102_DEL', 'prefix_163246_DEL', 'prefix_163355_DEL', + 'prefix_163854_DEL', 'prefix_164063_DUP', 'prefix_164549_DEL', 'prefix_164556_DEL', 'prefix_164633_DUP', + 'prefix_165128_DUP', 'prefix_168109_DUP', 'prefix_168895_DUP', 'prefix_168897_DEL', 'prefix_169791_DUP', + 'prefix_170374_DUP', 'prefix_170705_DEL', 'prefix_172571_DUP', 'prefix_172946_DEL', 'prefix_17321_DUP', + 'prefix_173394_DEL', 'prefix_173845_DUP', 'prefix_174288_DUP', 'prefix_174291_DUP', 'prefix_174292_DUP', + 'prefix_17444_DEL', 'prefix_17444_DEL', 'prefix_174517_DUP', 'prefix_175277_DUP', 'prefix_177361_DEL', 'prefix_17764_DEL', + 'prefix_177659_DEL', 'prefix_177950_DEL', 'prefix_178465_DEL', 'prefix_178540_DUP', 'prefix_179099_DUP', + 'prefix_180040_DEL', 'prefix_180738_DEL', 'prefix_180819_DEL', 'prefix_181154_DUP', 'prefix_182745_DEL', + 'prefix_182778_DUP', 'prefix_183768_DUP', 'prefix_184019_DEL', 'prefix_184139_DUP', 'prefix_184140_DUP', + 'prefix_184914_DUP', 'prefix_185204_DEL', 'prefix_185484_DUP', 'prefix_185763_DUP', 'prefix_190154_DUP', + 'prefix_191534_DUP', 'prefix_191897_DEL', 'prefix_191899_DEL', 'prefix_192723_DEL', 'prefix_192753_DEL', + 'prefix_192789_DEL', 'prefix_192810_DUP', 'prefix_192814_DEL', 'prefix_192815_DUP', 'prefix_192893_DEL', + 'prefix_193101_DUP', 'prefix_19341_DUP', 'prefix_19359_DUP', 'prefix_19494_DEL', 'prefix_19514_DEL', 'prefix_195407_DUP', + 'prefix_195410_DUP', 'prefix_196172_DUP', 'prefix_196462_DEL', 'prefix_196930_DUP', 'prefix_198944_DEL', + 'prefix_199367_DEL', 'prefix_200326_DEL', 'prefix_201581_DUP', 'prefix_202196_DEL', 'prefix_203149_DUP', + 'prefix_204824_DUP', 'prefix_206046_DUP', 'prefix_20681_DUP', 'prefix_207500_DEL', 'prefix_210512_DEL', + 'prefix_21173_DUP', 'prefix_212735_DUP', 'prefix_215162_DUP', 'prefix_21548_DUP', 'prefix_215773_DEL', 'prefix_2163_DUP', + 'prefix_217013_DEL', 'prefix_217024_DUP', 'prefix_217326_DUP', 'prefix_217327_DUP', 'prefix_21741_DEL', + 'prefix_222096_DUP', 'prefix_22259_DUP', 'prefix_223605_DUP', 'prefix_225452_DUP', 'prefix_22721_DEL', 'prefix_228404_DEL', + 'prefix_22880_DEL', 'prefix_230711_DEL', 'prefix_232671_DUP', 'prefix_232878_DUP', 'prefix_232948_DEL', 'prefix_234881_DEL', + 'prefix_235365_DUP', 'prefix_236208_DUP', 'prefix_237157_DEL', 'prefix_237619_DUP', 'prefix_237805_DEL', + 'prefix_237917_DUP', 'prefix_23794_DEL', 'prefix_238329_DUP', 'prefix_23844_DEL', 'prefix_238575_DEL', 'prefix_238946_DEL', + 'prefix_238949_DEL', 'prefix_239129_DEL', 'prefix_23925_DEL', 'prefix_240064_DUP', 'prefix_240065_DUP', 'prefix_240066_DUP', + 'prefix_240067_DUP', 'prefix_240075_DUP', 'prefix_240077_DUP', 'prefix_240079_DUP', 'prefix_240080_DUP', 'prefix_240082_DUP', + 'prefix_240084_DUP', 'prefix_240087_DUP', 'prefix_240088_DUP', 'prefix_240090_DUP', 'prefix_240091_DUP', 'prefix_240098_DUP', + 'prefix_240102_DUP', 'prefix_240103_DUP', 'prefix_240104_DUP', 'prefix_240107_DUP', 'prefix_240110_DUP', 'prefix_240116_DUP', + 'prefix_240168_DEL', 'prefix_240306_DUP', 'prefix_242363_DUP', 'prefix_242364_DUP', 'prefix_242485_DUP', 'prefix_24445_DEL', + 'prefix_244719_DEL', 'prefix_244925_DUP', 'prefix_245163_DUP', 'prefix_245178_DEL', 'prefix_245675_DUP', 'prefix_245708_DUP', + 'prefix_245785_DUP', 'prefix_246638_DEL', 'prefix_246946_DUP', 'prefix_247619_DUP', 'prefix_247835_DEL', 'prefix_249326_DUP', + 'prefix_250124_DEL', 'prefix_251066_DEL', 'prefix_251476_DUP', 'prefix_251879_DEL', 'prefix_252006_DEL', 'prefix_25227_DUP', + 'prefix_253446_DEL', 'prefix_254368_DEL', 'prefix_254594_DUP', 'prefix_25463_DEL', 'prefix_254654_DUP', 'prefix_255337_DEL', + 'prefix_255338_DEL', 'prefix_255699_DEL', 'prefix_255947_DEL', 'prefix_257636_DEL', 'prefix_260980_DEL', 'prefix_261010_DEL', + 'prefix_262712_DEL', 'prefix_263578_DUP', 'prefix_263619_DEL', 'prefix_263637_DUP', 'prefix_263641_DUP', 'prefix_263645_DUP', + 'prefix_263647_DUP', 'prefix_264278_DUP', 'prefix_26438_DEL', 'prefix_264706_DUP', 'prefix_264786_DEL', 'prefix_264797_DEL', + 'prefix_266509_DEL', 'prefix_266705_DEL', 'prefix_26790_DEL', 'prefix_27178_DEL', 'prefix_272072_DUP', 'prefix_27418_DUP', + 'prefix_277330_DEL', 'prefix_277330_DEL', 'prefix_278495_DUP', 'prefix_281404_DEL', 'prefix_28173_DUP', 'prefix_283002_DUP', + 'prefix_28329_DUP', 'prefix_283636_DUP', 'prefix_284375_DEL', 'prefix_284482_DEL', 'prefix_284499_DEL', 'prefix_284666_DUP', + 'prefix_284724_DEL', 'prefix_28564_DEL', 'prefix_28677_DEL', 'prefix_287359_DEL', 'prefix_28768_DEL', 'prefix_28815_DEL', + 'prefix_28955_DEL', 'prefix_29050_DEL', 'prefix_29052_DEL', 'prefix_290_DUP', 'prefix_29115_DEL', 'prefix_29132_DEL', + 'prefix_291814_DUP', 'prefix_29319_DEL', 'prefix_29359_DEL', 'prefix_29409_DEL', 'prefix_29465_DEL', 'prefix_296068_DEL', + 'prefix_296070_DEL', 'prefix_296071_DEL', 'prefix_296072_DEL', 'prefix_297161_DUP', 'prefix_297163_DUP', 'prefix_297193_DUP', + 'prefix_297203_DUP', 'prefix_297244_DEL', 'prefix_297325_DUP', 'prefix_298246_DEL', 'prefix_299171_DEL', 'prefix_299729_DUP', + 'prefix_30029_DEL', 'prefix_301500_DEL', 'prefix_301521_DEL', 'prefix_303270_DEL', 'prefix_31042_DEL', 'prefix_31146_DUP', + 'prefix_31636_DUP', 'prefix_32411_DEL', 'prefix_32572_DEL', 'prefix_32817_DEL', 'prefix_32938_DEL', 'prefix_32943_DEL', + 'prefix_33036_DEL', 'prefix_33259_DEL', 'prefix_33269_DEL', 'prefix_35408_DUP', 'prefix_36013_DUP', 'prefix_3660_DEL', + 'prefix_37114_DUP', 'prefix_37120_DUP', 'prefix_37284_DEL', 'prefix_37542_DEL', 'prefix_37710_DUP', 'prefix_37751_DEL', + 'prefix_38251_DUP', 'prefix_38525_DEL', 'prefix_40157_DEL', 'prefix_40443_DEL', 'prefix_41131_DEL', 'prefix_42483_DUP', + 'prefix_42564_DUP', 'prefix_43104_DEL', 'prefix_46281_DUP', 'prefix_46357_DEL', 'prefix_46467_DUP', 'prefix_46483_DEL', + 'prefix_46566_DEL', 'prefix_46681_DEL', 'prefix_47968_DUP', 'prefix_49538_DEL', 'prefix_4996_DEL', 'prefix_50673_DUP', + 'prefix_50678_DEL', 'prefix_518_DEL', 'prefix_52282_DUP', 'prefix_5289_DEL', 'prefix_54347_DUP', 'prefix_54569_DUP', + 'prefix_55426_DEL', 'prefix_56509_DUP', 'prefix_56903_DUP', 'prefix_58485_DEL', 'prefix_5932_DUP', 'prefix_6262_DEL', + 'prefix_63135_DUP', 'prefix_6344_DEL', 'prefix_64090_DEL', 'prefix_64813_DUP', 'prefix_65486_DEL', 'prefix_66447_DEL', + 'prefix_67003_DEL', 'prefix_7120_DEL', 'prefix_71708_DEL', 'prefix_72647_DUP', 'prefix_75175_DEL', 'prefix_75176_DEL', + 'prefix_75177_DEL', 'prefix_75180_DEL', 'prefix_75506_DUP', 'prefix_75843_DUP', 'prefix_76309_DUP', 'prefix_7665_DUP', + 'prefix_76693_DEL', 'prefix_7802_DUP', 'prefix_78417_DEL', 'prefix_78921_DUP', 'prefix_78943_DUP', 'prefix_78963_DUP', + 'prefix_78971_DUP', 'prefix_78976_DUP', 'prefix_78978_DUP', 'prefix_78980_DUP', 'prefix_78982_DUP', 'prefix_78988_DUP', + 'prefix_78994_DUP', 'prefix_79001_DUP', 'prefix_7962_DEL', 'prefix_796_DUP', 'prefix_79989_DUP', 'prefix_82092_DEL', + 'prefix_83579_DEL', 'prefix_83600_DUP', 'prefix_83786_DEL', 'prefix_83974_DEL', 'prefix_84263_DUP', 'prefix_84718_DEL', + 'prefix_86064_DUP', 'prefix_86065_DUP', 'prefix_89136_DEL', 'prefix_895_DUP', 'prefix_90181_DUP', 'prefix_91282_DEL', + 'prefix_91373_DUP', 'prefix_91952_DEL', 'prefix_91952_DEL', 'prefix_92457_DUP', 'prefix_92756_DUP', 'prefix_93796_DEL', + 'prefix_94231_DUP', 'prefix_943_DUP', 'prefix_94660_DEL', 'prefix_96293_DEL', 'prefix_97207_DUP', 'prefix_9762_DUP', + 'prefix_98217_DUP', 'prefix_98339_DUP', 'prefix_98578_DUP', 'prefix_98763_DUP', 'prefix_98910_DEL', 'prefix_99458_DEL', + 'suffix_20012_DEL_2', 'suffix_231565_DUP_2', 'suffix_264599_DEL_2', 'suffix_54380_DEL_2', 'suffix_69178_DEL_2', + 'CMG.phase1_CMG_DEL_chr15_535', 'CMG.phase1_CMG_DEL_chr15_748', 'CMG.phase1_CMG_DEL_chr16_1524', + 'CMG.phase1_CMG_DEL_chr16_1529', 'CMG.phase1_CMG_DEL_chr16_1842', 'CMG.phase1_CMG_DEL_chr17_398', + 'CMG.phase1_CMG_DEL_chr17_908', 'CMG.phase1_CMG_DEL_chr1_3107', 'CMG.phase1_CMG_DEL_chr1_4610', + 'CMG.phase1_CMG_DEL_chr2_5843', 'CMG.phase1_CMG_DEL_chr3_168', 'CMG.phase1_CMG_DEL_chr3_174', 'CMG.phase1_CMG_DEL_chr3_175', + 'CMG.phase1_CMG_DEL_chr3_744', 'CMG.phase1_CMG_DEL_chr5_1661', 'CMG.phase1_CMG_DEL_chr5_1936', + 'CMG.phase1_CMG_DEL_chr7_3298', 'CMG.phase1_CMG_DEL_chrX_1942', 'CMG.phase1_CMG_DEL_chrX_521', 'CMG.phase1_CMG_DEL_chrX_524', + 'CMG.phase1_CMG_DEL_chrX_544', 'phase2_DEL_chr16_4247', +} + + +class Command(BaseCommand): + + def handle(self, *args, **options): + num_updated = SavedVariant.objects.filter( + genotypes={}, saved_variant_json__genotypes__isnull=False, + ).exclude(saved_variant_json__genotypes={}).update(genotypes=F('saved_variant_json__genotypes')) + logger.info(f'Updated genotypes for {num_updated} variants') + + variant_ids = SavedVariant.objects.filter( + key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh38, + saved_variant_json__populations__isnull=False, # Omit manual variants + ).values_list('variant_id', flat=True).distinct() + ids_by_dataset_type = { + Sample.DATASET_TYPE_VARIANT_CALLS: [], Sample.DATASET_TYPE_MITO_CALLS: [], Sample.DATASET_TYPE_SV_CALLS: [], + } + for variant_id in variant_ids: + parsed_id = parse_variant_id(variant_id) + if not parsed_id and variant_id.endswith('-'): + # Some AnVIL data was loaded with "-" as the alt allele + parsed_id = parse_variant_id(variant_id[:-1]) + if parsed_id: + is_mito = parsed_id[0].replace('chr', '').startswith('M') + dataset_type = Sample.DATASET_TYPE_MITO_CALLS if is_mito else Sample.DATASET_TYPE_VARIANT_CALLS + else: + dataset_type = Sample.DATASET_TYPE_SV_CALLS + ids_by_dataset_type[dataset_type].append(variant_id) + + no_key_mito = self._set_variant_keys(ids_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS], Sample.DATASET_TYPE_MITO_CALLS) + + no_key_snv_indel = self._set_variant_keys( + ids_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Sample.DATASET_TYPE_VARIANT_CALLS, + ) + if no_key_snv_indel: + self._resolve_missing_variants(no_key_snv_indel, GENOME_VERSION_GRCh38) + + no_keys_svs = self._set_variant_keys( + ids_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS], f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', + ) + no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}') + if no_keys_svs: + self._resolve_reloaded_svs(no_keys_svs) + + variant_ids_37 = SavedVariant.objects.filter( + key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh37, + ).values_list('variant_id', flat=True).distinct() + + no_keys_37 = self._set_variant_keys(variant_ids_37, Sample.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37) + if no_keys_37: + self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37) + + num_updated = SavedVariant.objects.filter(key__isnull=False).exclude(saved_variant_json={}).update( + saved_variant_json={}, + ) + logger.info(f'Cleared saved json for {num_updated} variants with keys') + + logger.info('Done') + + @staticmethod + def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_GRCh38, variant_id_updates=None): + if not variants_ids: + return set() + logger.info(f'Finding keys for {len(variants_ids)} {dataset_type} (GRCh{genome_version}) variant ids') + variant_key_map = get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids) + logger.info(f'Found {len(variant_key_map)} keys') + if not variant_key_map: + return set(variants_ids) + + mapped_variant_ids = variant_key_map.keys() + if variant_id_updates: + reverse_lookup = {v: k for k, v in variant_id_updates.items()} + mapped_variant_ids = [reverse_lookup[vid] for vid in mapped_variant_ids] + saved_variants = SavedVariant.objects.filter( + family__project__genome_version=genome_version, variant_id__in=mapped_variant_ids, + ) + for variant in saved_variants: + if variant_id_updates: + variant.variant_id = variant_id_updates[variant.variant_id] + variant.key = variant_key_map[variant.variant_id] + variant.dataset_type = dataset_type + update_fields = ['key', 'dataset_type'] + if variant_id_updates: + update_fields.append('variant_id') + num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields, batch_size=10000) + logger.info(f'Updated keys for {num_updated} {dataset_type} (GRCh{genome_version}) variants') + + no_key = set(variants_ids) - set(variant_key_map.keys()) + if no_key: + logger.info(f'No key found for {len(no_key)} variants') + return no_key + + @classmethod + def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GENOME_VERSION_GRCh38): + missing_variants = SavedVariant.objects.filter( + variant_id__in=variant_ids, family__project__genome_version=genome_version, + ) + num_missing = missing_variants.count() + missing_with_data_qs = missing_variants.filter(family__individual__sample__is_active=True).distinct() + missing_with_search_data = missing_with_data_qs.values( + 'variant_id', *variant_fields, + ).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id') + return missing_with_search_data, num_missing + + @classmethod + def _resolve_missing_variants(cls, variant_ids, genome_version): + missing_with_search_data, num_missing = cls._query_missing_variants( + variant_ids, ['saved_variant_json__populations__seqr__ac'], genome_version, + ) + num_data= len(missing_with_search_data) + in_backend = [ + f"{var['variant_id']} - {'; '.join(var['family_ids'])}" + for var in missing_with_search_data if var['saved_variant_json__populations__seqr__ac'] + ] + logger.info( + f'{num_missing} variants have no key, {num_missing - num_data} of which have no search data, {num_data - len(in_backend)} of which are absent from the hail backend.' + ) + if in_backend: + logger.info(f'{len(in_backend)} remaining variants: {", ".join(in_backend)}') + + @classmethod + def _resolve_reloaded_svs(cls, variant_ids): + num_known_dropped = len(SV_DROPPED_IDS.intersection(variant_ids)) + missing_with_search_data, num_missing = cls._query_missing_variants( + list(set(variant_ids) - SV_DROPPED_IDS), ['family__individual__sample__sample_type'], + ) + logger.info( + f'{num_missing + num_known_dropped} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data, {num_known_dropped} of which are known to have dropped out of the callset.' + ) + if not missing_with_search_data: + return + + gcnv_id_map = cls._load_gcnv_id_map() + missing_by_sample_type = defaultdict(list) + update_variants_by_sample_type = defaultdict(dict) + for variant in missing_with_search_data: + variant_id = variant['variant_id'] + sample_type = variant['family__individual__sample__sample_type'] + update_id = SV_ID_UPDATE_MAP[sample_type].get(variant_id) + if not update_id and sample_type == 'WES' : + suffix = next((suff for suff in ['_DEL', '_DUP'] if variant_id.endswith(suff)), None) + if suffix: + base_id = gcnv_id_map.get(variant_id.rsplit(suffix)[0]) + if base_id: + update_id = f'{base_id}{suffix}' + if update_id: + update_variants_by_sample_type[sample_type][variant_id] = update_id + elif re.match(r'.*_(DEL|DUP)_\d+', variant_id): + update_variants_by_sample_type[sample_type][variant_id] = variant_id.rsplit('_', 1)[0] + else: + missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" ) + + for sample_type, variant_id_updates in sorted(update_variants_by_sample_type.items()): + logger.info(f'Mapping reloaded SV_{sample_type} IDs to latest version') + failed_mapping = cls._set_variant_keys( + list(variant_id_updates.values()), f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}', + genome_version=GENOME_VERSION_GRCh38, variant_id_updates=variant_id_updates, + ) + if failed_mapping: + logger.info(f'{len(failed_mapping)} variants failed ID mapping: {list(failed_mapping)}') + + for sample_type, variants in missing_by_sample_type.items(): + logger.info(f'{len(variants)} remaining SV {sample_type} variants {", ".join(variants)}') + + @staticmethod + def _load_gcnv_id_map(): + file_content = file_iter(GCNV_CALLSET_PATH) + header = next(file_content).split('\t') + variant_name_idx = header.index('variant_name') + old_id_idx = header.index('any_ovl') + id_map = {} + for raw_row in file_content: + row = raw_row.split('\t') + for old_id in row[old_id_idx].split(';'): + id_map[old_id.strip()] = row[variant_name_idx].strip() + return id_map diff --git a/clickhouse_search/management/tests/__init__.py b/clickhouse_search/management/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py new file mode 100644 index 0000000000..56029887ee --- /dev/null +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -0,0 +1,130 @@ +from django.core.management import call_command +import mock + +from seqr.models import Project, Sample, SavedVariant +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase + +MOCK_GCNV_DATA = [ + b'chr\tstart\tend\tname\tsample\tsample_fix\tsvtype\tGT\tCN\tNP\tQA\tQS\tQSE\tQSS\tploidy\tstrand\tvariant_name\tID\trmsstd\tdefragmented\tvaf\tvac\tlt100_raw_calls\tlt10_highQS_rare_calls\tPASS_SAMPLE\tPASS_FREQ\tPASS_QS\tHIGH_QUALITY\tgenes_any_overlap\tgenes_any_overlap_exonsPerGene\tgenes_any_overlap_totalExons\tgenes_strict_overlap\tgenes_strict_overlap_exonsPerGene\tgenes_strict_overlap_totalExons\tgenes_CG\tgenes_LOF\tgenes_any_overlap_Ensemble_ID\tidentical\tpartial_ovl\tany_ovl\tno_ovl\tsource', + b'chr10\t122599967\t122600191\tCASE_7_Y_cnv_57308\tC1933_010-001_v1_Exome_GCP\t010-001_v1_Exome_C1933\tDEL\t1\t1\t1\t7\t7\t7\t72\t\t-\tprefix_70191\t326536\t0\tFALSE\t0.05066805\t1077\tTRUE\tTRUE\tTRUE\tFALSE\tFALSE\tFALSE\tDMBT1\t1\t1\tNA\t0\t0\tNA\tDMBT1\tENSG00000187908.19\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117;prefix_19107\tFALSE\tround2', +] + + +class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + + SKIP_RESET_VARIANT_JSON = True + MOCK_GCNV_DATA = MOCK_GCNV_DATA + + @classmethod + def setUpTestData(cls): + Project.objects.filter(id=3).update(genome_version='38') + Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) + SavedVariant.objects.update(key=None) + + @mock.patch('seqr.utils.file_utils.subprocess.Popen') + def test_command(self, mock_subprocess): + mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Updated genotypes for 7 variants', None), + ('Finding keys for 1 MITO (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated keys for 1 MITO (GRCh38) variants', None), + ('Finding keys for 1 SNV_INDEL (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None), + ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None), + ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None), + ('Mapping reloaded SV_WES IDs to latest version', None), + ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated keys for 1 SV_WES (GRCh38) variants', None), + ('Mapping reloaded SV_WGS IDs to latest version', None), + ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated keys for 1 SV_WGS (GRCh38) variants', None), + ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), + ('Found 1 keys', None), + ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), + ('No key found for 6 variants', None), + ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), + ('Cleared saved json for 6 variants with keys', None), + ('Done', None), + ]) + + saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'dataset_type', 'genotypes', 'saved_variant_json')) + expected_saved_variants = [ + {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000002_na19675': mock.ANY, 'I000017_na20889': mock.ANY}, 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000018_na21234': {'sampleId': 'NA20885', 'ab': 0.0, 'gq': 99.0, 'numAlt': 1}}, 'saved_variant_json': {}}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_70191_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}}, + {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'dataset_type': 'MITO', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, + {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + ] + self.assertListEqual(saved_variants, expected_saved_variants) + self.assertTrue(all(v['genotypes'] == v['saved_variant_json']['genotypes'] for v in saved_variants[:1] + saved_variants[7:])) + + # Reloading is a no-op + self.reset_logs() + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Updated genotypes for 0 variants', None), + ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), + ('Found 0 keys', None), + ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), + ('Cleared saved json for 0 variants with keys', None), + ('Done', None), + ]) + self.assertListEqual( + list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'dataset_type', 'genotypes', 'saved_variant_json')), + expected_saved_variants, + ) + +class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest): + fixtures = ['users', '1kg_project', 'report_variants'] + + MOCK_GCNV_DATA = MOCK_GCNV_DATA[:1] + + @mock.patch('seqr.utils.file_utils.subprocess.Popen') + def test_command(self, mock_subprocess): + mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Updated genotypes for 7 variants', None), + ('Finding keys for 1 MITO (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), + ('1 remaining variants: M-14783-T-C - 14', None), + ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None), + ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None), + ('Mapping reloaded SV_WGS IDs to latest version', None), + ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None), + ('Found 0 keys', None), + ("1 variants failed ID mapping: ['suffix_19107_DEL']", None), + ('1 remaining SV WES variants prefix_19107_DEL - 12', None), + ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), + ('Found 0 keys', None), + ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), + ('Cleared saved json for 0 variants with keys', None), + ('Done', None), + ]) + diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1e70fd06fe..3012a33e2a 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -15,11 +15,47 @@ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, PATH_FREQ_OVERRIDE_CUTOFF, \ - HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET + HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS -class AnnotationsQuerySet(QuerySet): +class SearchQuerySet(QuerySet): + + @property + def table_basename(self): + return self.model._meta.db_table.rsplit('/', 1)[0] + + @property + def clinvar_field_prefix(self): + return 'clinvar_join' + + @property + def clinvar_fields(self): + return OrderedDict({ + f'{self.clinvar_field_prefix}__{field.name}': (field.db_column or field.name, field) + for field in reversed(self.clinvar_model._meta.local_fields) if field.name != 'key' + }) + + def _clinvar_tuple(self): + return Tuple( + *self.clinvar_fields.keys(), + output_field=NamedTupleField(list(self.clinvar_fields.values()), null_if_empty=True, null_empty_arrays=True), + ) + + def _seqr_pop_fields(self, seqr_populations): + sample_types = [ + sample_type.lower() for sample_type in + ([self.single_sample_type] if self.single_sample_type else list(Sample.SAMPLE_TYPE_LOOKUP.keys())) + ] + seqr_pop_fields = [] + for _, sub_fields in seqr_populations: + seqr_pop_fields += [f"'{sub_fields['ac']}_{sample_type}'" for sample_type in sample_types] + if sub_fields.get('hom'): + seqr_pop_fields += [f"'{sub_fields['hom']}_{sample_type}'" for sample_type in sample_types] + return seqr_pop_fields + + +class AnnotationsQuerySet(SearchQuerySet): TRANSCRIPT_CONSEQUENCE_FIELD = 'sorted_transcript_consequences' SORTED_GENE_CONSEQUENCE_FIELD = 'sorted_gene_consequences' @@ -83,7 +119,7 @@ def annotation_fields(self): return [ field.name for field in self.model._meta.local_fields if (field.db_column or field.name) not in self.annotation_values and field.name not in self.TRANSCRIPT_FIELDS - ] + self.ENTRY_FIELDS + ] @property def prediction_fields(self): @@ -96,6 +132,28 @@ def transcript_fields(self): ] if hasattr(self.model, field)) return set(dict(transcript_field_configs).keys()) + @property + def single_sample_type(self): + return getattr(self.model, 'SV_TYPES', None) + + @property + def entry_field(self): + return next(obj.name for obj in self.model._meta.related_objects if obj.name.startswith('entries')) + + @property + def entry_model(self): + return getattr(self.model, f'{self.entry_field}_set').rel.related_model + + @property + def clinvar_model(self): + if not hasattr(self.entry_model, 'clinvar_join'): + return None + return self.entry_model.clinvar_join.rel.related_model + + @property + def clinvar_field_prefix(self): + return f'{self.entry_field}__clinvar_join' + def subquery_join(self, subquery, join_key='key'): # Add key to intermediate select if not already present @@ -145,8 +203,9 @@ def _get_join_query_values(self, query, alias, conditional_selects): query_select = query.annotation_values for select_func in (conditional_selects or []): query_select.update(select_func(query, prefix=f'{alias}_')) + annotation_fields = query.annotation_fields + self.ENTRY_FIELDS return query.values( - **{f'{alias}_{field}': F(field) for field in query.annotation_fields if field not in query_select}, + **{f'{alias}_{field}': F(field) for field in annotation_fields if field not in query_select}, **{f'{alias}_{field}': value for field, value in query_select.items()}, ) @@ -158,25 +217,55 @@ def search(self, parsed_locus=None, **kwargs): results = self.filter_annotations(results, **parsed_locus, **kwargs) return results - def result_values(self): + def result_values(self, skip_entry_fields=False): override_model_annotations = {'populations', 'pos', 'end'} values = {**self.annotation_values} values.update(self._conditional_selected_transcript_values(self)) - values.update(self._genotype_override_values(self)) + if not skip_entry_fields: + values.update(self._genotype_override_values(self)) initial_values = {k: v for k, v in values.items() if k not in override_model_annotations} - fields = [field for field in self.annotation_fields if field not in values] - if self.has_annotation('clinvar'): - fields.append('clinvar') - + fields = [*self.annotation_fields] if self.has_annotation('familyGenotypes'): - fields = [field for field in fields if field not in self.ENTRY_FIELDS] fields.append('familyGenotypes') + elif not skip_entry_fields: + fields += self.ENTRY_FIELDS + + if self.has_annotation('clinvar'): + fields.append('clinvar') + fields = [field for field in fields if field not in values] return self.values(*fields, **initial_values).annotate( **{k: values[k] for k in override_model_annotations if k in values}, ) + def join_seqr_pop(self): + results = self + seqr_populations = self.model.SEQR_POPULATIONS + if seqr_populations: + seqr_pop_fields = self._seqr_pop_fields(seqr_populations) + results = results.annotate(seqrPop=Tuple( + *[ + DictGet( + 'key', + dict_name=f"{self.table_basename}/gt_stats_dict", + fields=seqr_pop_field, + ) for seqr_pop_field in seqr_pop_fields], + output_field=models.TupleField([models.UInt32Field() for _ in seqr_pop_fields]) + )) + + return results + + def join_clinvar(self, keys): + results = self + if self.clinvar_model: + results = results.annotate(clinvar=self._clinvar_tuple()) + # Due to django modeling, adding a clinvar annotation will add a join to the entries table and then to clinvar + # Manipulating the underlying join removes the entry join entirely + entry_table = f'{self.table_basename}/entries' + results.query.alias_map[f'{self.table_basename}/clinvar'].parent_alias = results.query.alias_map[entry_table].parent_alias + results.query.alias_refcount[entry_table] = 0 + return results def _conditional_selected_transcript_values(self, query, prefix=''): if not hasattr(self.model, self.TRANSCRIPT_CONSEQUENCE_FIELD): @@ -223,7 +312,7 @@ def search_compound_hets(self, primary_q, secondary_q): secondary_q = secondary_q.explode_gene_id(secondary_gene_field) conditional_fields = lambda query, **kwargs: { - field: F(field) for field in [self.SELECTED_GENE_FIELD, 'clinvar', 'family_carriers', 'carriers'] + field: F(field) for field in [self.SELECTED_GENE_FIELD, 'clinvar', 'family_carriers', 'carriers', 'has_hom_alt', 'no_hom_alt_families'] if field in query.query.annotations } @@ -258,7 +347,7 @@ def populations(self): def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): frequencies = freqs or {} - clinvar_override_q = self._clinvar_path_q(pathogenicity) + clinvar_override_q = self._clinvar_path_q(pathogenicity) if self.has_annotation(CLINVAR_KEY) else None for population, pop_filter in frequencies.items(): pop_subfields = self.populations.get(population) @@ -532,7 +621,7 @@ def has_annotation(self, field): return field in self.query.annotations -class EntriesManager(QuerySet): +class EntriesManager(SearchQuerySet): GENOTYPE_LOOKUP = { REF_REF: (0,), REF_ALT: (1,), @@ -559,7 +648,8 @@ class EntriesManager(QuerySet): INHERITANCE_FILTERS = { **INHERITANCE_FILTERS, - COMPOUND_HET: {**INHERITANCE_FILTERS[COMPOUND_HET], AFFECTED: COMP_HET_ALT} + COMPOUND_HET: {**INHERITANCE_FILTERS[COMPOUND_HET], AFFECTED: COMP_HET_ALT}, + COMPOUND_HET_ALLOW_HOM_ALTS: {**INHERITANCE_FILTERS[COMPOUND_HET], AFFECTED: HAS_ALT}, } @property @@ -595,12 +685,8 @@ def genotype_fields(self): }) @property - def clinvar_fields(self): - clinvar_model = self.model.clinvar_join.rel.related_model - return OrderedDict({ - f'clinvar_join__{field.name}': (field.db_column or field.name, field) - for field in reversed(clinvar_model._meta.local_fields) if field.name != 'key' - }) + def clinvar_model(self): + return self.model.clinvar_join.rel.related_model def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): entries = self.filter_intervals(**(parsed_locus or {})) @@ -625,12 +711,18 @@ def _join_annotations(self, entries): if self._has_clinvar(): entries = entries.annotate( clinvar_key=F('clinvar_join__key'), - clinvar=Tuple(*self.clinvar_fields.keys(), output_field=NamedTupleField(list(self.clinvar_fields.values()), null_if_empty=True, null_empty_arrays=True)) + clinvar=self._clinvar_tuple(), ) - seqr_popualtions = self.annotations_model.SEQR_POPULATIONS - if seqr_popualtions: - entries = entries.annotate(seqrPop=self._seqr_pop_expression(seqr_popualtions)) + seqr_populations = self.annotations_model.SEQR_POPULATIONS + if seqr_populations: + seqr_pop_fields = self._seqr_pop_fields(seqr_populations) + entries = entries.annotate(seqrPop=DictGet( + 'key', + dict_name=f"{self.table_basename}/gt_stats_dict", + fields=', '.join(seqr_pop_fields), + output_field=models.TupleField([models.UInt32Field() for _ in seqr_pop_fields]) + )) return entries @@ -640,24 +732,10 @@ def result_values(self, sample_data): return self._search_call_data(entries, sample_data) return self._annotate_calls(entries) - def _seqr_pop_expression(self, seqr_popualtions): - sample_types = [self.single_sample_type.lower()] if self.single_sample_type else ['wes', 'wgs'] - seqr_pop_fields = [] - for _, sub_fields in seqr_popualtions: - seqr_pop_fields += [f"'{sub_fields['ac']}_{sample_type}'" for sample_type in sample_types] - if sub_fields.get('hom'): - seqr_pop_fields += [f"'{sub_fields['hom']}_{sample_type}'" for sample_type in sample_types] - return DictGet( - 'key', - dict_name=f"{self.model._meta.db_table.rsplit('/', 1)[0]}/gt_stats_dict", - fields=', '.join(seqr_pop_fields), - output_field=models.TupleField([models.UInt32Field() for _ in seqr_pop_fields]) - ) - def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') - def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, **kwargs): + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, **kwargs): project_guids = {s['project_guid'] for s in sample_data} project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) @@ -683,7 +761,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if inheritance_mode or individual_genotype_filter or quality_filter: clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), - ) + ) if self._has_clinvar() else None call_q = None multi_sample_type_quality_q = None multi_sample_type_any_affected_q = None @@ -725,7 +803,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit q |= clinvar_override_q entries = entries.filter(q) - return self._annotate_calls(entries, sample_data, annotate_carriers, multi_sample_type_families) + return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, multi_sample_type_families) @staticmethod def _get_family_sample_types(sample_data): @@ -870,10 +948,12 @@ def _annotate_failed_family_samples(cls, entries, family_sample_gt_filters): ) return entries - def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, multi_sample_type_families=None): + def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, multi_sample_type_families=None): carriers_expression = self._carriers_expression(sample_data) if annotate_carriers else None if carriers_expression: entries = entries.annotate(carriers=carriers_expression) + if annotate_hom_alts: + entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) genotype_override_annotations = { f'sample_{col}': ArrayMap( @@ -907,11 +987,19 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, mu else: family_carriers = Cast(Tuple('familyGuids', GroupArray('carriers')), map_field) entries = entries.annotate(family_carriers=family_carriers) + if annotate_hom_alts: + entries = entries.annotate(no_hom_alt_families=ArrayMap( + ArrayFilter(GroupArray(Tuple('family_guid', 'has_hom_alt')), conditions=[{2: (None, 'NOT {field}')}]), + mapped_expression='x.1', + output_field=models.ArrayField(models.StringField()), + )) if any((multi_sample_type_families or {}).values()): entries = self._multi_sample_type_filtered_entries(entries) else: if carriers_expression: fields.append('carriers') + if annotate_hom_alts: + fields.append('has_hom_alt') entries = entries.values( *fields, familyGuids=Array('family_guid'), @@ -986,7 +1074,7 @@ def _multi_sample_type_filtered_entries(cls, entries): failed_samples_expression = If( failed_samples_expression, GroupArrayIntersect(ArrayConcat('failed_family_samples', 'missing_family_samples')), - condition='count() = 1', + condition='count() = 1, ', ) passes_inheritance_expression = ArraySymmetricDifference( 'familyGuids', diff --git a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py new file mode 100644 index 0000000000..d1fea85abc --- /dev/null +++ b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py @@ -0,0 +1,120 @@ +# Generated by Django 4.2.21 on 2025-08-06 00:31 + +import clickhouse_backend.models +import clickhouse_search.backend.fields +import clickhouse_search.backend.engines +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0011_clinvar_materialized_view'), + ] + + operations = [ + migrations.DeleteModel( + name='AnnotationsDiskGcnv', + ), + migrations.DeleteModel( + name='AnnotationsDiskMito', + ), + migrations.DeleteModel( + name='AnnotationsDiskSv', + ), + migrations.DeleteModel( + name='AnnotationsGcnv', + ), + migrations.DeleteModel( + name='AnnotationsMito', + ), + migrations.DeleteModel( + name='AnnotationsSv', + ), + migrations.CreateModel( + name='AnnotationsMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index cd9ea1cc44..9432df87f7 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -57,7 +57,7 @@ def db_for_write(self, model, **hints): return None def allow_migrate(self, db, app_label, model_name=None, **hints): - if f'{app_label}.{model_name}' in self.route_model_names or hints.get('clickhouse'): + if app_label == 'clickhouse_search' or hints.get('clickhouse'): return db == 'clickhouse_write' elif db in {'clickhouse', 'clickhouse_write'}: return False @@ -72,7 +72,28 @@ def __init__(self, name, select='*', order_by=None): self.order_by = order_by -class BaseAnnotations(models.ClickhouseModel): +class FixtureLoadableClickhouseModel(models.ClickhouseModel): + + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered + # and JOIN tables can not be altered this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + + class Meta: + abstract = True + + +class BaseAnnotations(FixtureLoadableClickhouseModel): CHROMOSOME_CHOICES = [(i+1, chrom) for i, chrom in enumerate(CHROMOSOMES)] SEQR_POPULATIONS = [ @@ -281,7 +302,7 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/annotations_disk' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/annotations', primary_key='key', flatten_nested=0) -class BaseAnnotationsMito(BaseAnnotationsMitoSnvIndel): +class AnnotationsMito(BaseAnnotationsMitoSnvIndel): ANNOTATION_CONSTANTS = { 'chrom': 'M', 'liftedOverChrom': 'MT', @@ -337,23 +358,11 @@ class BaseAnnotationsMito(BaseAnnotationsMitoSnvIndel): populations = NamedTupleField(POPULATION_FIELDS) sorted_transcript_consequences = NestedField(BaseAnnotationsMitoSnvIndel.TRANSCRIPTS_FIELDS, db_column='sortedTranscriptConsequences', group_by_key='geneId') - class Meta: - abstract = True - -class AnnotationsMito(BaseAnnotationsMito): - class Meta: db_table = 'GRCh38/MITO/annotations_memory' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) - -class AnnotationsDiskMito(BaseAnnotationsMito): + engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) - class Meta: - db_table = 'GRCh38/MITO/annotations_disk' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) - - -class BaseAnnotationsSv(BaseAnnotationsSvGcnv): +class AnnotationsSv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('gnomad_svs', NamedTupleField([ ('af', models.DecimalField(max_digits=9, decimal_places=5)), @@ -381,22 +390,11 @@ class BaseAnnotationsSv(BaseAnnotationsSvGcnv): sv_type_detail = models.Enum8Field(db_column='svTypeDetail', return_int=False, choices=SV_TYPE_DETAILS, null=True, blank=True) populations = NamedTupleField(POPULATION_FIELDS) - class Meta: - abstract = True - -class AnnotationsSv(BaseAnnotationsSv): - class Meta: db_table = 'GRCh38/SV/annotations_memory' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) + engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) -class AnnotationsDiskSv(BaseAnnotationsSv): - - class Meta: - db_table = 'GRCh38/SV/annotations_disk' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) - -class BaseAnnotationsGcnv(BaseAnnotationsSvGcnv): +class AnnotationsGcnv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('sv_callset', NamedTupleField([ ('ac', models.UInt32Field()), @@ -418,23 +416,11 @@ class BaseAnnotationsGcnv(BaseAnnotationsSvGcnv): num_exon = models.UInt16Field(db_column='numExon') populations = NamedTupleField(POPULATION_FIELDS) - class Meta: - abstract = True - -class AnnotationsGcnv(BaseAnnotationsGcnv): - class Meta: db_table = 'GRCh38/GCNV/annotations_memory' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) - -class AnnotationsDiskGcnv(BaseAnnotationsGcnv): - - class Meta: - db_table = 'GRCh38/GCNV/annotations_disk' - engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) - + engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) -class BaseClinvar(models.ClickhouseModel): +class BaseClinvar(FixtureLoadableClickhouseModel): CLINVAR_ASSERTIONS = [ 'Affects', @@ -493,21 +479,6 @@ class BaseClinvarAllVariants(BaseClinvar): version = models.DateField() variant_id = models.StringField(db_column='variantId', primary_key=True) - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = models.MergeTree( @@ -530,21 +501,6 @@ class Meta(BaseClinvarAllVariants.Meta): class BaseClinvarJoin(BaseClinvar): - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) @@ -566,7 +522,7 @@ class Meta(BaseClinvarJoin.Meta): db_table = 'GRCh38/MITO/clinvar' -class BaseEntries(models.ClickhouseModel): +class BaseEntries(FixtureLoadableClickhouseModel): project_guid = models.StringField(low_cardinality=True) family_guid = models.StringField() xpos = UInt64FieldDeltaCodecField() @@ -575,21 +531,6 @@ class BaseEntries(models.ClickhouseModel): objects = EntriesManager.as_manager() - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = CollapsingMergeTree( @@ -769,27 +710,12 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/transcripts' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/transcripts', primary_key='key', flatten_nested=0) -class BaseKeyLookup(models.ClickhouseModel): +class BaseKeyLookup(FixtureLoadableClickhouseModel): variant_id = models.StringField(db_column='variantId', primary_key=True) class Meta: abstract = True - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class KeyLookupGRCh37SnvIndel(BaseKeyLookup): key = OneToOneField('AnnotationsGRCh37SnvIndel', db_column='key', on_delete=CASCADE) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fe7ef8783a..31ed7fba11 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -2,22 +2,26 @@ from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist -from django.db.models import F, Min +from django.db.models import F, Min, Q from django.db.models.functions import JSONObject +import json from clickhouse_search.backend.fields import NamedTupleField -from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, Tuple +from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ + ArrayDistinct, ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv from reference_data.models import GeneConstraint, Omim from seqr.models import Sample, PhenotypePrioritization from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ - PRIORITIZED_GENE_SORT, COMPOUND_HET, RECESSIVE + PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE +from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from settings import CLICKHOUSE_SERVICE_HOSTNAME logger = SeqrLogger(__name__) +BATCH_SIZE = 10000 TRANSCRIPT_CONSEQUENCES_FIELD = 'sortedTranscriptConsequences' SELECTED_GENE_FIELD = 'selectedGeneId' @@ -42,16 +46,16 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno family_guid = sample_data[0]['family_guid'] if inheritance_mode != COMPOUND_HET: - result_q = _get_search_results_queryset(entry_cls, annotations_cls, search, sample_data) + result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, **search) results += list(result_q[:MAX_VARIANTS + 1]) if has_comp_het: - result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, search, sample_data) + result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, **search) results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, search) + results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, **search) cache_results = get_clickhouse_cache_results(results, sort, family_guid) previous_search_results.update(cache_results) @@ -61,18 +65,15 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno return format_clickhouse_results(cache_results['all_results'][(page-1)*num_results:page*num_results], genome_version) -def _get_search_results_queryset(entry_cls, annotations_cls, search, sample_data): - entries = entry_cls.objects.search(sample_data, **search) - results = annotations_cls.objects.subquery_join(entries).search(**search) +def _get_search_results_queryset(entry_cls, annotations_cls, sample_data, **search_kwargs): + entries = entry_cls.objects.search(sample_data, **search_kwargs) + results = annotations_cls.objects.subquery_join(entries).search(**search_kwargs) return results.result_values() -def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, search): - comp_het_search = {**search, 'inheritance_mode': COMPOUND_HET} - annotations_secondary = search.get('annotations_secondary') +def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: - annotations = search['annotations'] - comp_het_search['annotations'] = { + annotations = { **annotations, **{k: v + annotations[k] if k in annotations else v for k, v in annotations_secondary.items()}, } @@ -90,14 +91,14 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b continue entries = entry_cls.objects.search([ s for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] if s['family_guid'] in families - ], comp_het_search, annotate_carriers=True) - snv_indel_q = annotations_cls.objects.subquery_join(entries).search(**comp_het_search) + ], **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True) + snv_indel_q = annotations_cls.objects.subquery_join(entries).search(**search_kwargs, annotations=annotations) sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search([ s for s in sample_data_by_dataset_type[sv_dataset_type] if s['family_guid'] in families - ], comp_het_search, annotate_carriers=True) + ], **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET, annotate_carriers=True) sv_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][sv_dataset_type] - sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**comp_het_search) + sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations) result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] @@ -105,16 +106,15 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b return results -def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, search, sample_data): +def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): entries = entry_cls.objects.search( - sample_data, **{**search, 'inheritance_mode': COMPOUND_HET}, annotate_carriers=True, + sample_data, **search_kwargs, inheritance_mode=COMPOUND_HET, annotations=annotations, annotate_carriers=True, ) - primary_q = annotations_cls.objects.subquery_join(entries).search(**search) - - annotations_secondary = search.get('annotations_secondary') - secondary_search = {**search, 'annotations': annotations_secondary} if annotations_secondary else search - secondary_q = annotations_cls.objects.subquery_join(entries).search(**secondary_search) + primary_q = annotations_cls.objects.subquery_join(entries).search(annotations=annotations, **search_kwargs) + secondary_q = annotations_cls.objects.subquery_join(entries).search( + annotations=annotations_secondary or annotations, **search_kwargs, + ) return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(sample_data)) @@ -133,6 +133,18 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ ]), ) + if results.has_annotation('primary_has_hom_alt') or results.has_annotation('primary_no_hom_alt_families'): + is_overlapped_del = Q(secondary_svType='DEL', primary_pos__gte=F('secondary_pos'), primary_pos__lte=F('secondary_end')) + if results.has_annotation('primary_has_hom_alt'): + results = results.filter(is_overlapped_del | Q(primary_has_hom_alt=False)) + else: + results = results.annotate(primary_familyGuids=If( + is_overlapped_del, + F('primary_familyGuids'), + ArrayIntersect('primary_familyGuids', 'primary_no_hom_alt_families'), + condition='', + )) + if num_families > 1: results = results.annotate( primary_familyGuids=ArrayIntersect( @@ -159,7 +171,7 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ def _result_as_tuple(results, field_prefix): fields = { name: (name.replace(field_prefix, ''), getattr(col, 'target', col.output_field)) for name, col in results.query.annotations.items() - if name.startswith(field_prefix) and not name.endswith('carriers') + if name.startswith(field_prefix) and not name.endswith('carriers') and not 'hom_alt' in name } return Tuple(*fields.keys(), output_field=NamedTupleField(list(fields.values()))) @@ -174,12 +186,16 @@ def get_clickhouse_cache_results(results, sort, family_guid): return {'all_results': sorted_results, 'total_results': total_results} +def get_transcripts_queryset(genome_version, keys): + return TRANSCRIPTS_CLASS_MAP[genome_version].objects.filter(key__in=keys) + + def format_clickhouse_results(results, genome_version, **kwargs): keys_with_transcripts = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant } transcripts_by_key = dict( - TRANSCRIPTS_CLASS_MAP[genome_version].objects.filter(key__in=keys_with_transcripts).values_list('key', 'transcripts') + get_transcripts_queryset(genome_version, keys_with_transcripts).values_list('key', 'transcripts') ) formatted_results = [] @@ -401,9 +417,7 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): - logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) - +def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): entry_cls = ENTRY_CLASS_MAP[genome_version][data_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][data_type] @@ -424,11 +438,16 @@ def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, else: results = results.filter_variant_ids(variant_ids=[variant_id]) - variants = results.result_values()[:1] - if not variants: + return results.result_values().first() + +def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): + logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) + + variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) + if not variant: raise ObjectDoesNotExist('Variant not present in seqr') - variant = format_clickhouse_results(variants, genome_version)[0] + variant = format_clickhouse_results([variant], genome_version)[0] _add_liftover_genotypes(variant, data_type, variant_id) return variant @@ -451,3 +470,64 @@ def _add_liftover_genotypes(variant, data_type, variant_id): if lifted_entry_data: variant['familyGenotypes'].update(lifted_entry_data[0]['familyGenotypes']) variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) + + +def get_clickhouse_variant_by_id(variant_id, samples, genome_version, dataset_type): + if dataset_type == Sample.DATASET_TYPE_SV_CALLS: + data_types = [ + f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + for sample_type in samples.values_list('sample_type', flat=True).distinct() + ] + else: + data_types = [dataset_type] + for data_type in data_types: + variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) + if variant: + return format_clickhouse_results([variant], genome_version)[0] + return None + + +def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): + sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] + entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( + project_guid=project_guid, family_guid__in=family_guids, key__in=keys, + ) + return { + key: json.loads(json.dumps(genotypes, cls=DjangoJSONEncoderWithSets)) for key, genotypes in + entries.annotate(genotypes=entries.genotype_expression(sample_data)).values_list('key', 'genotypes') + } + + +def get_annotations_queryset(genome_version, dataset_type, keys): + annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] + return annotations_cls.objects.filter(key__in=keys) + + +def get_clickhouse_annotations(genome_version, dataset_type, keys): + qs = get_annotations_queryset(genome_version, dataset_type, keys) + results = qs.join_seqr_pop().join_clinvar(keys).result_values(skip_entry_fields=True) + return format_clickhouse_results(results, genome_version) + + +def get_clickhouse_genes(genome_version, dataset_type, keys): + results = get_annotations_queryset(genome_version, dataset_type, keys) + return results.aggregate( + gene_ids=ArrayDistinct(GroupArrayArray(ArrayMap(results.transcript_field, mapped_expression='x.geneId')), output_field=ArrayField(StringField())), + )['gene_ids'] + + +def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): + results = get_annotations_queryset(genome_version, dataset_type, keys) + return list(results.filter( + **{f'{results.transcript_field}__array_exists': {'geneId': (f"'{gene_id}'",)}}, + ).values_list('key', flat=True)) + + +def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False): + key_lookup_class = KEY_LOOKUP_CLASS_MAP[genome_version][dataset_type] + lookup = {} + fields = ('variant_id', 'key') if not reverse else ('key', 'variant_id') + for i in range(0, len(variants_ids), BATCH_SIZE): + batch = variants_ids[i:i + BATCH_SIZE] + lookup.update(dict(key_lookup_class.objects.filter(variant_id__in=batch).values_list(*fields))) + return lookup diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 25e67328ae..d3c466f762 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -14,51 +14,63 @@ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ - format_cached_variant + MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample +from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper -from seqr.utils.search.utils import query_variants, variant_lookup, sv_variant_lookup +from seqr.utils.search.utils import query_variants, variant_lookup, sv_variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'localhost') -class ClickhouseSearchTests(SearchTestHelper, TestCase): +class ClickhouseSearchTests(DifferentDbTransactionSupportMixin, SearchTestHelper, TestCase): databases = '__all__' - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): super().set_up() - with connections['clickhouse'].cursor() as cursor: + self.mock_redis.get.return_value = None + + @classmethod + def setUpTestData(cls): + with connections['clickhouse_write'].cursor() as cursor: for table_base in ['GRCh38/SNV_INDEL', 'GRCh38/MITO', 'GRCh38/SV', 'GRCh37/SNV_INDEL']: cursor.execute(f'SYSTEM REFRESH VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') + cursor.execute(f'SYSTEM WAIT VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') + cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') Project.objects.update(genome_version='38') - def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', **search_kwargs): + def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', results_model=None, **search_kwargs): + results_model = results_model or self.results_model self.search_model.search.update(search_kwargs or {}) self.search_model.search['qualityFilter'] = quality_filter self.search_model.search['inheritance']['mode'] = inheritance_mode if inheritance_filter is not None: self.search_model.search['inheritance']['filter'] = inheritance_filter - variants, total = query_variants(self.results_model, user=self.user, sort=sort) + variants, total = query_variants(results_model, user=self.user, sort=sort) encoded_variants = self._assert_expected_variants(variants, expected_results) self.assertEqual(total, len(expected_results)) - self._assert_expected_search_cache(encoded_variants, total, cached_variant_fields, sort) + self._assert_expected_search_cache(encoded_variants, total, cached_variant_fields, sort, results_model) + + if gene_counts: + gene_counts_json = get_variant_query_gene_counts(results_model, self.user) + self.assertDictEqual(gene_counts_json, gene_counts) def _assert_expected_variants(self, variants, expected_results): encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) self.assertListEqual(encoded_variants, expected_results) return encoded_variants - def _assert_expected_search_cache(self, variants, total, cached_variant_fields, sort): + def _assert_expected_search_cache(self, variants, total, cached_variant_fields, sort, results_model): cached_variants = [ self._get_cached_variant(variant, (cached_variant_fields[i] if cached_variant_fields else None)) for i, variant in enumerate(variants) ] results_cache = {'all_results': cached_variants, 'total_results': total} - self.assert_cached_results(results_cache, sort=sort) + self.assert_cached_results(results_cache, sort=sort, cache_key=f'search_results__{results_model.guid}__{sort}') @classmethod def _get_cached_variant(cls, variant, cached_variant_fields): @@ -102,13 +114,8 @@ def test_single_family_search(self): **ALL_SNV_INDEL_PASS_FILTERS, ) - mito_gene_counts = { - 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198886': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, - } self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=mito_gene_counts, locus={'rawItems': 'M:1-100000000'}, + [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=MITO_GENE_COUNTS, locus={'rawItems': 'M:1-100000000'}, annotations=None, pathogenicity=None, ) @@ -126,20 +133,44 @@ def test_single_family_search(self): self._assert_expected_search( [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts={**variant_gene_counts, **mito_gene_counts, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, + gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, ) self._set_grch37_search() self._assert_expected_search([GRCH37_VARIANT]) + def test_standard_searches(self): + results_model = self._saved_search_results_model('De Novo/Dominant Restrictive') + self._assert_expected_search( + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + {'selectedTranscript': None}, {}, {}, {}, + ], + ) + + results_model = self._saved_search_results_model('De Novo/Dominant Permissive') + self._assert_expected_search( + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + {'selectedTranscript': None}, {}, {}, {}, + ], + ) + + results_model = self._saved_search_results_model('Recessive Restrictive') + self._assert_expected_search([MITO_VARIANT3], results_model=results_model) + + results_model = self._saved_search_results_model('Recessive Permissive') + self._assert_expected_search([MITO_VARIANT3], results_model=results_model) + + def _saved_search_results_model(self, name): + results_model = VariantSearchResults.objects.create(variant_search=VariantSearch.objects.get(name=name), search_hash=name) + results_model.families.set(self.families.filter(guid='F000002_2')) + return results_model + def test_single_project_search(self): variant_gene_counts = { - 'ENSG00000097046': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, + 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2, 'F000003_3': 1}}, + 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2, 'F000003_3': 1}}, 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198886': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, + **MITO_GENE_COUNTS, } self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, @@ -150,14 +181,14 @@ def test_single_project_search(self): self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000135_na20870']) self._assert_expected_search( [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts={ - 'ENSG00000129562': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000013364': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000079616': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000103495': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000167371': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280789': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280893': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000281348': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000129562': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000013364': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000079616': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000103495': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000167371': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000280789': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000280893': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000281348': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, @@ -166,17 +197,23 @@ def test_single_project_search(self): def test_multi_project_search(self): self._set_multi_project_search() + gene_counts = { + **GENE_COUNTS, + **MITO_GENE_COUNTS, + **GCNV_GENE_COUNTS, + 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2, 'F000011_11': 1}}, + } self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts=GENE_COUNTS, + gene_counts=gene_counts, locus={'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, - MITO_VARIANT2, MITO_VARIANT3], gene_counts={**GENE_COUNTS, **SV_GENE_COUNTS}, + MITO_VARIANT2, MITO_VARIANT3], gene_counts={**gene_counts, **SV_GENE_COUNTS}, ) def test_both_sample_types_search(self): @@ -187,7 +224,7 @@ def test_both_sample_types_search(self): self._add_sample_type_samples('WES', individual__family__guid='F000011_11') self._assert_expected_search( - MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, + MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'}, ) self._set_single_family_search() @@ -199,7 +236,7 @@ def test_both_sample_types_search(self): # Variant 4 is de novo in exome, but inherited in genome in the same parent that has variant 3. self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - inheritance_mode='de_novo', + inheritance_mode='de_novo', locus=None, ) self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) @@ -272,12 +309,12 @@ def test_inheritance_filter(self): self._set_multi_project_search() self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, GCNV_VARIANT3, MITO_VARIANT3], - inheritance_mode=inheritance_mode, + inheritance_mode=inheritance_mode, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4], inheritance_mode=inheritance_mode, + [SV_VARIANT4], inheritance_mode=inheritance_mode, locus=None, ) gt_inheritance_filter = {'genotype': {'I000006_hg00733': 'ref_ref', 'I000005_hg00732': 'has_alt'}} @@ -341,7 +378,10 @@ def test_inheritance_filter(self): **COMP_HET_ALL_PASS_FILTERS, gene_counts={ 'ENSG00000171621': {'total': 2, 'families': {'F000014_14': 2}}, 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, + 'ENSG00000275023': {'total': 3, 'families': {'F000002_2': 3}}, + 'ENSG00000277258': {'total': 3, 'families': {'F000002_2': 3}}, + 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, }, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], @@ -357,10 +397,11 @@ def test_inheritance_filter(self): 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, }, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], **ALL_SNV_INDEL_PASS_FILTERS, + ], **ALL_SNV_INDEL_PASS_FILTERS, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) self._set_single_family_search() @@ -368,7 +409,7 @@ def test_inheritance_filter(self): [VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], + ], locus=None, ) self._reset_search_families() @@ -391,6 +432,7 @@ def test_inheritance_filter(self): 'ENSG00000275023': {'total': 4, 'families': {'F000002_2': 4}}, 'ENSG00000277258': {'total': 4, 'families': {'F000002_2': 4}}, 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, + 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, }, **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], @@ -405,13 +447,28 @@ def test_inheritance_filter(self): self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, gene_counts={ - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, + 'ENSG00000171621': {'total': 2, 'families': {'F000014_14': 2}}, + 'ENSG00000184986': {'total': 1, 'families': {'F000014_14': 1}}, }, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, ], ) + # Test deletion in trans with hom alt snp/indel + for sample in Sample.objects.filter(individual__family_id=14): + sample.pk = None + sample.dataset_type = 'SNV_INDEL' + sample.save() + self._assert_expected_search( + [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4], + inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, + cached_variant_fields=[ + [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], + [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], + {}, {}, + ], + ) + def test_quality_filter(self): quality_filter = {'vcf_filter': 'pass'} self._assert_expected_search( @@ -596,30 +653,30 @@ def test_variant_lookup(self): variant_lookup(self.user, 'suffix_140608_DEL') self.assertEqual(str(cm.exception), 'Variant not present in seqr') -# def test_multi_variant_lookup(self): -# self._test_multi_lookup(VARIANT_ID_SEARCH['variant_ids'], 'SNV_INDEL', [VARIANT1]) -# -# self._test_multi_lookup([['7', 143270172, 'A', 'G']], 'SNV_INDEL', [GRCH37_VARIANT], genome_version='GRCh37') -# -# self._test_multi_lookup([['M', 4429, 'G', 'A'], ['M', 14783, 'T', 'C']], 'MITO', [MITO_VARIANT1, MITO_VARIANT3]) -# -# self._test_multi_lookup( -# ['cohort_2911.chr1.final_cleanup_INS_chr1_160', 'phase2_DEL_chr14_4640'], -# 'SV_WGS', [SV_VARIANT2, SV_VARIANT4], -# ) -# -# self._test_multi_lookup(['suffix_140608_DUP'], 'SV_WES', [NO_GENOTYPE_GCNV_VARIANT]) -# -# def _test_multi_lookup(self, variant_ids, data_type, results, genome_version='GRCh38'): -# body = {'genome_version': genome_version, 'data_type': data_type, 'variant_ids': variant_ids} -# async with self.client.request('POST', '/multi_lookup', json=body) as resp: -# self.assertEqual(resp.status, 200) -# resp_json = resp.json() -# self.assertDictEqual(resp_json, {'results': [ -# {k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}} -# for variant in results -# ]}) -# + def test_get_single_variant(self): + self._set_single_family_search() + variant = get_single_variant(self.results_model.families.first(), VARIANT_IDS[0]) + self._assert_expected_variants([variant], [VARIANT1]) + + with self.assertRaises(InvalidSearchException) as cm: + get_single_variant(self.results_model.families.first(), VARIANT_IDS[1]) + self.assertEqual(str(cm.exception), 'Variant 1-91511686-TCA-G not found') + + variant = get_single_variant(self.results_model.families.first(), 'M-4429-G-A') + self._assert_expected_variants([variant], [MITO_VARIANT1]) + + variant = get_single_variant(self.results_model.families.first(), 'suffix_140608_DUP') + self._assert_expected_variants([variant], [GCNV_VARIANT4]) + + self._set_sv_family_search() + variant = get_single_variant(self.results_model.families.first(), 'phase2_DEL_chr14_4640') + self._assert_expected_variants([variant], [SV_VARIANT4]) + + self._set_grch37_search() + variant = get_single_variant(self.results_model.families.first(), '7-143270172-A-G') + self._assert_expected_variants([variant], [GRCH37_VARIANT]) + + def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} # seqr af filter is ignored for SNV_INDEL @@ -1075,27 +1132,6 @@ def test_in_silico_filter(self): self._set_grch37_search() self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico) -# def test_search_errors(self): -# search_body = get_hail_search_body(sample_data=FAMILY_2_MISSING_SAMPLE_DATA) -# async with self.client.request('POST', '/search', json=search_body) as resp: -# self.assertEqual(resp.status, 400) -# reason = resp.reason -# self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') -# -# search_body = get_hail_search_body(sample_data=MULTI_PROJECT_MISSING_SAMPLE_DATA) -# async with self.client.request('POST', '/search', json=search_body) as resp: -# self.assertEqual(resp.status, 400) -# reason = resp.reason -# self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') -# -# search_body = get_hail_search_body( -# intervals=LOCATION_SEARCH['intervals'] + [['1', 1, 999999999]], omit_data_type='SV_WES', -# ) -# async with self.client.request('POST', '/search', json=search_body) as resp: -# self.assertEqual(resp.status, 400) -# reason = resp.reason -# self.assertEqual(reason, 'Invalid intervals: 1:1-999999999') - def test_sort(self): self._set_single_family_search() self._assert_expected_search( @@ -1186,13 +1222,13 @@ def test_sort(self): self._set_multi_project_search() self._assert_expected_search( [MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, PROJECT_2_VARIANT], - sort='family_guid', + sort='family_guid', locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) # size sort only applies to SVs, so has no impact on other variant self._reset_search_families() self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='size', + [GCNV_VARIANT1, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='size', locus=None, ) self._set_sv_family_search() diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 1d53ca6b1f..2f3758e368 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -268,6 +268,76 @@ for k in {'familyGuids', 'genotypes'}: VARIANT_LOOKUP_VARIANT.pop(k) +PROJECT_4_COMP_HET_VARIANT = { + 'key': 22, + 'variantId': '1-9310123-T-C', + 'chrom': '1', + 'pos': 9310123, + 'ref': 'T', + 'alt': 'C', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': '1', + 'liftedOverPos': 9310113, + 'xpos': 1009310123, + 'rsid': None, + 'familyGuids': ['F000014_14'], + 'genotypes': { + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', 'familyGuid': 'F000014_14', + 'numAlt': 2, 'dp': 45, 'gq': 0, 'ab': 0, 'filters': [], + }, + 'I000019_na21987': { + 'sampleId': 'NA21987', 'sampleType': 'WGS', 'individualGuid': 'I000019_na21987', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'dp': 29, 'gq': 58, 'ab': 0.17241, 'filters': [], + }, + 'I000021_na21654': { + 'sampleId': 'NA21654', 'sampleType': 'WGS', 'individualGuid': 'I000021_na21654', 'familyGuid': 'F000014_14', + 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0, 'filters': [], + } + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 3, 'hom': 1}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, + 'gnomad_exomes': {'af': 0.00004, 'ac': 4, 'an': 90386, 'hom': 0, 'hemi': 0, 'filter_af': 0.0001}, + 'gnomad_genomes': {'af': 0.00004, 'ac': 13, 'an': 264690, 'hom': 13, 'hemi': 0, 'filter_af': 0.0002}, + }, + 'predictions': { + 'cadd': 1.92299, + 'eigen': 2.24799, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': 0.01, + 'splice_ai_consequence': 'No consequence', + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + }, + 'transcripts': {'ENSG00000171621': [{ + 'alphamissense': {'pathogenicity': None}, 'aminoAcids': 'T/I', 'canonical': 1, 'codons': 'aCc/aTc', + 'geneId': 'ENSG00000171621', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', + 'transcriptId': 'ENST00000257261', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'transcriptRank': 0, + 'consequenceTerms': ['missense_variant'], 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', + 'exon': {'index': 1, 'total': 12}, 'intron': None, 'manePlusClinical': None, 'maneSelect': None, + 'refseqTranscriptId': 'NM_001281501.1', 'spliceregion': {'extended_intronic_splice_region_variant': False}, + 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, + }]}, + 'mainTranscriptId': 'ENST00000257261', + 'selectedMainTranscriptId': None, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'CAID': None, +} + CACHED_CONSEQUENCES_BY_KEY = {1: [], 2: [{ 'alphamissensePathogenicity': 0.99779, 'canonical': 1, @@ -379,6 +449,14 @@ 'consequenceTerms': ['non_coding_transcript_exon_variant', 'non_coding_transcript_variant'], 'geneId': 'ENSG00000176227', }], +22: [{ + 'alphamissensePathogenicity': None, + 'canonical': 1, + 'consequenceTerms': ['missense_variant'], + 'extendedIntronicSpliceRegionVariant': False, + 'fiveutrConsequence': None, + 'geneId': 'ENSG00000171621', +}], } def format_cached_variant(variant): @@ -391,8 +469,13 @@ def format_cached_variant(variant): GENE_COUNTS = { 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000011_11': 1}}, - 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 1, 'F000011_11': 1}}, + 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2, 'F000011_11': 1}}, + 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1, 'F000011_11': 1}}, +} +MITO_GENE_COUNTS = { + 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000198886': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, } VARIANT_IDS = ['1-10439-AC-A', '1-91511686-TCA-G'] @@ -418,10 +501,10 @@ def format_cached_variant(variant): NEW_SV_FILTER = {'new_structural_variants': ['NEW']} SV_GENE_COUNTS = { - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000083544': {'total': 1, 'families': {'F000011_11': 1}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, - 'null': {'total': 1, 'families': {'F000011_11': 1}}, + 'ENSG00000171621': {'total': 2, 'families': {'F000014_14': 2}}, + 'ENSG00000083544': {'total': 1, 'families': {'F000014_14': 1}}, + 'ENSG00000184986': {'total': 1, 'families': {'F000014_14': 1}}, + 'null': {'total': 1, 'families': {'F000014_14': 1}}, } GCNV_GENE_COUNTS = { diff --git a/matchmaker/matchmaker_utils.py b/matchmaker/matchmaker_utils.py index 1a12dc9df8..e93ff9057d 100644 --- a/matchmaker/matchmaker_utils.py +++ b/matchmaker/matchmaker_utils.py @@ -204,7 +204,8 @@ def _submission_gene_to_external_genomic_features(submission_gene, individual): _, end = get_chrom_pos(variant.xpos_end) feature['variant']['end'] = end - genotype = variant.saved_variant_json.get('genotypes', {}).get(individual.guid) + genotypes = variant.genotypes or variant.saved_variant_json.get('genotypes', {}) + genotype = genotypes.get(individual.guid) if genotype and genotype.get('numAlt', -1) > 0: feature['zygosity'] = genotype['numAlt'] diff --git a/matchmaker/views/external_api_tests.py b/matchmaker/views/external_api_tests.py index d850204ec7..f486a8e091 100644 --- a/matchmaker/views/external_api_tests.py +++ b/matchmaker/views/external_api_tests.py @@ -12,7 +12,7 @@ @mock.patch('matchmaker.views.external_api.MME_NODES', TEST_MME_NODES) class ExternalAPITest(TestCase): - databases = '__all__' + databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] def _check_mme_authenticated(self, url): diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index abc9540c09..fa0fa3cf15 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -42,12 +42,12 @@ def get_individual_mme_matches(request, submission_guid): Status code and results """ submission = MatchmakerSubmission.objects.get(guid=submission_guid) - check_mme_permissions(submission, request.user) + project = check_mme_permissions(submission, request.user) results = MatchmakerResult.objects.filter(submission=submission) response_json = get_json_for_saved_variants_with_tags( - SavedVariant.objects.filter(family=submission.individual.family), add_details=True) + SavedVariant.objects.filter(family=submission.individual.family), add_details=True, genome_version=project.genome_version) gene_ids = set() for variant in response_json['savedVariantsByGuid'].values(): diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index a067465f02..bd7e123046 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -173,7 +173,6 @@ def __init__(self, json=None): class MatchmakerAPITest(AuthenticationTestCase): - databases = '__all__' fixtures = ['users', '1kg_project', 'reference_data'] def test_get_individual_mme_matches(self): diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 8be28496f3..a6e9ec55f3 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1850,7 +1850,6 @@ "liftedOverGenomeVersion": "", "familyGuids": ["F000001_1", "F000002_2"], "liftedOverPos": "", - "genotypeFilters": "pass", "mainTranscriptId": "ENST00000258436", "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 2.4418633044922146e-05}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.000242306760358614}, "topmed": {"ac": null, "an": null, "af": null}}, "genomeVersion": "37", @@ -1860,7 +1859,6 @@ "rsid": null, "selectedMainTranscriptId": null, "liftedOverChrom": "", - "originalAltAlleles": ["G"], "transcripts": {"ENSG00000135953": [ {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, {"transcriptId": "ENST00000411991", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000411991.1:c.*211_*213delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "558-560"}, @@ -1887,11 +1885,24 @@ "ref": "TC", "alt": "T", "variant_id": "1-248367227-TC-T", + "key": 100, + "dataset_type": "SNV_INDEL", + "genotypes": { + "I000004_hg00731": { + "numAlt": 2 + }, + "I000005_hg00732": { + "numAlt": 1 + } + }, "saved_variant_json": { "clinvar": { - "clinicalSignificance": "", - "alleleId": null, - "variationId": null, + "pathogenicity": "Uncertain_significance", + "alleleId": 12345, + "assertions": null, + "conditions": null, + "conflictingPathogenicities": null, + "submitters": null, "goldStars": null }, "liftedOverGenomeVersion": "38", @@ -1899,7 +1910,6 @@ "F000001_1" ], "liftedOverPos": "", - "genotypeFilters": "pass", "populations": { "callset": { "ac": null, @@ -1962,11 +1972,11 @@ }, "rsid": null, "liftedOverChrom": "", - "originalAltAlleles": [ - "T" - ], "transcripts": { - "ENSG00000135953": [] + "ENSG00000135953": [ + {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000135953"} + ], + "ENSG00000240361": [] }, "chrom": "1", "genotypes": { @@ -1979,6 +1989,7 @@ }, "CAID": "CA1501729" }, + "selected_main_transcript_id": "ENST00000371839", "family": 2 } }, @@ -2000,7 +2011,6 @@ "liftedOverGenomeVersion": "38", "familyGuids": ["F000001_1"], "liftedOverPos": "", - "genotypeFilters": "pass", "mainTranscriptId": "ENST00000505820", "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 8.142526788913136e-06}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "topmed": {"ac": null, "an": null, "af": null}}, "pos": 1560662, @@ -2008,7 +2018,6 @@ "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", - "originalAltAlleles": ["T"], "transcripts": { "ENSG00000197530": [ {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "CELSR1", "majorConsequence": "splice_region_variant", "canonical": "YES", "hgvs": "ENST00000505820.2:c.1067-4G>A", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000505820.2:c.1067-4G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} @@ -2042,7 +2051,6 @@ "liftedOverGenomeVersion": "38", "familyGuids": ["F000001_1"], "liftedOverPos": "", - "genotypeFilters": "pass", "mainTranscriptId": "ENST00000505820", "populations": { "callset": { @@ -2093,7 +2101,6 @@ "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", - "originalAltAlleles": ["G"], "transcripts": { "ENSG00000197530": [ {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} @@ -2123,9 +2130,23 @@ "ref": "TC", "alt": "T", "variant_id": "1-248367227-TC-T", + "key": 100, + "dataset_type": "SNV_INDEL", + "genotypes": { + "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, + "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} + }, "saved_variant_json": { - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "38", "liftedOverPos": "", "genotypeFilters": "pass", + "clinvar": { + "pathogenicity": "Uncertain_significance", + "alleleId": 12345, + "assertions": null, + "conditions": null, + "conflictingPathogenicities": null, + "submitters": null, + "goldStars": null + }, + "liftedOverGenomeVersion": "38", "liftedOverPos": "", "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.00012925741614425127}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 6.505916317651364e-05}, @@ -2135,7 +2156,7 @@ "eigen": null, "revel": null, "sift": null, "cadd": "27.2", "metasvm": "", "mpc": null, "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", "originalAltAlleles": ["T"], + "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", "mainTranscriptId": "ENST00000505820", "transcripts": { "ENSG00000135953": [ {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000228198"} @@ -2188,6 +2209,7 @@ "variantId": "prefix_19107_DEL", "chrom": "1", "genomeVersion": "37", + "populations": {"sv_callset": {}}, "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} }, "family": 12 @@ -2206,6 +2228,11 @@ "ref": "TC", "alt": "T", "variant_id": "1-248367227-TC-T", + "key": 100, + "dataset_type": "SNV_INDEL", + "genotypes": { + "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} + }, "saved_variant_json": { "liftedOverGenomeVersion": "37", "liftedOverPos": "", "genomeVersion": "38", "pos": 248367227, "transcripts": {}, "chrom": "1", "genotypes": { diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index fb5e2e4ad7..9ff4fdd18d 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -36,11 +36,8 @@ "liftedOverChrom": "19", "variantId": "19-1912634-C-T", "chrom": "19", - "mainTranscriptId": "ENST00000371839", "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361"} - ] + "ENSG00000240361": [] }, "CAID": "CA403171634" }, @@ -130,6 +127,83 @@ "family": 2 } }, +{ + "model": "seqr.savedvariant", + "pk": 12, + "fields": { + "guid": "SV0000009_25000014783_r0004_no", + "created_date": "2018-05-15T14:51:56.943Z", + "created_by": null, + "last_modified_date": "2018-05-31T16:36:02.805Z", + "xpos": 25000014783, + "xpos_end": 25000014784, + "ref": "T", + "alt": "C", + "variant_id": "M-14783-T-C", + "key": 100, + "dataset_type": "MITO", + "genotypes": { + "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} + }, + "saved_variant_json": { + "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, + "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, + "rsid": "rs193302982", "familyGuids": ["F000002_2"], "genotypes": { + "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} + }, "populations": {"seqr": {"af": 0.019480518996715546, "ac": 3, "an": 154}, "seqr_heteroplasmy": {"af": 0.006493506487458944, "ac": 1, "an": 154}, + "gnomad_mito": {"af": 0.05534649267792702, "ac": 3118, "an": 56336}, "gnomad_mito_heteroplasmy": {"af": 5.3251918870955706e-05, "ac": 3, "an": 56336, "max_hl": 1.0}, + "helix": {"af": 0.04884607344865799, "ac": 9573, "an": 195983}, "helix_heteroplasmy": {"af": 9.184470400214195e-05, "ac": 18, "an": 195983, "max_hl": 0.962689995765686}}, + "predictions": {"apogee": null, "haplogroup_defining": "Y", "hmtvar": null, "mitotip": null, "mut_taster": null, "sift": null, "mlc": 0.7514}, + "commonLowHeteroplasmy": true, "mitomapPathogenic": true, "clinvar": null, "transcripts": { + "ENSG00000198727": [{"aminoAcids": "L", "canonical": 1, "codons": "Tta/Cta", "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "transcriptId": "ENST00000361789", "isLofNagnag": null, "transcriptRank": 0, "biotype": "protein_coding", "lofFilters": null, "majorConsequence": "synonymous_variant", "consequenceTerms": ["synonymous_variant"]}]}, + "mainTranscriptId": "ENST00000361789", "selectedMainTranscriptId": null + }, + "family": 14 + } +}, { + "model": "seqr.savedvariant", + "pk": 13, + "fields": { + "guid": "SV0000013_prefix_19107_DEL_r00", + "created_date": "2018-06-15T14:51:56.943Z", + "created_by": null, + "last_modified_date": "2018-05-31T16:36:02.805Z", + "xpos": 1249045487, + "xpos_end": 1249045898, + "ref": null, + "alt": null, + "variant_id": "suffix_19107_DEL_013746", + "key": 101, + "dataset_type": "SV_WGS", + "genotypes": { + "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} + }, + "saved_variant_json": { + "liftedOverGenomeVersion": null, + "pos": 249045487, + "end": 249045898, + "xpos": 1249045487, + "predictions": {"strvctvre": 0.374}, + "alt": null, + "numExon": 2, + "genotypeFilters": [], + "ref": null, + "genotypes": { + "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} + }, + "liftedOverPos": null, + "liftedOverChrom": null, + "svType": "DEL", + "variantId": "suffix_19107_DEL", + "chrom": "1", + "endChrom": "1", + "genomeVersion": "37", + "populations": {"sv_callset": {}}, + "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} + }, + "family": 14 + } +}, { "model": "seqr.varianttag", "pk": 1726974, @@ -142,6 +216,30 @@ "variant_tag_type": 4, "search_hash": null } +}, { + "model": "seqr.varianttag", + "pk": 1926974, + "fields": { + "guid": "VT1926974_2103343353_r0006_tes", + "created_date": "2018-05-29T16:32:51.449Z", + "created_by": null, + "last_modified_date": "2018-05-29T16:32:51.449Z", + "saved_variants": [12], + "variant_tag_type": 1, + "search_hash": null + } +}, { + "model": "seqr.varianttag", + "pk": 1926975, + "fields": { + "guid": "VT1926975_2103343353_r0006_tes", + "created_date": "2018-05-29T16:32:51.449Z", + "created_by": null, + "last_modified_date": "2018-05-29T16:32:51.449Z", + "saved_variants": [13], + "variant_tag_type": 4, + "search_hash": null + } }, { "model": "seqr.variantnote", diff --git a/seqr/fixtures/variant_searches.json b/seqr/fixtures/variant_searches.json index 4366d53d92..53529b97e3 100644 --- a/seqr/fixtures/variant_searches.json +++ b/seqr/fixtures/variant_searches.json @@ -23,8 +23,7 @@ "af": 0.001 }, "sv_callset": { - "ac": null, - "af": 0.001 + "ac": 10 }, "gnomad_exomes": { "ac": null, @@ -118,8 +117,7 @@ "af": 0.01 }, "sv_callset": { - "ac": null, - "af": 0.01 + "ac": 100 }, "gnomad_exomes": { "ac": null, @@ -242,8 +240,7 @@ "af": 0.001 }, "sv_callset": { - "ac": null, - "af": 0.001 + "ac": 10 }, "gnomad_exomes": { "ac": null, @@ -360,8 +357,7 @@ "af": 0.01 }, "sv_callset": { - "ac": null, - "af": 0.01 + "ac": 100 }, "gnomad_exomes": { "ac": null, diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index ced91d43e2..899bb6a475 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -7,12 +7,13 @@ import logging import re +from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 from seqr.models import Family, Sample, SavedVariant, Project, Individual from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status -from seqr.utils.search.utils import parse_valid_variant_id +from seqr.utils.search.utils import parse_valid_variant_id, backend_specific_call from seqr.utils.search.hail_search_utils import hail_variant_multi_lookup, search_data_type from seqr.utils.xpos_utils import get_xpos, CHROMOSOMES, MIN_POS, MAX_POS from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS @@ -29,11 +30,16 @@ CLICKHOUSE_MIGRATION_SENTINEL = 'hail_search_to_clickhouse_migration' RUN_FILE_PATH_TEMPLATE = '{data_dir}/{genome_version}/{dataset_type}/runs/{run_version}/{file_name}' SUCCESS_FILE_NAME = '_SUCCESS' +CLICKHOUSE_SUCCESS_FILE_NAME = '_CLICKHOUSE_LOAD_SUCCESS' VALIDATION_ERRORS_FILE_NAME = 'validation_errors.json' ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' RUN_PATH_FIELDS = ['genome_version', 'dataset_type', 'run_version', 'file_name'] DATASET_TYPE_MAP = {'GCNV': Sample.DATASET_TYPE_SV_CALLS} +CLICKHOUSE_DATASET_TYPE_MAP = { + 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES', + Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', +} USER_EMAIL = 'manage_command' MAX_LOOKUP_VARIANTS = 1000 MAX_RELOAD_VARIANTS = 100000 @@ -63,7 +69,8 @@ def add_arguments(self, parser): def handle(self, *args, **options): runs = self._get_runs(**options) - success_run_dirs = [run_dir for run_dir, run_details in runs.items() if SUCCESS_FILE_NAME in run_details['files']] + success_file_name = backend_specific_call(None, SUCCESS_FILE_NAME, CLICKHOUSE_SUCCESS_FILE_NAME) + success_run_dirs = [run_dir for run_dir, run_details in runs.items() if success_file_name in run_details['files']] if success_run_dirs: self._load_success_runs(runs, success_run_dirs) if not success_run_dirs: @@ -106,9 +113,15 @@ def _load_success_runs(self, runs, success_run_dirs): # Reset cached results for all projects, as seqr AFs will have changed for all projects when new data is added reset_cached_search_results(project=None) + backend_specific_call( + lambda *args: None, self._reload_dataset_type_shared_variant_annotations, lambda *args: None, + )(updated_families_by_data_type, updated_variants_by_data_type) + + @classmethod + def _reload_dataset_type_shared_variant_annotations(cls, updated_families_by_data_type, updated_variants_by_data_type): for data_type_key, updated_families in updated_families_by_data_type.items(): try: - self._reload_shared_variant_annotations( + cls._reload_shared_variant_annotations( *data_type_key, updated_variants_by_data_type[data_type_key], exclude_families=updated_families, ) except Exception as e: @@ -200,6 +213,7 @@ def _report_anvil_project_validation_error(cls, project, error_messages): @classmethod def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_version, **kwargs): + clickhouse_dataset_type = CLICKHOUSE_DATASET_TYPE_MAP.get(dataset_type, dataset_type) dataset_type = DATASET_TYPE_MAP.get(dataset_type, dataset_type) logger.info(f'Loading new samples from {genome_version}/{dataset_type}: {run_version}') @@ -235,7 +249,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples, *args = match_and_update_search_samples( + new_samples, updated_samples, *args = match_and_update_search_samples( projects=samples_by_project.keys(), sample_project_tuples=sample_project_tuples, sample_data={'data_source': run_version, 'elasticsearch_index': ';'.join(metadata['callsets'])}, @@ -265,9 +279,10 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON + update_function = backend_specific_call(None, None, cls._update_project_saved_variant_genotypes) updated_variants_by_id = update_projects_saved_variant_json([ - (project.id, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type) + (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() + ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=update_function, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) return search_data_type(dataset_type, sample_type), set(family_project_map.keys()), updated_variants_by_id @@ -403,6 +418,27 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) fields=['filter_flags', 'pop_platform_filters', 'population'], ) + @classmethod + def _update_project_saved_variant_genotypes(cls, project_id, genome_version, user_email, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): + updates = {} + for family_guid in family_guids: + variant_models_by_key = { + v.key: v for v in get_saved_variants(genome_version, project_id, [family_guid], clickhouse_dataset_type=clickhouse_dataset_type) + } + if not variant_models_by_key: + continue + variants = [] + genotypes_by_key = get_clickhouse_genotypes( + project_guid, [family_guid], genome_version, clickhouse_dataset_type, variant_models_by_key.keys(), samples, + ) + for key, genotypes in genotypes_by_key.items(): + variant = variant_models_by_key[key] + variant.genotypes = genotypes + variants.append(variant) + logger.info(f'Reloading genotypes for {len(variants)} {clickhouse_dataset_type} variants in family {family_guid}') + SavedVariant.bulk_update_models(None, variants, ['genotypes']) + updates.update({v.id: v for v in variants}) + return updates @classmethod def _reload_shared_variant_annotations(cls, data_type, genome_version, updated_variants_by_id=None, exclude_families=None, chromosomes=None): diff --git a/seqr/management/commands/lift_project_to_hg38.py b/seqr/management/commands/lift_project_to_hg38.py deleted file mode 100644 index ae182f42ca..0000000000 --- a/seqr/management/commands/lift_project_to_hg38.py +++ /dev/null @@ -1,148 +0,0 @@ -import logging -from collections import defaultdict -from django.core.management.base import BaseCommand, CommandError -from django.db.models.query_utils import Q -from pyliftover.liftover import LiftOver - -from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, SavedVariant, Sample -from seqr.views.utils.json_to_orm_utils import update_model_from_json -from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants -from seqr.views.utils.variant_utils import reset_cached_search_results -from seqr.utils.search.add_data_utils import add_new_es_search_samples -from seqr.utils.search.utils import get_variants_for_variant_ids, get_single_variant -from seqr.utils.xpos_utils import get_xpos, get_chrom_pos - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = 'Transfer projects to the new seqr schema' - - def add_arguments(self, parser): - parser.add_argument('--project', required=True) - parser.add_argument('--es-index', required=True) - - def handle(self, *args, **options): - """transfer project""" - project_arg = options['project'] - project = Project.objects.get(Q(name=project_arg) | Q(guid=project_arg)) - logger.info('Updating project genome version for {}'.format(project.name)) - - # Get expected saved variants - saved_variant_models = SavedVariant.objects.filter(family__project=project) - saved_variant_models_by_guid = {v.guid: v for v in saved_variant_models} - expected_families = {sv.family for sv in saved_variant_models} - - logger.info('Validating es index {}'.format(options['es_index'])) - add_new_es_search_samples({ - 'elasticsearchIndex': options['es_index'], - 'datasetType': Sample.DATASET_TYPE_VARIANT_CALLS, - 'genomeVersion': GENOME_VERSION_GRCh38, - }, project, user=None, expected_families=expected_families) - - # Lift-over saved variants - saved_variants = get_json_for_saved_variants(saved_variant_models, add_details=True) - saved_variants_to_lift, hg37_to_hg38_xpos, lift_failed = _get_variants_to_lift(saved_variants) - - saved_variants_map = defaultdict(list) - variant_ids = [] - for v in saved_variants_to_lift: - if hg37_to_hg38_xpos.get(v['xpos']): - variant_model = saved_variant_models_by_guid[v['variantGuid']] - xpos_38 = hg37_to_hg38_xpos[v['xpos']] - saved_variants_map[(xpos_38, v['ref'], v['alt'])].append(variant_model) - - chrom, pos = get_chrom_pos(xpos_38) - variant_ids.append(f"{'MT' if chrom == 'M' else chrom}-{pos}-{v['ref']}-{v['alt']}") - - es_variants = get_variants_for_variant_ids( - expected_families, variant_ids, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - - missing_variants =_validate_missing_variants(es_variants, saved_variants_map) - - logger.info('Successfully lifted over {} variants'.format(len(es_variants))) - - missing_family_count = _update_saved_variants(es_variants, saved_variants_map) - - logger.info('Successfully updated {} variants'.format(len(es_variants))) - - # Update project and sample data - update_model_from_json(project, {'genome_version': GENOME_VERSION_GRCh38}, None) - - reset_cached_search_results(project) - - logger.info('---Done---') - logger.info('Succesfully lifted over {} variants. Skipped {} failed variants. Family data not updated for {} variants'.format( - len(es_variants), len(missing_variants) + len(lift_failed), missing_family_count)) - -def _get_variants_to_lift(saved_variants): - saved_variants_to_lift = [v for v in saved_variants if v.get('genomeVersion') != GENOME_VERSION_GRCh38] - num_already_lifted = len(saved_variants) - len(saved_variants_to_lift) - if num_already_lifted: - if input('Found {} saved variants already on Hg38. Continue with liftover (y/n)? '.format( - num_already_lifted)) != 'y': - raise CommandError('Error: found {} saved variants already on Hg38'.format(num_already_lifted)) - logger.info('Lifting over {} variants (skipping {} that are already lifted)'.format( - len(saved_variants_to_lift), num_already_lifted)) - - liftover_to_38 = LiftOver('hg19', 'hg38') - hg37_to_hg38_xpos = {} - lift_failed = {} - for v in saved_variants_to_lift: - if not (hg37_to_hg38_xpos.get(v['xpos']) or v['xpos'] in lift_failed): - hg38_coord = liftover_to_38.convert_coordinate('chr{}'.format(v['chrom'].lstrip('chr')), int(v['pos'])) - if hg38_coord and hg38_coord[0]: - hg37_to_hg38_xpos[v['xpos']] = get_xpos(hg38_coord[0][0], hg38_coord[0][1]) - else: - lift_failed[v['xpos']] = v - - if lift_failed: - if input( - 'Unable to lift over the following {} coordinates. Continue with update (y/n)?: {} '.format( - len(lift_failed), ', '.join([ - '{}:{}-{}-{} ({})'.format(v['chrom'], v['pos'], v['ref'], v['alt'], ', '.join(v['familyGuids'])) - for v in lift_failed.values()]))) != 'y': - raise CommandError('Error: unable to lift over {} variants'.format(len(lift_failed))) - - return saved_variants_to_lift, hg37_to_hg38_xpos, lift_failed - -def _validate_missing_variants(es_variants, saved_variants_map): - missing_variants = set(saved_variants_map.keys()) - {(v['xpos'], v['ref'], v['alt']) for v in es_variants} - if missing_variants: - missing_variant_strings = [] - for xpos, ref, alt in missing_variants: - var_id = '{}-{}-{}'.format(xpos, ref, alt) - for v in saved_variants_map[(xpos, ref, alt)]: - tags = v.varianttag_set.all() - notes = v.variantnote_set.all() - missing_variant_strings.append('{var_id} {family_id}: {tags} ({guid})'.format( - var_id=var_id, family_id=v.family.family_id, guid=v.guid, - tags=', '.join([tag.variant_tag_type.name for tag in tags]) if tags else 'No Tags; {}'.format( - '; '.join([note.note for note in notes])) - )) - if input('Unable to find the following {} variants in the index. Continue with update (y/n)?:\n{}\n'.format( - len(missing_variants), '\n'.join(missing_variant_strings))) != 'y': - raise CommandError('Error: unable to find {} lifted-over variants'.format(len(missing_variants))) - return missing_variants - -def _update_saved_variants(es_variants, saved_variants_map): - missing_family_count = 0 - for var in es_variants: - saved_variant_models = saved_variants_map[(var['xpos'], var['ref'], var['alt'])] - missing_saved_variants = [v for v in saved_variant_models if v.family.guid not in var['familyGuids']] - if missing_saved_variants: - variant_id = '{}-{}-{}-{}'.format(var['chrom'], var['pos'], var['ref'], var['alt']) - if input(('Variant {} (hg37: {}) not find for expected families {}. Continue with update (y/n)? '.format( - variant_id, missing_saved_variants[0].xpos, - ', '.join(['{} ({})'.format(v.family.guid, v.guid) for v in missing_saved_variants])) - )) == 'y': - var = get_single_variant([v.family for v in saved_variant_models], variant_id, return_all_queried_families=True) - missing_family_count += len(missing_saved_variants) - else: - raise CommandError('Error: unable to find family data for lifted over variant') - for saved_variant in saved_variant_models: - saved_variant.xpos = var['xpos'] - saved_variant.saved_variant_json = var - saved_variant.save() - return missing_family_count \ No newline at end of file diff --git a/seqr/management/commands/lift_variant_to_hg38.py b/seqr/management/commands/lift_variant_to_hg38.py deleted file mode 100644 index 1468aa0566..0000000000 --- a/seqr/management/commands/lift_variant_to_hg38.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging -from django.core.management.base import BaseCommand, CommandError - -from seqr.models import SavedVariant -from seqr.utils.search.utils import get_single_variant - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = 'Transfer projects to the new seqr schema' - - def add_arguments(self, parser): - parser.add_argument('saved_variant_guid') - parser.add_argument('variant_id') - - def handle(self, *args, **options): - """transfer project""" - saved_variant_guid = options['saved_variant_guid'] - variant_id = options['variant_id'] - - saved_variant = SavedVariant.objects.get(guid=saved_variant_guid) - - if input('Are you sure you want to update {}-{}-{} to {} (y/n)? '.format( - saved_variant.xpos, saved_variant.ref, saved_variant.alt, variant_id)) != 'y': - raise CommandError('Error: user did not confirm') - - es_variant = get_single_variant([saved_variant.family], variant_id, return_all_queried_families=True) - - saved_variant.xpos = es_variant['xpos'] - saved_variant.ref = es_variant['ref'] - saved_variant.alt = es_variant['alt'] - saved_variant.saved_variant_json = es_variant - saved_variant.save() - - logger.info('---Done---') diff --git a/seqr/management/commands/reload_saved_variant_annotations.py b/seqr/management/commands/reload_saved_variant_annotations.py index 53bdc9a506..a2481e1731 100644 --- a/seqr/management/commands/reload_saved_variant_annotations.py +++ b/seqr/management/commands/reload_saved_variant_annotations.py @@ -1,14 +1,19 @@ -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from reference_data.models import GENOME_VERSION_LOOKUP from seqr.models import Sample from seqr.management.commands.check_for_new_samples_from_pipeline import reload_shared_variant_annotations from seqr.utils.search.hail_search_utils import search_data_type +from seqr.utils.search.utils import backend_specific_call DATA_TYPE_CHOICES = { search_data_type(dt, st) for dt in Sample.DATASET_TYPE_LOOKUP for st in [Sample.SAMPLE_TYPE_WGS, Sample.SAMPLE_TYPE_WES] } +def _clickhouse_error(): + raise CommandError('Reloading variant annotations is not supported in clickhouse') + + class Command(BaseCommand): help = 'Reload shared variant annotations for all saved variants' @@ -18,4 +23,5 @@ def add_arguments(self, parser): parser.add_argument('chromosomes', nargs='*', help='Chromosome(s) to reload. If not specified, defaults to all chromosomes.') def handle(self, *args, **options): + backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() reload_shared_variant_annotations(options['data_type'], options['genome_version'], chromosomes=options['chromosomes']) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index eea208cf32..5b38509d02 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -1,12 +1,17 @@ import logging -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.db.models.query_utils import Q from seqr.models import Project +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.variant_utils import update_projects_saved_variant_json logger = logging.getLogger(__name__) +def _clickhouse_error(): + raise CommandError('Reloading variants is not supported in clickhouse') + + class Command(BaseCommand): help = 'Transfer projects to the new seqr schema' @@ -15,7 +20,7 @@ def add_arguments(self, parser): parser.add_argument('--family-guid', help='optional family to reload variants for') def handle(self, *args, **options): - """transfer project""" + backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() projects_to_process = options['projects'] family_guid = options['family_guid'] @@ -27,6 +32,6 @@ def handle(self, *args, **options): logging.info("Processing all %s projects" % len(projects)) family_ids = [family_guid] if family_guid else None - project_list = [(*project, family_ids) for project in projects.values_list('id', 'name', 'genome_version')] + project_list = [(*project, family_ids) for project in projects.values_list('id', 'guid', 'name', 'genome_version')] update_projects_saved_variant_json(project_list, user_email='manage_command') logger.info("Done") diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 9c7b23501f..53294bd9c5 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -176,16 +176,23 @@ b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/auto__2023-08-09/', b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh37/SNV_INDEL/runs/manual__2023-11-02/', b'gs://seqr-hail-search-data/v3.1/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/README.txt', b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', + b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2025-03-14/', + b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2025-03-14/_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', ] OPENED_RUN_JSON_FILES = [{ 'callsets': ['1kg.vcf.gz', 'new_samples.vcf.gz'], @@ -338,7 +345,6 @@ def mock_opened_file(index): @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda *args: True) -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) @mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_URL', 'http://testairtable') @mock.patch('seqr.utils.communication_utils.BASE_URL', SEQR_URL) @mock.patch('seqr.utils.search.add_data_utils.BASE_URL', SEQR_URL) @@ -369,7 +375,7 @@ def set_up(self): self.addCleanup(patcher.stop) Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_logs=None, reload_calls=None, num_runs=5): + def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_logs=None, reload_calls=None, num_runs=5, has_reload_calls=False): self._set_loading_files() self.reset_logs() responses.calls.reset() @@ -409,9 +415,9 @@ def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_ self.mock_redis.return_value.delete.assert_called_with('search_results__*', 'variant_lookup_results__*') # Test reload saved variants - num_airtable_loading_calls, num_airtable_validation_calls = self._assert_expected_airtable_calls(bool(reload_calls), single_call) + num_airtable_loading_calls, num_airtable_validation_calls = self._assert_expected_airtable_calls(has_reload_calls, single_call) if not reload_calls: - self.assertEqual(len(responses.calls), num_airtable_validation_calls) + self.assertEqual(len(responses.calls), num_airtable_loading_calls + num_airtable_validation_calls) return reload_annotation_calls = [] if single_call else [ @@ -446,24 +452,6 @@ def _additional_loading_logs(self, data_type, version): def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): mock_max_reload_variants.__lt__.return_value = False - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [{'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}], - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - # Test errors self._set_empty_loading_files() with self.assertRaises(CommandError) as ce: @@ -523,14 +511,6 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ] - reload_project_variants_logs = [ - ('Reloading saved variants in 2 projects', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] update_sample_qc_logs = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], @@ -567,7 +547,7 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('update 2 Familys', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), - ] + update_sample_qc_logs + reload_project_variants_logs, + ] + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, 'GRCh38/MITO': [ ('Loading 2 WGS MITO samples in 1 projects', None) ], @@ -578,13 +558,7 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('update 3 Samples', {'dbUpdate': mock.ANY}), ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), - (mock.ANY, {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Error reloading variants in Test Reprocessed Project: Bad Request', {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Reload Summary: ', None), - ('Skipped the following 1 project with no saved variants: 1kg project nåme with uniçøde', None), - ('1 failed projects', None), - (' Test Reprocessed Project: Bad Request', None), - ], + ] + self.RELOAD_SV_VARIANT_LOGS, }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', @@ -676,26 +650,7 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ) self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'Rncc') - # Test SavedVariant model updated - updated_variants = SavedVariant.objects.filter(saved_variant_json__updated_field='updated_value') - self.assertEqual(len(updated_variants), 2) - self.assertSetEqual( - {v.guid for v in updated_variants}, - {'SV0000006_1248367227_r0004_non', 'SV0000002_1248367227_r0390_100'} - ) - reloaded_variant = next(v for v in updated_variants if v.guid == 'SV0000006_1248367227_r0004_non') - annotation_updated_variant = next(v for v in updated_variants if v.guid == 'SV0000002_1248367227_r0390_100') - self.assertEqual(len(reloaded_variant.saved_variant_json), 3) - self.assertListEqual(reloaded_variant.saved_variant_json['familyGuids'], ['F000014_14']) - self.assertEqual(len(annotation_updated_variant.saved_variant_json), 19) - self.assertListEqual(annotation_updated_variant.saved_variant_json['familyGuids'], ['F000001_1']) - - annotation_updated_json = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json), 18) - self.assertEqual(annotation_updated_json['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json['rsid'], 'rs123') - self.assertEqual(annotation_updated_json['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json['genotypes']), 3) + self._assert_saved_variant_models_updated() # Test notifications self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) @@ -794,9 +749,10 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): self._test_call(num_runs=2, reload_annotations_logs=reload_fetched_annotations_logs + [ ('Skipped reloading all 2 saved variant annotations for SNV_INDEL GRCh38', None), ], reload_calls=reload_snv_indel_calls, run_loading_logs={ - 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_qc_logs + reload_project_variants_logs, + 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, }) +@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): fixtures = ['users', '1kg_project'] @@ -810,6 +766,22 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): LIST_FILE_LOGS = [] AIRTABLE_LOGS = [] VALIDATION_LOGS = [] + RELOAD_PROJECT_VARIANT_LOGS = [ + ('Reloading saved variants in 2 projects', None), + ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 1 families for project Non-Analyst Project', None), + ('Reload Summary: ', None), + (' Non-Analyst Project: Updated 1 variants', None), + ] + RELOAD_SV_VARIANT_LOGS = [ + (mock.ANY, {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), + ('Error reloading variants in Test Reprocessed Project: Bad Request', {'severity': 'ERROR','@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), + ('Reload Summary: ', None), + ('Skipped the following 1 project with no saved variants: 1kg project nåme with uniçøde', None), + ('1 failed projects', None), + (' Test Reprocessed Project: Bad Request', None), + ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'seqr-data-loading', @@ -837,6 +809,26 @@ def setUp(self): self.set_up() super().setUp() + def test_command(self, *args, **kwargs): + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ + 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], + 'total': 1, + }) + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=400) + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ + 'results': [{'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}], + }) + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ + 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], + 'total': 1, + }) + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=400) + responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ + 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], + 'total': 1, + }) + super().test_command(*args, **kwargs) + def _set_empty_loading_files(self): self.mock_glob.return_value = [] @@ -869,16 +861,36 @@ def _assert_expected_loading_file_calls(self, single_call): def _assert_expected_airtable_calls(self, *args, **kwargs): return 0, 0 + def _assert_saved_variant_models_updated(self): + updated_variants = SavedVariant.objects.filter(saved_variant_json__updated_field='updated_value') + self.assertEqual(len(updated_variants), 2) + self.assertSetEqual( + {v.guid for v in updated_variants}, + {'SV0000006_1248367227_r0004_non', 'SV0000002_1248367227_r0390_100'} + ) + reloaded_variant = next(v for v in updated_variants if v.guid == 'SV0000006_1248367227_r0004_non') + annotation_updated_variant = next(v for v in updated_variants if v.guid == 'SV0000002_1248367227_r0390_100') + self.assertEqual(len(reloaded_variant.saved_variant_json), 3) + self.assertListEqual(reloaded_variant.saved_variant_json['familyGuids'], ['F000014_14']) + self.assertEqual(len(annotation_updated_variant.saved_variant_json), 17) + self.assertListEqual(annotation_updated_variant.saved_variant_json['familyGuids'], ['F000001_1']) + + annotation_updated_json = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json + self.assertEqual(len(annotation_updated_json), 16) + self.assertEqual(annotation_updated_json['updated_new_field'], 'updated_value') + self.assertEqual(annotation_updated_json['rsid'], 'rs123') + self.assertEqual(annotation_updated_json['mainTranscriptId'], 'ENST00000505820') + self.assertEqual(len(annotation_updated_json['genotypes']), 3) + + class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project'] + fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] airtable_samples_url = 'http://testairtable/app3Y97xtbbaOopVR/Samples' airtable_pdo_url = 'http://testairtable/app3Y97xtbbaOopVR/PDO' airtable_loading_tracking_url = "http://testairtable/appUelDNM3BnWaR7M/AnVIL%20Seqr%20Loading%20Requests%20Tracking" AIRTABLE_LOADING_QUERY_TEMPLATE = "?fields[]=Status&pageSize=2&filterByFormula=AND({{AnVIL Project URL}}='https://seqr.broadinstitute.org/project/{}/project_page',OR(Status='Loading',Status='Loading Requested'))" - CLICKHOUSE_HOSTNAME = '' - MOCK_DATA_DIR = 'gs://seqr-hail-search-data/v3.1' PROJECT_EMAIL_TEXT = ANVIL_TEXT_EMAIL PROJECT_EMAIL_HTML = ANVIL_HTML_EMAIL @@ -913,6 +925,21 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe '==> gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '==> gsutil mv /mock/tmp/* gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', ] + RELOAD_PROJECT_VARIANT_LOGS = [ + ('Reloading saved variants in 2 projects', None), + ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), + ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 1 families for project Non-Analyst Project', None), + ('Reload Summary: ', None), + (' Non-Analyst Project: Updated 1 variants', None), + ] + RELOAD_SV_VARIANT_LOGS = [ + ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), + ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), + ('Reload Summary: ', None), + ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'anvil-data-loading', @@ -982,6 +1009,10 @@ def test_command(self, *args, **kwargs): }))) super().test_command(*args, **kwargs) + def _test_call(self, *args, reload_annotations_logs=None, reload_calls=None, **kwargs): + # No reloading for clickhouse + super()._test_call(*args, has_reload_calls=bool(reload_calls), **kwargs) + def _set_empty_loading_files(self): self.mock_subprocess.return_value.communicate.return_value = b'', b'One or more URLs matched no objects' @@ -991,7 +1022,7 @@ def _assert_has_expected_empty_list_file_calls(self): ) def _set_reloading_loading_files(self): - self.mock_ls_process.communicate.return_value = b'\n'.join([RUN_PATHS[6], RUN_PATHS[12]]), b'' + self.mock_ls_process.communicate.return_value = b'\n'.join(RUN_PATHS[6:8] + RUN_PATHS[15:17]), b'' self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): @@ -1025,12 +1056,10 @@ def _additional_loading_logs(self, data_type, version): def _assert_expected_airtable_calls(self, has_reload_calls, single_call): # Test request tracking updates for validation errors if single_call: - index = -3 fields = {'Status': 'Available in Seqr'} else: - index = -1 fields = {'Status': 'Loading request canceled', 'Notes': 'Callset validation failed'} - update_loading_tracking_request = responses.calls[index].request + update_loading_tracking_request = responses.calls[-1].request self.assertEqual(update_loading_tracking_request.url, self.airtable_loading_tracking_url) self.assertEqual(update_loading_tracking_request.method, 'PATCH') self.assertDictEqual(json.loads(update_loading_tracking_request.body), {'records': [ @@ -1073,3 +1102,11 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): if single_call: return 8, 0 return 7, 2 + + def _assert_saved_variant_models_updated(self): + saved_variant = SavedVariant.objects.get(key=100, family_id=14) + self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { + 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], + 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', + }}) + self.assertDictEqual(saved_variant.saved_variant_json, {}) diff --git a/seqr/management/tests/lift_project_to_hg38_tests.py b/seqr/management/tests/lift_project_to_hg38_tests.py deleted file mode 100644 index 36c52c83f8..0000000000 --- a/seqr/management/tests/lift_project_to_hg38_tests.py +++ /dev/null @@ -1,259 +0,0 @@ -#-*- coding: utf-8 -*- -import mock -from copy import deepcopy -from django.core.management.base import CommandError -from seqr.models import Family, Sample -from pyliftover.liftover import LiftOver -from seqr.views.utils.test_utils import VARIANTS, SINGLE_VARIANT - -from django.core.management import call_command -from django.test import TestCase - -PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' -PROJECT_GUID = 'R0001_1kg' -ELASTICSEARCH_INDEX = 'test_new_index' -SAMPLE_TYPE = 'WES' -GENOME_VERSION = '38' -SAMPLE_METADATA = { - 'genomeVersion': GENOME_VERSION, 'sampleType': SAMPLE_TYPE, 'sourceFilePath': 'data.vcf', 'fields': {'samples': {}}, -} -SAMPLE_IDS = ["NA19679", "NA19675_1", "NA19678", "HG00731", "HG00732", "HG00733"] - -liftover_to_38 = LiftOver('hg19', 'hg38') - -LIFT_MAP = { - 21003343353: [('chr21', 3343400)], - 1248367227: [('chr1', 248203925)], - 1001562437: [('chr1', 1627057)], - 1001560662: [('chr1', 46394160)], -} - - -def mock_convert_coordinate(chrom, pos): - pos = int(chrom.replace('chr', ''))*int(1e9) + pos - return(LIFT_MAP[pos]) - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -@mock.patch('seqr.utils.search.elasticsearch.es_utils.get_index_metadata', lambda index, client, **kwargs: {index: SAMPLE_METADATA}) -@mock.patch('seqr.management.commands.lift_project_to_hg38.logger') -@mock.patch('seqr.utils.search.elasticsearch.es_utils._get_es_sample_ids') -class LiftProjectToHg38Test(TestCase): - fixtures = ['users', '1kg_project'] - - def _get_num_new_index_samples(self): - return Sample.objects.filter(elasticsearch_index=ELASTICSEARCH_INDEX, is_active=True).count() - - @mock.patch('seqr.management.commands.lift_project_to_hg38.input') - @mock.patch('seqr.management.commands.lift_project_to_hg38.get_variants_for_variant_ids') - @mock.patch('seqr.management.commands.lift_project_to_hg38.LiftOver') - def test_command(self, mock_liftover, mock_get_es_variants, mock_input, mock_get_es_samples, mock_logger): - mock_get_es_samples.return_value = SAMPLE_IDS - mock_get_es_variants.return_value = VARIANTS - mock_liftover_to_38 = mock_liftover.return_value - mock_liftover_to_38.convert_coordinate.side_effect = mock_convert_coordinate - mock_input.return_value = 'y' - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - mock.call('Lifting over 3 variants (skipping 1 that are already lifted)'), - mock.call('Successfully lifted over 3 variants'), - mock.call('Successfully updated 3 variants'), - mock.call('---Done---'), - mock.call('Succesfully lifted over 3 variants. Skipped 2 failed variants. Family data not updated for 0 variants') - ] - mock_logger.info.assert_has_calls(calls) - - mock_get_es_samples.assert_called_with(ELASTICSEARCH_INDEX, 'samples', mock.ANY) - self.assertEqual(self._get_num_new_index_samples(), 6) - - calls = [ - mock.call('chr21', 3343353), - mock.call('chr1', 248367227), - mock.call('chr1', 1560662), - ] - mock_liftover_to_38.convert_coordinate.assert_has_calls(calls, any_order = True) - - families = {family for family in Family.objects.filter(pk__in = [1, 2])} - self.assertSetEqual(families, mock_get_es_variants.call_args.args[0]) - self.assertSetEqual({'21-3343400-GAGA-G', '1-248203925-TC-T', '1-46394160-G-A'}, - set(mock_get_es_variants.call_args.args[1])) - - # Test discontinue on lifted variants - mock_logger.reset_mock() - mock_input.side_effect = 'n' - with self.assertRaises(CommandError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Error: found 2 saved variants already on Hg38') - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - ] - mock_logger.info.assert_has_calls(calls) - - def test_command_error_unmatched_sample(self, mock_get_es_samples, mock_logger): - mock_get_es_samples.return_value = ['ID_NOT_EXIST'] - - with self.assertRaises(ValueError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Matches not found for sample ids: ID_NOT_EXIST. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.') - self.assertEqual(self._get_num_new_index_samples(), 0) - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index') - ] - mock_logger.info.assert_has_calls(calls) - - mock_get_es_samples.assert_called_with(ELASTICSEARCH_INDEX, 'samples', mock.ANY) - - def test_command_error_missing_indvididuals(self, mock_get_es_samples, mock_logger): - mock_get_es_samples.return_value = ['NA19675_1'] - - with self.assertRaises(ValueError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), - 'The following families are included in the callset but are missing some family members: 1 (NA19678).') - self.assertEqual(self._get_num_new_index_samples(), 0) - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - ] - mock_logger.info.assert_has_calls(calls) - - mock_get_es_samples.assert_called_with(ELASTICSEARCH_INDEX, 'samples', mock.ANY) - - def test_command_error_missing_families(self, mock_get_es_samples, mock_logger): - mock_get_es_samples.return_value = ['HG00731', 'HG00732', 'HG00733'] - - with self.assertRaises(ValueError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), - 'The following families have saved variants but are missing from the callset: 1.') - self.assertEqual(self._get_num_new_index_samples(), 0) - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - ] - mock_logger.info.assert_has_calls(calls) - - mock_get_es_samples.assert_called_with(ELASTICSEARCH_INDEX, 'samples', mock.ANY) - - @mock.patch('seqr.management.commands.lift_project_to_hg38.input') - @mock.patch('seqr.management.commands.lift_project_to_hg38.get_variants_for_variant_ids') - @mock.patch('seqr.management.commands.lift_project_to_hg38.get_single_variant') - @mock.patch('seqr.management.commands.lift_project_to_hg38.LiftOver') - def test_command_other_exceptions(self, mock_liftover, mock_single_es_variants, - mock_get_es_variants, mock_input, mock_get_es_samples, mock_logger): - mock_get_es_samples.return_value = SAMPLE_IDS - - # Test elasticsearch is disabled - with mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', ''): - with self.assertRaises(Exception) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - self.assertEqual(str(ce.exception), 'Adding samples is disabled without the elasticsearch backend') - - # Test discontinue on a failed lift - mock_liftover_to_38 = mock_liftover.return_value - mock_liftover_to_38.convert_coordinate.return_value = None - mock_input.return_value = 'n' - with self.assertRaises(CommandError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Error: found 1 saved variants already on Hg38') - - mock_input.side_effect = ['y', 'n'] - with self.assertRaises(CommandError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Error: unable to lift over 3 variants') - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - mock.call('Lifting over 3 variants (skipping 1 that are already lifted)') - ] - mock_logger.info.assert_has_calls(calls) - - # Test discontinue on failure of finding a variant in the index - mock_get_es_variants.return_value = VARIANTS - mock_liftover_to_38.convert_coordinate.side_effect = mock_convert_coordinate - mock_logger.reset_mock() - mock_input.side_effect = ['y', 'n', 'n'] - with self.assertRaises(CommandError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Error: unable to find 2 lifted-over variants') - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - mock.call('Lifting over 3 variants (skipping 1 that are already lifted)') - ] - mock_logger.info.assert_has_calls(calls) - - families = {f for f in Family.objects.filter(pk__in=[1,2])} - self.assertSetEqual(mock_get_es_variants.call_args.args[0], families) - self.assertSetEqual( - set(mock_get_es_variants.call_args.args[1]), - {'21-3343400-GAGA-G', '1-248203925-TC-T', '1-46394160-G-A'} - ) - - # Test discontinue on missing family data while updating the saved variants - variants = deepcopy(VARIANTS) - variants.append(SINGLE_VARIANT) - mock_get_es_variants.return_value = variants - mock_logger.reset_mock() - mock_input.side_effect = ['y', 'y', 'n'] - with self.assertRaises(CommandError) as ce: - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - self.assertEqual(str(ce.exception), 'Error: unable to find family data for lifted over variant') - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - mock.call('Lifting over 3 variants (skipping 1 that are already lifted)'), - mock.call('Successfully lifted over 4 variants') - ] - mock_logger.info.assert_has_calls(calls) - - mock_single_es_variants.return_value = SINGLE_VARIANT - mock_logger.reset_mock() - mock_input.reset_mock(side_effect = True) - mock_input.return_value = 'y' - call_command('lift_project_to_hg38', '--project={}'.format(PROJECT_NAME), - '--es-index={}'.format(ELASTICSEARCH_INDEX)) - - calls = [ - mock.call('Updating project genome version for {}'.format(PROJECT_NAME)), - mock.call('Validating es index test_new_index'), - mock.call('Lifting over 2 variants (skipping 2 that are already lifted)'), - mock.call('Successfully lifted over 4 variants'), - mock.call('Successfully updated 4 variants'), - mock.call('---Done---'), - mock.call('Succesfully lifted over 4 variants. Skipped 1 failed variants. Family data not updated for 1 variants') - ] - mock_logger.info.assert_has_calls(calls) - - families = [f for f in Family.objects.filter(pk=1)] - mock_single_es_variants.assert_called_with(families, '1-46394160-G-A', return_all_queried_families=True) diff --git a/seqr/management/tests/lift_variant_to_hg38_tests.py b/seqr/management/tests/lift_variant_to_hg38_tests.py deleted file mode 100644 index 70091667ee..0000000000 --- a/seqr/management/tests/lift_variant_to_hg38_tests.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -import mock - -from django.core.management import call_command -from django.test import TestCase -from django.core.management.base import CommandError -from seqr.views.utils.test_utils import PARSED_VARIANTS -from seqr.models import SavedVariant - -SAVED_VARIANT_GUID = 'SV0000001_2103343353_r0390_100' -VARIANT_ID = '21-3343353-GAGA-G' - - -class LiftVariantToHg38Test(TestCase): - fixtures = ['users', '1kg_project'] - - @mock.patch('seqr.management.commands.lift_variant_to_hg38.input') - @mock.patch('seqr.management.commands.lift_variant_to_hg38.logger') - @mock.patch('seqr.management.commands.lift_variant_to_hg38.get_single_variant') - def test_command(self, mock_single_es_variants, mock_logger, mock_input): - # Test user did not confirm. - mock_input.return_value = 'n' - with self.assertRaises(CommandError) as ce: - call_command('lift_variant_to_hg38', SAVED_VARIANT_GUID, VARIANT_ID) - - self.assertEqual(str(ce.exception), 'Error: user did not confirm') - - # Test user did confirm. - mock_single_es_variants.return_value = PARSED_VARIANTS[0] - mock_input.return_value = 'y' - call_command('lift_variant_to_hg38', SAVED_VARIANT_GUID, VARIANT_ID) - mock_logger.info.assert_called_with('---Done---') - - saved_variant = SavedVariant.objects.get(guid = SAVED_VARIANT_GUID) - mock_single_es_variants.assert_called_with([saved_variant.family], VARIANT_ID, - return_all_queried_families = True) - - self.assertListEqual( - [PARSED_VARIANTS[0]['xpos'], PARSED_VARIANTS[0]['ref'], PARSED_VARIANTS[0]['alt'], PARSED_VARIANTS[0]], - [saved_variant.xpos, saved_variant.ref, saved_variant.alt, saved_variant.saved_variant_json]) diff --git a/seqr/management/tests/reload_clinvar_all_variants_tests.py b/seqr/management/tests/reload_clinvar_all_variants_tests.py index c1056d7d44..9ffa96ae02 100644 --- a/seqr/management/tests/reload_clinvar_all_variants_tests.py +++ b/seqr/management/tests/reload_clinvar_all_variants_tests.py @@ -14,6 +14,7 @@ ) from reference_data.models import DataVersions from seqr.management.commands.reload_clinvar_all_variants import BATCH_SIZE, WEEKLY_XML_RELEASE +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin WEEKLY_XML_RELEASE_HEADER = '''''' WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' @@ -82,7 +83,7 @@ @mock.patch('seqr.management.commands.reload_clinvar_all_variants.safe_post_to_slack') @mock.patch('seqr.management.commands.reload_clinvar_all_variants.logger.info') -class ReloadClinvarAllVariantsTest(TestCase): +class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): databases = '__all__' fixtures = ['clinvar_all_variants'] diff --git a/seqr/management/tests/reload_saved_variant_annotations_tests.py b/seqr/management/tests/reload_saved_variant_annotations_tests.py index 1ceab1ee71..9df44a79d6 100644 --- a/seqr/management/tests/reload_saved_variant_annotations_tests.py +++ b/seqr/management/tests/reload_saved_variant_annotations_tests.py @@ -15,6 +15,8 @@ class ReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): fixtures = ['users', '1kg_project'] + CLICKHOUSE_HOSTNAME = None + @responses.activate def test_command(self): responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ @@ -67,12 +69,12 @@ def test_command(self): }) annotation_updated_json_1 = SavedVariant.objects.get(guid='SV0000002_1248367227_r0390_100').saved_variant_json - self.assertEqual(len(annotation_updated_json_1), 19) + self.assertEqual(len(annotation_updated_json_1), 17) self.assertListEqual(annotation_updated_json_1['familyGuids'], ['F000001_1']) self.assertEqual(annotation_updated_json_1['updated_field'], 'updated_value') annotation_updated_json_2 = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json_2), 18) + self.assertEqual(len(annotation_updated_json_2), 16) self.assertEqual(annotation_updated_json_2['updated_new_field'], 'updated_value') self.assertEqual(annotation_updated_json_2['rsid'], 'rs123') self.assertEqual(annotation_updated_json_2['mainTranscriptId'], 'ENST00000505820') @@ -116,3 +118,14 @@ def test_command(self): 'data_type': 'SV_WGS', 'variant_ids': ['prefix_19107_DEL'], }) + + +class ClickhouseReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): + + fixtures = ['users', '1kg_project'] + + def test_command(self): + with self.assertRaises(CommandError) as ce: + call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') + self.assertEqual(str(ce.exception), 'Reloading variant annotations is not supported in clickhouse') + diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index abe15b58d0..46d47bde1a 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -1,7 +1,7 @@ #-*- coding: utf-8 -*- import mock -from django.core.management import call_command +from django.core.management import call_command, CommandError from django.test import TestCase from seqr.models import Family @@ -10,6 +10,7 @@ FAMILY_GUID = 'F000001_1' +@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') class ReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] @@ -82,3 +83,16 @@ def test_with_param_command(self, mock_get_variants, mock_logger): mock_logger.info.assert_has_calls(logger_info_calls) mock_logger.error.assert_called_with('Error reloading variants in 1kg project n\xe5me with uni\xe7\xf8de: Database error.') + + +@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') +class ClickhouseReloadSavedVariantJsonTest(TestCase): + + fixtures = ['users', '1kg_project'] + + def test_command(self): + with self.assertRaises(CommandError) as ce: + call_command('reload_saved_variant_json', PROJECT_NAME) + self.assertEqual(str(ce.exception), 'Reloading variants is not supported in clickhouse') + + diff --git a/seqr/migrations/0080_savedvariant_dataset_type_savedvariant_genotypes_and_more.py b/seqr/migrations/0080_savedvariant_dataset_type_savedvariant_genotypes_and_more.py new file mode 100644 index 0000000000..a0f2530974 --- /dev/null +++ b/seqr/migrations/0080_savedvariant_dataset_type_savedvariant_genotypes_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.22 on 2025-07-16 18:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0079_alter_family_analysis_status'), + ] + + operations = [ + migrations.AddField( + model_name='savedvariant', + name='dataset_type', + field=models.CharField(blank=True, choices=[('SNV_INDEL', 'Variant Calls'), ('MITO', 'Mitochondria calls'), ('SV_WGS', 'SV WGS Calls'), ('SV_WES', 'gCNV Calls')], max_length=13, null=True), + ), + migrations.AddField( + model_name='savedvariant', + name='genotypes', + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name='savedvariant', + name='key', + field=models.PositiveBigIntegerField(blank=True, null=True), + ), + ] diff --git a/seqr/models.py b/seqr/models.py index 922f34c9a6..180ad8e04f 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -822,6 +822,13 @@ class Meta: class SavedVariant(ModelWithGUID): + DATASET_TYPE_CHOICES = ( + (Sample.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), + (Sample.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), + (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', 'SV WGS Calls'), + (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}', 'gCNV Calls'), + ) + family = models.ForeignKey('Family', on_delete=models.CASCADE) xpos = models.BigIntegerField() @@ -829,9 +836,12 @@ class SavedVariant(ModelWithGUID): ref = models.TextField(null=True) alt = models.TextField(null=True) variant_id = models.TextField(db_index=True) + key = models.PositiveBigIntegerField(null=True, blank=True) selected_main_transcript_id = models.CharField(max_length=20, null=True) saved_variant_json = JSONField(default=dict) + genotypes = JSONField(default=dict) + dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True) acmg_classification = JSONField(null=True) # ACMG based classification diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py index c16cf24371..ca4ee9d9a9 100644 --- a/seqr/utils/search/constants.py +++ b/seqr/utils/search/constants.py @@ -22,6 +22,7 @@ HAS_REF = 'has_ref' COMPOUND_HET = 'compound_het' +COMPOUND_HET_ALLOW_HOM_ALTS = 'compound_het_allow_hom_alts' RECESSIVE = 'recessive' X_LINKED_RECESSIVE = 'x_linked_recessive' HOMOZYGOUS_RECESSIVE = 'homozygous_recessive' diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 8640f6fbef..a9196c5bb1 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -1318,7 +1318,7 @@ def setup_responses(): @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') @mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) class EsUtilsTest(TestCase): - databases = '__all__' + databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] def setUp(self): @@ -1402,39 +1402,29 @@ def test_get_es_variants_for_variant_ids(self): @urllib3_responses.activate def test_get_single_es_variant(self): setup_responses() - variant = get_single_variant(self.families, '2-103343353-GAGA-G') - self.assertDictEqual(variant, PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[1]) + family = self.families.filter(guid='F000003_3').first() + variant = get_single_variant(family, '2-103343353-GAGA-G') + self.assertDictEqual(variant, {**PARSED_ANY_AFFECTED_VARIANTS[1], 'selectedMainTranscriptId': None}) self.assertExecutedSearch( filters=[{'terms': {'variantId': ['2-103343353-GAGA-G']}}], size=1, index=INDEX_NAME ) - variant = get_single_variant(self.families, 'prefix_19107_DEL') + family = self.families.filter(guid='F000002_2').first() + variant = get_single_variant(family, 'prefix_19107_DEL') self.assertDictEqual(variant, PARSED_SV_VARIANT) self.assertExecutedSearch( filters=[{'terms': {'variantId': ['prefix_19107_DEL']}}], size=1, index=SV_INDEX_NAME, ) - variant = get_single_variant(self.families, 'M-10195-C-A') + variant = get_single_variant(family, 'M-10195-C-A') self.assertDictEqual(variant, PARSED_MITO_VARIANT) self.assertExecutedSearch( filters=[{'terms': {'variantId': ['M-10195-C-A']}}], size=1, index=MITO_WGS_INDEX_NAME, ) - variant = get_single_variant(self.families, '1-248367227-TC-T', return_all_queried_families=True) - all_family_variant = deepcopy(PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[0]) - all_family_variant['familyGuids'] = ['F000002_2', 'F000003_3', 'F000005_5'] - all_family_variant['genotypes']['I000004_hg00731'] = { - 'ab': 0, 'ad': None, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 0, 'dp': 88, 'pl': None, 'sampleType': 'WES', - } - self.assertDictEqual(variant, all_family_variant) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['1-248367227-TC-T']}}], - size=1, index=INDEX_NAME, - ) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.families, '10-10334333-A-G') + get_single_variant(family, '10-10334333-A-G') self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_COMPOUND_HET_GENES', 1) diff --git a/seqr/utils/search/hail_search_utils.py b/seqr/utils/search/hail_search_utils.py index ab67501d0f..0cc5f53226 100644 --- a/seqr/utils/search/hail_search_utils.py +++ b/seqr/utils/search/hail_search_utils.py @@ -1,6 +1,6 @@ from collections import defaultdict -from django.db.models import F, Min, Count, Case, When +from django.db.models import F, Min, Case, When from time import sleep from urllib3.connectionpool import connection_from_url @@ -86,7 +86,7 @@ def _execute_multi_sample_types_searches(sv_data_types, search_body, user): return {'total': total, 'results': sorted(results, key=lambda x: x[0]['_sort'] if isinstance(x, list) else x['_sort'])[:search_body['num_results']]} -def get_hail_variants_for_variant_ids(samples, genome_version, parsed_variant_ids, user, user_email=None, return_all_queried_families=False): +def get_hail_variants_for_variant_ids(samples, genome_version, parsed_variant_ids, user, user_email=None): search = { 'variant_ids': [parsed_id for parsed_id in parsed_variant_ids.values() if parsed_id], 'variant_keys': [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id], @@ -94,10 +94,6 @@ def get_hail_variants_for_variant_ids(samples, genome_version, parsed_variant_id search_body = _format_search_body(samples, genome_version, len(parsed_variant_ids), search) response_json = _execute_search(search_body, user, user_email=user_email) - if return_all_queried_families: - expected_family_guids = set(samples.values_list('individual__family__guid', flat=True)) - _validate_expected_families(response_json['results'], expected_family_guids) - return response_json['results'] @@ -177,35 +173,3 @@ def _get_sort_metadata(sort, samples): ).values('gene_id').annotate(min_rank=Min('rank')) } return sort_metadata - - -def _validate_expected_families(results, expected_families): - # In the ES backed we could force return variants even if all families are hom ref - # This is not possible in the hail backend as those rows are removed at loading, so fail if missing - invalid_family_variants = [] - for result in results: - missing_families = expected_families - set(result['familyGuids']) - if missing_families: - invalid_family_variants.append((result['variantId'], missing_families)) - - if invalid_family_variants: - from seqr.utils.search.utils import InvalidSearchException - missing = ', '.join([ - f'{variant_id} ({"; ".join(sorted(families))})' for variant_id, families in invalid_family_variants - ]) - raise InvalidSearchException(f'Unable to return all families for the following variants: {missing}') - - -MAX_FAMILY_COUNTS = {Sample.SAMPLE_TYPE_WES: 200, Sample.SAMPLE_TYPE_WGS: 35} - - -def validate_hail_backend_no_location_search(samples): - sample_counts = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).values('sample_type').annotate( - family_count=Count('individual__family_id', distinct=True), - project_count=Count('individual__family__project_id', distinct=True), - ) - from seqr.utils.search.utils import InvalidSearchException - if sample_counts and (len(sample_counts) > 1 or sample_counts[0]['project_count'] > 1): - raise InvalidSearchException('Location must be specified to search across multiple projects') - if sample_counts and sample_counts[0]['family_count'] > MAX_FAMILY_COUNTS[sample_counts[0]['sample_type']]: - raise InvalidSearchException('Location must be specified to search across multiple families in large projects') diff --git a/seqr/utils/search/hail_search_utils_tests.py b/seqr/utils/search/hail_search_utils_tests.py index 12413b8d1e..b1e1700687 100644 --- a/seqr/utils/search/hail_search_utils_tests.py +++ b/seqr/utils/search/hail_search_utils_tests.py @@ -39,7 +39,7 @@ @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HOST) class HailSearchUtilsTests(SearchTestHelper, TestCase): - databases = '__all__' + databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] def setUp(self): @@ -84,7 +84,7 @@ def _test_expected_search_call(self, search_fields=None, gene_ids=None, interval self._test_minimal_search_call(**expected_search, **kwargs) - @mock.patch('seqr.utils.search.hail_search_utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) + @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) @responses.activate def test_query_variants(self): variants, total = query_variants(self.results_model, user=self.user) @@ -335,37 +335,32 @@ def test_sv_variant_lookup(self): @responses.activate def test_get_single_variant(self): - variant = get_single_variant(self.families, '2-103343353-GAGA-G', user=self.user) + family = self.families.filter(guid='F000002_2').first() + variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) self.assertDictEqual(variant, HAIL_BACKEND_VARIANTS[0]) self._test_minimal_search_call( variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], - num_results=1, sample_data={'SNV_INDEL': ALL_AFFECTED_SAMPLE_DATA['SNV_INDEL']}) + num_results=1, sample_data={'SNV_INDEL': FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']}) - get_single_variant(self.families, 'prefix_19107_DEL', user=self.user) + get_single_variant(family, 'prefix_19107_DEL', user=self.user) self._test_minimal_search_call( variant_ids=[], variant_keys=['prefix_19107_DEL'], num_results=1, sample_data=EXPECTED_SAMPLE_DATA, omit_data_type='SNV_INDEL') - get_single_variant(self.families, 'M-10195-C-A', user=self.user) + get_single_variant(family, 'M-10195-C-A', user=self.user) self._test_minimal_search_call( variant_ids=[['M', 10195, 'C', 'A']], variant_keys=[], num_results=1, sample_data=EXPECTED_MITO_SAMPLE_DATA) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.families, '1-91502721-G-A', user=self.user, return_all_queried_families=True) - self.assertEqual( - str(cm.exception), - 'Unable to return all families for the following variants: 1-38724419-T-G (F000003_3; F000005_5), 1-91502721-G-A (F000005_5)', - ) - get_single_variant(self.families.filter(guid='F000002_2'), '2-103343353-GAGA-G', user=self.user, return_all_queried_families=True) + get_single_variant(family, '2-103343353-GAGA-G', user=self.user) self._test_minimal_search_call( variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], num_results=1, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=200, json={'results': [], 'total': 0}) with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.families, '10-10334333-A-G', user=self.user) + get_single_variant(family, '10-10334333-A-G', user=self.user) self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') @responses.activate diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index e85e40e087..681b438f3f 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -10,11 +10,13 @@ from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ query_variants, variant_lookup, sv_variant_lookup, InvalidSearchException -from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS + SV_SAMPLES = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] MITO_SAMPLES = ['S000149_hg00733'] NON_SNP_INDEL_SAMPLES = SV_SAMPLES + MITO_SAMPLES +FAMILY_3_SAMPLE = 'S000135_na20870' class SearchTestHelper(object): @@ -51,7 +53,7 @@ def set_up(self): 'S000137_na20874', ]) self.affected_search_samples = Sample.objects.filter(guid__in=[ - 'S000132_hg00731', 'S000133_hg00732', 'S000134_hg00733', 'S000135_na20870', + 'S000132_hg00731', 'S000133_hg00732', 'S000134_hg00733', FAMILY_3_SAMPLE, 'S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733', ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) @@ -98,37 +100,37 @@ def _mock_get_variants(samples, search, user, previous_search_results, genome_ve self.mock_redis.get.assert_called_with(cache_key) def test_get_single_variant(self, mock_get_variants_for_ids): - mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] - variant = get_single_variant(self.families, '2-103343353-GAGA-G', user=self.user) + family = self.families.filter(guid='F000002_2').first() + variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) self.assertDictEqual(variant, PARSED_VARIANTS[0]) - mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', {'2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G')}, self.user, return_all_queried_families=False, - user_email=None, - ) - expected_samples = {s for s in self.search_samples if s.guid not in NON_SNP_INDEL_SAMPLES} - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - - get_single_variant(self.families, '2-103343353-GAGA-G', user=self.user, return_all_queried_families=True) - mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', {'2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G')}, self.user, return_all_queried_families=True, - user_email=None, + expected_samples = {s for s in self.affected_search_samples if s.guid not in NON_SNP_INDEL_SAMPLES and s.guid != FAMILY_3_SAMPLE} + self._assert_expected_get_single_variant_call( + mock_get_variants_for_ids, ('2', 103343353, 'GAGA', 'G'), expected_samples, ) - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - get_single_variant(self.families, 'prefix_19107_DEL', user=self.user) - mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', {'prefix_19107_DEL': None}, self.user, return_all_queried_families=False, user_email=None, - ) + get_single_variant(family, 'prefix_19107_DEL', user=self.user) expected_samples = { s for s in self.search_samples if s.guid in ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] } - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) + self._assert_expected_get_single_variant_call( + mock_get_variants_for_ids, 'prefix_19107_DEL', expected_samples, dataset_type='SV', + ) mock_get_variants_for_ids.return_value = [] with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.families, '10-10334333-A-G') + get_single_variant(family, '10-10334333-A-G') self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') + def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, parsed_variant_id, expected_samples, **kwargs): + if isinstance(parsed_variant_id, str): + variants_id_map = {parsed_variant_id: None} + else: + variants_id_map = {'-'.join([str(v) for v in parsed_variant_id]): parsed_variant_id} + mock_get_variants_for_ids.assert_called_with( + mock.ANY, '37', variants_id_map, self.user, + ) + self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) + def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): variant_ids = ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL', 'MT-10195-C-A'] get_variants_for_variant_ids(self.families, variant_ids, user=self.user) @@ -161,6 +163,7 @@ def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): expected_samples = {s for s in self.search_samples if s.guid not in skipped_samples} self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) + @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def _test_invalid_search_params(self, search_func): with self.assertRaises(InvalidSearchException) as cm: @@ -272,12 +275,26 @@ def _test_invalid_search_params(self, search_func): 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde, Test Reprocessed Project; 38 - Non-Analyst Project', ) + self._test_invalid_no_location_search_params() + self.results_model.families.set(Family.objects.filter(guid='F000014_14')) self.search_model.search['locus']['rawItems'] = build_specific_genes with self.assertRaises(InvalidSearchException) as cm: search_func(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + def _test_invalid_no_location_search_params(self): + self.results_model.families.set(self.families) + self.search_model.search['inheritance'] = {} + with self.assertRaises(InvalidSearchException) as cm: + query_variants(self.results_model) + self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple families in large projects') + + self.results_model.families.set(Family.objects.filter(id__in=[2, 11])) + with self.assertRaises(InvalidSearchException) as cm: + query_variants(self.results_model) + self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') + def test_invalid_search_query_variants(self): with self.assertRaises(InvalidSearchException) as se: query_variants(self.results_model, sort='prioritized_gene', num_results=2) @@ -507,7 +524,7 @@ def test_cached_get_variant_query_gene_counts(self): @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): - databases = '__all__' + databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] def setUp(self): @@ -551,6 +568,7 @@ def test_sv_variant_lookup(self, *args, **kwargs): @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') def test_get_single_variant(self, mock_get_variants_for_ids): + mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] super(ElasticsearchSearchUtilsTests, self).test_get_single_variant(mock_get_variants_for_ids) @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') @@ -613,10 +631,14 @@ def test_cached_get_variant_query_gene_counts(self): 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}} }) + def _test_invalid_no_location_search_params(self): + # Elasticsearch has no limits on no-location searches + pass + @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') class HailSearchUtilsTests(TestCase, SearchUtilsTests): - databases = '__all__' + databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] def setUp(self): @@ -639,6 +661,7 @@ def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') def test_get_single_variant(self, mock_call): + mock_call.return_value = [PARSED_VARIANTS[0]] super(HailSearchUtilsTests, self).test_get_single_variant(mock_call) @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') @@ -666,7 +689,7 @@ def test_cached_get_variant_query_gene_counts(self): @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') -class ClickhouseSearchUtilsTests(TestCase, SearchUtilsTests): +class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): databases = '__all__' fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] @@ -694,10 +717,14 @@ def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, ex variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, ) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') + @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') def test_get_single_variant(self, mock_call): - with self.assertRaises(NotImplementedError): - super().test_get_single_variant(mock_call) + mock_call.return_value = PARSED_VARIANTS[0] + super().test_get_single_variant(mock_call) + + def _assert_expected_get_single_variant_call(self, mock_call, variant_id, expected_samples, dataset_type='SNV_INDEL', **kwargs): + mock_call.assert_called_with(variant_id, mock.ANY, '37', dataset_type) + self.assertSetEqual(set(mock_call.call_args.args[1]), expected_samples) @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') def test_get_variants_for_variant_ids(self, mock_call): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 4e51026535..4b1e01d86d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -1,9 +1,10 @@ from collections import defaultdict from copy import deepcopy from datetime import timedelta +from django.db.models import Count from clickhouse_search.search import clickhouse_backend_enabled, get_clickhouse_variants, format_clickhouse_results, \ - get_clickhouse_cache_results, clickhouse_variant_lookup + get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json @@ -14,7 +15,7 @@ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.search.hail_search_utils import get_hail_variants, get_hail_variants_for_variant_ids, ping_hail_backend, \ - hail_variant_lookup, validate_hail_backend_no_location_search + hail_variant_lookup from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS @@ -58,7 +59,7 @@ def _raise_clickhouse_not_implemented(*args, **kwargs): raise NotImplementedError('Clickhouse backend is not implemented for this function.') -def backend_specific_call(es_func, hail_backend_func, clickhouse_func=_raise_clickhouse_not_implemented): +def backend_specific_call(es_func, hail_backend_func, clickhouse_func): if es_backend_enabled(): return es_func elif clickhouse_backend_enabled(): @@ -68,11 +69,12 @@ def backend_specific_call(es_func, hail_backend_func, clickhouse_func=_raise_cli def ping_search_backend(): - backend_specific_call(ping_elasticsearch, ping_hail_backend)() + # Clickhouse backend does not need special uptime testing, will be checked with the other database connection pings + backend_specific_call(ping_elasticsearch, ping_hail_backend, lambda: None)() def ping_search_backend_admin(): - backend_specific_call(ping_kibana, lambda: True)() + backend_specific_call(ping_kibana, lambda: True, lambda: True)() def get_search_backend_status(): @@ -90,8 +92,8 @@ def get_search_samples(projects, active_only=True): return _get_filtered_search_samples({'individual__family__project__in': projects}, active_only=active_only) -def _get_families_search_data(families, dataset_type): - samples = _get_filtered_search_samples({'individual__family__in': families}) +def _get_families_search_data(families, dataset_type, sample_filter=None): + samples = _get_filtered_search_samples(sample_filter or {'individual__family__in': families}) if len(samples) < 1: raise InvalidSearchException('No search data found for families {}'.format( ', '.join([f.family_id for f in families]))) @@ -131,20 +133,34 @@ def delete_search_backend_data(data_id): )(data_id) -def get_single_variant(families, variant_id, return_all_queried_families=False, user=None): - variants = _get_variants_for_variant_ids( - families, [variant_id], user, return_all_queried_families=return_all_queried_families, - ) - if not variants: +def get_single_variant(family, variant_id, user=None): + parsed_variant_id = parse_variant_id(variant_id) + dataset_type = _variant_ids_dataset_type([parsed_variant_id]) + samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) + variant = backend_specific_call( + _process_ids_search(get_es_variants_for_variant_ids), + _process_ids_search(get_hail_variants_for_variant_ids), + _get_clickhouse_variant_by_id, + )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type, user) + if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return variants[0] + return variant + +def _process_ids_search(search_func): + def _search(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): + variants = search_func(samples, genome_version, {variant_id: parsed_variant_id}, user) + return variants[0] if variants else None + return _search -def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user=None, user_email=None): - return _get_variants_for_variant_ids(families, variant_ids, user, user_email, dataset_type=dataset_type) + +def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): + return get_clickhouse_variant_by_id( + parsed_variant_id or variant_id, samples, genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], + ) -def _get_variants_for_variant_ids(families, variant_ids, user, user_email=None, dataset_type=None, **kwargs): +def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user=None, user_email=None, **kwargs): parsed_variant_ids = {} for variant_id in variant_ids: parsed_variant_ids[variant_id] = parse_variant_id(variant_id) @@ -157,9 +173,9 @@ def _get_variants_for_variant_ids(families, variant_ids, user, user_email=None, } dataset_type = _variant_ids_dataset_type(parsed_variant_ids.values()) - return backend_specific_call(get_es_variants_for_variant_ids, get_hail_variants_for_variant_ids)( + return backend_specific_call(get_es_variants_for_variant_ids, get_hail_variants_for_variant_ids, _raise_clickhouse_not_implemented)( _get_families_search_data(families, dataset_type=dataset_type), _get_search_genome_version(families), - parsed_variant_ids, user, user_email=user_email, **kwargs + parsed_variant_ids, user, user_email=user_email, ) @@ -560,7 +576,21 @@ def _validate_search(search, samples, previous_search_results): ) if not has_location_filter: - backend_specific_call(lambda *args: None, validate_hail_backend_no_location_search, lambda *args: None)(samples) + backend_specific_call(lambda *args: None, _validate_no_location_search, _validate_no_location_search)(samples) + + +MAX_FAMILY_COUNTS = {Sample.SAMPLE_TYPE_WES: 200, Sample.SAMPLE_TYPE_WGS: 35} + + +def _validate_no_location_search(samples): + variant_samples = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + if variant_samples.values('individual__family__project_id').distinct().count() > 1: + raise InvalidSearchException('Location must be specified to search across multiple projects') + sample_counts = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).values('sample_type').annotate( + family_count=Count('individual__family_id', distinct=True), + ) + if any(sample_count['family_count'] > MAX_FAMILY_COUNTS[sample_count['sample_type']] for sample_count in sample_counts): + raise InvalidSearchException('Location must be specified to search across multiple families in large projects') def _filter_inheritance_family_samples(samples, inheritance_filter): diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index db52140e17..af71b1b34c 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,12 +5,14 @@ from collections import defaultdict from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import Count, Max, Q +from django.db.models import Count, Max, Q, F from django.db.models.fields.files import ImageFieldFile +from clickhouse_search.search import get_clickhouse_genes from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json @@ -51,16 +53,17 @@ def family_page_data(request, family_guid): add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] + additional_fields = backend_specific_call([],[],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( - 'saved_variant_json__transcripts', 'saved_variant_json__svType', 'xpos', 'xpos_end', + 'xpos', 'xpos_end', *additional_fields, + svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'), ) - gene_ids = { - gene_id for variant in discovery_variants - for gene_id in (variant['saved_variant_json__transcripts'] or {}).keys() - } + gene_ids = backend_specific_call( + _variants_gene_ids, _variants_gene_ids, _clickhouse_variants_gene_ids, + )(discovery_variants, project.genome_version) discovery_variant_intervals = [dict(zip( - ['chrom', 'start', 'end_chrom', 'end', 'svType'], - [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['saved_variant_json__svType']] + ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], + [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] )) for v in discovery_variants] omims = Omim.objects.filter( get_omim_intervals_query(discovery_variant_intervals) | Q(gene__gene_id__in=gene_ids) @@ -100,6 +103,24 @@ def family_page_data(request, family_guid): return create_json_response(response) +def _variants_gene_ids(variants, *args, **kwargs): + return {gene_id for variant in variants for gene_id in (variant['transcripts'] or {}).keys()} + + +def _clickhouse_variants_gene_ids(variants, genome_version): + keys_by_dataset_type = defaultdict(set) + no_key_variants = [] + for v in variants: + if v['key']: + keys_by_dataset_type[v['dataset_type']].add(v['key']) + else: + no_key_variants.append(v) + gene_ids = _variants_gene_ids(no_key_variants) + for dataset_type, keys in keys_by_dataset_type.items(): + gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) + return gene_ids + + def _intervals_overlap(interval1, interval2): return interval1['chrom'] == interval2['chrom'] and ( (interval2['start'] <= interval1['start'] <= interval2['end']) or @@ -124,7 +145,7 @@ def family_variant_tag_summary(request, family_guid): project = family.project check_project_permissions(project, request.user) - response = families_discovery_tags([{'familyGuid': family_guid}]) + response = families_discovery_tags([{'familyGuid': family_guid}], genome_version=project.genome_version) tags = VariantTag.objects.filter(saved_variants__family=family) family_tag_type_counts = tags.values('variant_tag_type__name').annotate(count=Count('*')) @@ -204,7 +225,7 @@ def delete_families_handler(request, project_guid): project_guid (string): GUID of project that contains these individuals. """ - project = get_project_and_check_pm_permissions(project_guid, request.user) + project = get_project_and_check_pm_permissions(project_guid, request.user, override_permission_func=external_anvil_project_can_edit) request_json = json.loads(request.body) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 7fbaf7d635..eb5450060e 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -261,8 +261,11 @@ def test_edit_families_handler(self, mock_pm_group): self.assertEqual(response.status_code, 200) @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') + @mock.patch('seqr.views.utils.permissions_utils.INTERNAL_NAMESPACES') @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') - def test_delete_families_handler(self, mock_pm_group): + def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): + internal_namespaces = ['my-seqr-billing'] + mock_internal_namespaces.__contains__.side_effect = lambda s: s in internal_namespaces url = reverse(delete_families_handler, args=[PROJECT_GUID]) self.check_manager_login(url) @@ -315,6 +318,16 @@ def test_delete_families_handler(self, mock_pm_group): 'families': [{'familyGuid': 'F000012_12'}]})) self.assertEqual(response.status_code, 200) + self.login_manager() + response = self.client.post(url, content_type='application/json', data=json.dumps({ + 'families': [{'familyGuid': 'F000011_11'}]})) + self.assertEqual(response.status_code, 403) + + internal_namespaces = [] + response = self.client.post(url, content_type='application/json', data=json.dumps({ + 'families': [{'familyGuid': 'F000011_11'}]})) + self.assertEqual(response.status_code, 200 if self.EXTERNAL_ANVIL_CAN_DELETE else 403) + def test_update_family_analysed_by(self): url = reverse(update_family_analysed_by, args=[FAMILY_GUID]) self.check_collaborator_login(url) @@ -649,6 +662,10 @@ def test_get_family_phenotype_gene_scores(self): class LocalFamilyAPITest(AuthenticationTestCase, FamilyAPITest): fixtures = ['users', '1kg_project', 'reference_data'] + EXTERNAL_ANVIL_CAN_DELETE = False + class AnvilFamilyAPITest(AnvilAuthenticationTestCase, FamilyAPITest): - fixtures = ['users', '1kg_project', 'reference_data'] + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] + + EXTERNAL_ANVIL_CAN_DELETE = True diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 6cfb6bdf96..a43b2e9a3d 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1146,7 +1146,7 @@ def test_import_gregor_metadata(self, mock_subprocess): ).order_by('family_id', 'variant_id').distinct().values( 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json__genomeVersion', 'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId', - 'saved_variant_json__hgvsc', + 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes' ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1154,14 +1154,16 @@ def test_import_gregor_metadata(self, mock_subprocess): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': 'F000012_12', - 'saved_variant_json__genomeVersion': '37', - 'saved_variant_json__transcripts': mock.ANY, - 'saved_variant_json__genotypes': mock.ANY, - 'saved_variant_json__mainTranscriptId': 'ENST00000505820', + 'saved_variant_json__genomeVersion': None, + 'saved_variant_json__transcripts': None, + 'saved_variant_json__genotypes': None, + 'saved_variant_json__mainTranscriptId': None, 'saved_variant_json__hgvsc': None, + 'key': 100, + 'dataset_type': 'SNV_INDEL', + 'genotypes': mock.ANY, }) - self.assertEqual(len(saved_variants[0]['saved_variant_json__transcripts']), 2) - self.assertEqual(len(saved_variants[0]['saved_variant_json__genotypes']), 2) + self.assertEqual(len(saved_variants[0]['genotypes']), 2) self.assertDictEqual(saved_variants[1], { 'guid': mock.ANY, 'variant_id': '1-249045487-A-G', @@ -1174,6 +1176,9 @@ def test_import_gregor_metadata(self, mock_subprocess): 'saved_variant_json__genotypes': {created_individual_guid: {'numAlt': 1}}, 'saved_variant_json__mainTranscriptId': None, 'saved_variant_json__hgvsc': None, + 'key': None, + 'dataset_type': None, + 'genotypes': {}, }) new_family_genotypes = {new_family_individual_guid: {'numAlt': 2}} self.assertDictEqual(saved_variants[2], { @@ -1181,13 +1186,14 @@ def test_import_gregor_metadata(self, mock_subprocess): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': new_family_guid, - 'saved_variant_json__genomeVersion': '37', - 'saved_variant_json__transcripts': { - 'ENSG00000135953': [{'hgvsc': 'c.3955G>A', 'hgvsp': 'c.1586-17C>G', 'transcriptId': 'ENST00000505820'}] - }, - 'saved_variant_json__genotypes': new_family_genotypes, - 'saved_variant_json__mainTranscriptId': 'ENST00000505820', + 'saved_variant_json__genomeVersion': None, + 'saved_variant_json__transcripts': None, + 'saved_variant_json__genotypes': None, + 'saved_variant_json__mainTranscriptId': None, 'saved_variant_json__hgvsc': None, + 'key': 100, + 'dataset_type': 'SNV_INDEL', + 'genotypes': new_family_genotypes, }) variant_tags = VariantTag.objects.filter(variant_tag_type__name='GREGoR Finding') @@ -1247,11 +1253,10 @@ def test_import_gregor_metadata(self, mock_subprocess): }}) self.assertDictEqual(response_json['individualsByGuid'], {}) - no_gene_saved_variant_json = SavedVariant.objects.get(family__guid=new_family_guid, variant_id='1-248367227-TC-T').saved_variant_json - self.assertDictEqual(no_gene_saved_variant_json['transcripts'], {}) - self.assertDictEqual(no_gene_saved_variant_json['genotypes'], new_family_genotypes) - self.assertNotIn('mainTranscriptId', no_gene_saved_variant_json) - self.assertNotIn('hgvsc', no_gene_saved_variant_json) + saved_variant = SavedVariant.objects.get(family__guid=new_family_guid, variant_id='1-248367227-TC-T') + self.assertEqual(saved_variant.key, 100) + self.assertDictEqual(saved_variant.genotypes, new_family_genotypes) + self.assertDictEqual(saved_variant.saved_variant_json, {}) def test_get_hpo_terms(self): url = reverse(get_hpo_terms, args=['HP:0011458']) @@ -1357,7 +1362,7 @@ def test_import_gregor_metadata(self, *args): class AnvilIndividualAPITest(AnvilAuthenticationTestCase, IndividualAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] HAS_EXTERNAL_PROJECT_ACCESS = True def setUp(self): diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 0cd9672bd2..648b5c95ca 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -250,7 +250,7 @@ def project_families(request, project_guid): **{key: family_id in data_families for key, data_families in has_data_families.items()}, }) - response = families_discovery_tags(families, project=project) + response = families_discovery_tags(families, genome_version=project.genome_version, project=project) return create_json_response(response) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 94a2a058a2..7aee432b5f 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -427,7 +427,7 @@ def test_project_families(self): len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families }, {0}) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953'}) + self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953', 'ENSG00000240361'}) # Test empty project empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) @@ -700,7 +700,7 @@ def _assert_expected_airtable_requests(self, *args, **kwargs): # Test for permissions from AnVIL only class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] PROJECT_COLLABORATORS = ANVIL_COLLABORATORS PROJECT_COLLABORATOR_GROUPS = None HAS_EMPTY_PROJECT = False diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index d6cb397db5..b85f2d852a 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -130,7 +130,7 @@ def _add_row(row, family_id, row_type): if not (discovery_row.get(GENE_COLUMN) or discovery_row.get('sv_type'))] if missing_gene_rows: raise ErrorsWarningsException( - [f'Discovery variant(s) {", ".join(missing_gene_rows)} in family {family_id} have no associated gene']) + [f'Discovery variant(s) {", ".join(sorted(missing_gene_rows))} in family {family_id} have no associated gene']) parsed_rows[row_type] += [{ 'entity:discovery_id': f'{discovery_row["chrom"]}_{discovery_row["pos"]}_{discovery_row["participant_id"]}', **{k: str(discovery_row.get(k.lower()) or '') for k in ['Zygosity', 'Chrom', 'Pos', 'Ref', 'Alt', 'Transcript']}, @@ -1014,9 +1014,10 @@ def _add_row(row, family_id, row_type): individual_samples={i: None for i in individuals}, individual_data_types={i.individual_id: i.data_types for i in individuals}, add_row=_add_row, - variant_json_fields=['clinvar', 'variantId'], + include_clinvar=True, + include_variant_id=True, variant_attr_fields=['tags'], - mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__saved_variant_json__variantId'), + mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__variant_id'), include_family_name_display=True, include_mondo=True, omit_airtable=True, diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 30871454bd..86350c7a2b 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -620,10 +620,10 @@ 'Full', '', '', 'SR-ES', 'This individual is published in PMID34415322', '', '', '', '', '', '', ], [ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh37', '1', - '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', + '248367227', 'TC', 'T', 'CA1501729', 'RP11', 'ENST00000371839', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', ], [ - 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV', 'GRCh38', '19', + 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV', 'GRCh37', '19', '1912632', 'G', 'C', '', 'OR4G11P', 'ENST00000371839', 'c.586_587delinsTT', 'p.Ala196Leu', 'Heterozygous', '', 'unknown', 'Broad_HG00731_19_1912634', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', 'The following variants are part of the multinucleotide variant 19-1912632-G-C (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', @@ -761,20 +761,20 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur '17-significance', '18-discovery_notes']) self.assertIn([ '1_248367227_HG00731', 'HG00731', 'HG00731', 'RP11', 'Known', 'paternal', - 'Homozygous', 'GRCh37', '1', '248367227', 'TC', 'T', '-', '-', '-', '-', '-', '-', '-'], discovery_file) + 'Homozygous', 'GRCh37', '1', '248367227', 'TC', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', '-'], discovery_file) self.assertIn([ '21_3343353_NA19675_1', 'NA19675_1', 'NA19675', 'RP11', 'Candidate', 'de novo', 'Heterozygous', 'GRCh37', '21', '3343353', 'GAGA', 'G', 'c.375_377delTCT', 'p.Leu126del', 'ENST00000258436.5', '-', '-', '-', 'This individual is published in PMID34415322'], discovery_file) self.assertIn([ - '19_1912633_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh38', '19', + '19_1912633_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh37', '19', '1912633', 'G', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', 'The following variants are part of the multinucleotide variant 19-1912632-G-C ' '(c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T'], discovery_file) self.assertIn([ - '19_1912634_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh38', '19', - '1912634', 'C', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', + '19_1912634_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh37', '19', + '1912634', 'C', 'T', '-', '-', '-', '-', '-', '-', 'The following variants are part of the multinucleotide variant 19-1912632-G-C (c.586_587delinsTT, ' 'p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T'], discovery_file) @@ -784,7 +784,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur response = self.client.get(no_analyst_project_url) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['errors'], - ['Discovery variant(s) 1-248367227-TC-T in family 14 have no associated gene']) + ['Discovery variant(s) 1-248367227-TC-T, MT-14783-T-C in family 14 have no associated gene']) @mock.patch('seqr.views.apis.report_api.GREGOR_DATA_MODEL_URL', MOCK_DATA_MODEL_URL) @mock.patch('seqr.views.apis.report_api.datetime') @@ -936,7 +936,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo '', '', '', '', '', '', ], [ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh37', '1', - '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', + '248367227', 'TC', 'T', 'CA1501729', 'RP11', 'ENST00000371839', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', ]], additional_calls=1) @@ -1347,7 +1347,7 @@ def test_variant_metadata(self): 'alt': 'T', 'chrom': '1', 'ClinGen_allele_ID': 'CA1501729', - 'clinvar': {'alleleId': None, 'clinicalSignificance': '', 'goldStars': None, 'variationId': None}, + 'clinvar': {'alleleId': 12345, 'pathogenicity': 'Uncertain_significance', 'goldStars': None, 'assertions': None, 'conditions': None, 'conflictingPathogenicities': None, 'submitters': None}, 'condition_id': 'MONDO:0044970', 'condition_inheritance': 'Unknown', 'displayName': '2', @@ -1364,6 +1364,7 @@ def test_variant_metadata(self): 'projectGuid': 'R0001_1kg', 'ref': 'TC', 'tags': ['Known gene for phenotype'], + 'transcript': 'ENST00000371839', 'variant_inheritance': 'paternal', 'variant_reference_assembly': 'GRCh37', 'zygosity': 'Homozygous', @@ -1393,7 +1394,7 @@ def test_variant_metadata(self): 'tags': ['Known gene for phenotype'], 'transcript': 'ENST00000371839', 'variant_inheritance': 'unknown', - 'variant_reference_assembly': 'GRCh38', + 'variant_reference_assembly': 'GRCh37', 'variant_type': 'SNV', 'zygosity': 'Heterozygous', } @@ -1415,7 +1416,7 @@ def test_variant_metadata(self): 'alt': 'T', 'chrom': '1', 'ClinGen_allele_ID': 'CA1501729', - 'clinvar': {'alleleId': None, 'clinicalSignificance': '', 'goldStars': None, 'variationId': None}, + 'clinvar': {'alleleId': 12345, 'pathogenicity': 'Uncertain_significance', 'goldStars': None, 'assertions': None, 'conditions': None, 'conflictingPathogenicities': None, 'submitters': None}, 'condition_id': 'OMIM:616126', 'condition_inheritance': 'Autosomal recessive', 'displayName': '12', @@ -1501,7 +1502,7 @@ class LocalReportAPITest(AuthenticationTestCase, ReportAPITest): fixtures = ['users', '1kg_project', 'reference_data', 'report_variants'] ADDITIONAL_FAMILIES = ['F000014_14'] - ADDITIONAL_FINDINGS = ['NA21234_1_248367227'] + ADDITIONAL_FINDINGS = ['NA21234_1_248367227', 'NA21234_MT_14783', 'NA21234_1_249045487_DEL'] HAS_PM_OVERRIDE = True STATS_DATA = { 'projectsCount': {'non_demo': 3, 'demo': 1}, @@ -1528,7 +1529,7 @@ def _test_gregor_export(self, url, *args): class AnvilReportAPITest(AnvilAuthenticationTestCase, ReportAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] HAS_PM_OVERRIDE = False STATS_DATA = { 'projectsCount': {'internal': 1, 'external': 1, 'no_anvil': 1, 'demo': 1}, diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index af850a667c..0044ea276b 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -1,7 +1,11 @@ import logging import json -from django.db.models import Q +from clickhouse_backend.models import ArrayField, StringField +from django.db.models import Q, F + +from clickhouse_search.backend.functions import ArrayFilter, ArrayMap +from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project from seqr.utils.search.utils import backend_specific_call @@ -38,7 +42,7 @@ def saved_variant_data(request, project_guid, variant_guids=None): variant_query = variant_query.filter(varianttag__isnull=get_note_only).distinct() add_locus_list_detail = request.GET.get(INCLUDE_LOCUS_LISTS_PARAM) == 'true' - response = get_variants_response(request, variant_query, add_locus_list_detail=add_locus_list_detail) + response = get_variants_response(request, variant_query, add_locus_list_detail=add_locus_list_detail, genome_version= project.genome_version) if 'individualsByGuid' in response and not family_guids: if 'projectsByGuid' not in response: response['projectsByGuid'] = {project_guid: {}} @@ -73,11 +77,11 @@ def create_saved_variant_handler(request): response = {} if variant_json.get('note'): - _, response = _create_variant_note(saved_variants, variant_json, request.user) + _, response = _create_variant_note(saved_variants, variant_json, request.user, family.project.genome_version) elif variant_json.get('tags'): _update_tags(saved_variants, variant_json, request.user) - response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True)) + response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=family.project.genome_version)) return create_json_response(response) @@ -100,7 +104,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response({'error': 'Note is required'}, status=400) # update saved_variants - note, response = _create_variant_note(saved_variants, request_json, request.user) + note, response = _create_variant_note(saved_variants, request_json, request.user, family.project.genome_version) note_json = get_json_for_variant_note(note) note_json['variantGuids'] = all_variant_guids response.update({ @@ -113,7 +117,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response(response) -def _create_variant_note(saved_variants, note_json, user): +def _create_variant_note(saved_variants, note_json, user, genome_version): note = create_model_from_json(VariantNote, { 'note': note_json.get('note'), 'report': note_json.get('report') or False, @@ -123,13 +127,7 @@ def _create_variant_note(saved_variants, note_json, user): response = {} if note_json.get('saveAsGeneNote'): - main_transcript_id = saved_variants[0].selected_main_transcript_id or saved_variants[0].saved_variant_json.get('mainTranscriptId') - if main_transcript_id: - gene_id = next( - gene_id for gene_id, transcripts in saved_variants[0].saved_variant_json['transcripts'].items() - if any(t['transcriptId'] == main_transcript_id for t in transcripts)) - else: - gene_id = next(gene_id for gene_id in sorted(saved_variants[0].saved_variant_json['transcripts'])) + gene_id = backend_specific_call(_variant_gene_id, _variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -138,6 +136,29 @@ def _create_variant_note(saved_variants, note_json, user): return note, response +def _variant_gene_id(variant, genome_version): + main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') + if main_transcript_id: + return next( + gene_id for gene_id, transcripts in variant.saved_variant_json['transcripts'].items() + if any(t['transcriptId'] == main_transcript_id for t in transcripts)) + return next(gene_id for gene_id in sorted(variant.saved_variant_json['transcripts'])) + + +def _clickhouse_variant_gene_id(variant, genome_version): + if not variant.key: + return _variant_gene_id(variant, genome_version) + if variant.selected_main_transcript_id: + qs = get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap( + ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]), + mapped_expression='x.geneId', output_field=ArrayField(StringField()), + )).annotate(gene_id=F('gene_ids__0')) + else: + annotations = get_annotations_queryset(genome_version, variant.dataset_type, [variant.key]) + qs = annotations.annotate(gene_id=F(f'{annotations.transcript_field}__0__geneId')) + return qs.values_list('gene_id', flat=True)[0] + + @login_and_policies_required def update_variant_note_handler(request, variant_guids, note_guid): note = VariantNote.objects.get(guid=note_guid) @@ -299,7 +320,7 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari @login_and_policies_required def update_saved_variant_json(request, project_guid): - backend_specific_call(lambda: True, _hail_backend_error)() + backend_specific_call(lambda: True, _backend_error, _backend_error)() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) reset_cached_search_results(project) try: @@ -311,8 +332,8 @@ def update_saved_variant_json(request, project_guid): return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids or []}) -def _hail_backend_error(*args, **kwargs): - raise ValueError('Endpoint is disabled for the hail backend') +def _backend_error(*args, **kwargs): + raise ValueError('Endpoint is disabled for current search backend') @login_and_policies_required diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7c23972fce..98b235f636 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -4,7 +4,7 @@ from django.urls.base import reverse -from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Family +from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Family, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, update_saved_variant_json, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler @@ -96,30 +96,40 @@ 'genotypes': {}, 'genomeVersion': '37', 'mainTranscriptId': None, - 'originalAltAlleles': ['A'], - 'populations': {'callset': {'ac': 2, 'af': 0.063, 'an': 32}}, + 'populations': { + 'seqr': {'ac': 0, 'hom': 0}, + 'exac': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': 0.0, 'hemi': 0, 'het': 0, 'hom': 0}, + 'gnomad_exomes': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': 0.0, 'hemi': 0, 'hom': 0}, + 'gnomad_genomes': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': 0.0, 'hemi': 0, 'hom': 0}, + 'topmed': {'ac': 0, 'af': 0.0, 'an': 0, 'het': 0, 'hom': 0}, + }, 'pos': 61413835, - 'predictions': {'cadd': 21.9}, + 'liftedOverGenomeVersion': '38', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'predictions': {'cadd': 21.9, + 'eigen': None, + 'fathmm': None, + 'mpc': None, + 'mut_pred': None, + 'mut_taster': None, + 'polyphen': None, + 'primate_ai': None, + 'revel': None, + 'sift': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None}, 'ref': 'AAAG', 'transcripts': {}, - 'projectGuid': 'R0001_1kg', 'familyGuids': ['F000001_1', 'F000002_2'], 'variantId': '2-61413835-AAAG-A', 'CAID': None, + 'rsid': None, + 'clinvar': None, + 'hgmd': None, } -CREATE_VARIANT_REQUEST_BODY = { - 'searchHash': 'd380ed0fd28c3127d07a64ea2ba907d7', - 'familyGuid': 'F000001_1', - 'tags': [{'name': 'Review', 'metadata': 'a note'}], - 'note': '', - 'functionalData': [], - 'variant': CREATE_VARIANT_JSON, -} - -INVALID_CREATE_VARIANT_REQUEST_BODY = deepcopy(CREATE_VARIANT_REQUEST_BODY) -INVALID_CREATE_VARIANT_REQUEST_BODY['variant']['chrom'] = '27' - class SavedVariantAPITest(object): @@ -138,9 +148,9 @@ def test_saved_variant_data(self): self.assertSetEqual(set(variants.keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID}) variant = variants[VARIANT_GUID] - self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), SAVED_VARIANT_DETAIL_FIELDS) + self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), self.SAVED_VARIANT_DETAIL_FIELDS) fields = {'mainTranscriptId', 'mmeSubmissions'} - fields.update(SAVED_VARIANT_DETAIL_FIELDS) + fields.update(self.SAVED_VARIANT_DETAIL_FIELDS) self.assertSetEqual(set(variant.keys()), fields) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) @@ -165,7 +175,7 @@ def test_saved_variant_data(self): gene_fields = {'locusListGuids'} gene_fields.update(GENE_VARIANT_FIELDS) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953'}) + self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953', 'ENSG00000240361'}) self.assertSetEqual(set(response_json['genesById']['ENSG00000135953'].keys()), gene_fields) self.assertDictEqual( @@ -348,13 +358,25 @@ def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) + create_variant_json = self._format_create_variant(CREATE_VARIANT_JSON) + create_variant_request_body = { + 'searchHash': 'd380ed0fd28c3127d07a64ea2ba907d7', + 'familyGuid': 'F000001_1', + 'tags': [{'name': 'Review', 'metadata': 'a note'}], + 'note': '', + 'functionalData': [], + 'variant': create_variant_json, + } + + invalide_create_variant_request_body = deepcopy(create_variant_request_body) + invalide_create_variant_request_body['variant']['chrom'] = '27' response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - INVALID_CREATE_VARIANT_REQUEST_BODY)) + invalide_create_variant_request_body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Invalid chromosome: 27'}) response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - CREATE_VARIANT_REQUEST_BODY)) + create_variant_request_body)) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) @@ -362,8 +384,8 @@ def test_create_saved_variant(self): saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') variant_json = {'xpos': 2061413835} - variant_json.update(CREATE_VARIANT_JSON) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + variant_json.update(create_variant_json) + self._assert_created_variant(saved_variant, variant_json) variant_json.update({ 'variantGuid': variant_guid, @@ -386,30 +408,43 @@ def test_create_saved_variant(self): # creating again without specifying the guid should not error and should not create a duplicate response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - CREATE_VARIANT_REQUEST_BODY)) + create_variant_request_body)) self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) + def _format_create_variant(self, variant_json, **kwargs): + return variant_json + + def _assert_created_variant(self, saved_variant, variant_json, **kwargs): + for field in ['xpos', 'ref', 'alt']: + self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) + def test_create_saved_sv_variant(self): + # SVs are only supported on build 38 + Project.objects.filter(id=1).update(genome_version='38') + create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - variant_json = { + variant_json = self._format_create_variant({ 'chrom': '2', 'genotypes': {}, - 'genomeVersion': '37', - 'mainTranscriptId': None, - 'populations': {'sv_callset': {'ac': 2, 'af': 0.063, 'an': 32}}, + 'genomeVersion': '38', + 'populations': {'sv_callset': {'ac': 2, 'af': 0.063, 'an': 32, 'het': 0, 'hom': 0}}, 'pos': 61413835, 'end': 61414175, + 'liftedOverChrom': '2', + 'liftedOverGenomeVersion': '37', + 'liftedOverPos': 61413835, + 'rg37LocusEnd': {'contig': '2', 'position': 61414175}, + 'numExon': 0, 'predictions': {'strvctvre': 21.9}, - 'transcripts': {'ENSG00000240361': []}, - 'projectGuid': 'R0001_1kg', + 'transcripts': {'ENSG00000240361': [{'geneId': 'ENSG00000240361', 'majorConsequence': 'LOF'}]}, 'familyGuids': ['F000001_1', 'F000002_2'], 'svType': 'DUP', 'variantId': 'batch_123_DUP', 'acmgClassification': None, - } + }) request_body = { 'familyGuid': 'F000001_1', @@ -430,7 +465,7 @@ def test_create_saved_sv_variant(self): saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') variant_json.update({'xpos': 2061413835}) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES') self.assertEqual(saved_variant.xpos_end, 2061414175) variant_json.update({ @@ -483,14 +518,14 @@ def test_create_saved_compound_hets(self): 'noteGuids': [], 'functionalDataGuids': [], } - expected_compound_het_3_json.update(COMPOUND_HET_3_JSON) + expected_compound_het_3_json.update(self._format_create_variant(COMPOUND_HET_3_JSON, key=None)) expected_compound_het_4_json = { 'variantGuid': new_compound_het_4_guid, 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], } - expected_compound_het_4_json.update(COMPOUND_HET_4_JSON) + expected_compound_het_4_json.update(self._format_create_variant(COMPOUND_HET_4_JSON, key=None)) response_json = response.json() response_compound_het_3_json = response_json['savedVariantsByGuid'][new_compound_het_3_guid] response_compound_het_4_json = response_json['savedVariantsByGuid'][new_compound_het_4_guid] @@ -545,7 +580,7 @@ def test_create_update_and_delete_variant_note(self): self.assertEqual(new_gene_note_response['note'], 'new_variant_note_as_gene_note') # save variant_note as gene_note for user selected main gene - create_variant_note_seetced_gene_url = reverse(create_variant_note_handler, args=[VARIANT_GUID]) + create_variant_note_seetced_gene_url = reverse(create_variant_note_handler, args=[VARIANT_GUID_2]) response = self.client.post(create_variant_note_seetced_gene_url, content_type='application/json', data=json.dumps( {'note': 'new user-selected gene note', 'saveAsGeneNote': True, 'familyGuid': 'F000001_1'} )) @@ -623,7 +658,7 @@ def test_create_partially_saved_compound_het_variant_note(self): 'tagGuids': [], 'functionalDataGuids': [], } - expected_compound_het_5_json.update(COMPOUND_HET_5_JSON) + expected_compound_het_5_json.update(self._format_create_variant(COMPOUND_HET_5_JSON, key=None)) response_json = response.json() response_compound_het_5_json = response_json['savedVariantsByGuid'][compound_het_5_guid] note_guids = response_compound_het_5_json.pop('noteGuids') @@ -995,6 +1030,11 @@ class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): fixtures = ['users', '1kg_project', 'reference_data'] SAVED_VARIANT_RESPONSE_KEYS = SAVED_VARIANT_RESPONSE_KEYS + SAVED_VARIANT_DETAIL_FIELDS = SAVED_VARIANT_DETAIL_FIELDS + + def _assert_created_variant(self, saved_variant, variant_json, **kwargs): + super()._assert_created_variant(saved_variant, variant_json) + self.assertDictEqual(variant_json, saved_variant.saved_variant_json) def assert_no_list_ws_has_al(self, acl_call_count): @@ -1007,9 +1047,10 @@ def assert_no_list_ws_has_al(self, acl_call_count): # Test for permissions from AnVIL only class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] SAVED_VARIANT_RESPONSE_KEYS = {*SAVED_VARIANT_RESPONSE_KEYS, 'totalSampleCounts'} + SAVED_VARIANT_DETAIL_FIELDS = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} def test_saved_variant_data(self, *args): super(AnvilSavedVariantAPITest, self).test_saved_variant_data(*args) @@ -1075,3 +1116,13 @@ def test_update_variant_main_transcript(self): def test_update_variant_acmg_classification(self): super(AnvilSavedVariantAPITest, self).test_update_variant_acmg_classification() assert_no_list_ws_has_al(self, 2) + + def _format_create_variant(self, variant_json, key=123, **kwargs): + return {**variant_json, 'key': key} + + def _assert_created_variant(self, saved_variant, variant_json, dataset_type='SNV_INDEL', **kwargs): + super()._assert_created_variant(saved_variant, variant_json) + self.assertEqual(variant_json['key'], saved_variant.key) + self.assertEqual(dataset_type, saved_variant.dataset_type) + self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) + self.assertDictEqual({}, saved_variant.saved_variant_json) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index f8c639f44a..8e747c3a02 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -1,12 +1,14 @@ from collections import defaultdict from datetime import datetime +from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import PermissionDenied from django.core.mail.message import EmailMessage from django.contrib.auth.models import User -from django.db.models import CharField, F, Value +from django.db.models import CharField, F, Q, Value from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf import json +from clickhouse_search.search import get_clickhouse_keys_for_gene from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission @@ -17,9 +19,10 @@ from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_variants_for_variant_ids +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_utils import create_json_response from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.orm_to_json_utils import get_json_for_matchmaker_submissions, get_json_for_saved_variants,\ add_individual_hpo_details, INDIVIDUAL_DISPLAY_NAME_EXPR, AIP_TAG_TYPE from seqr.views.utils.permissions_utils import analyst_required, user_is_analyst, get_project_guids_user_can_view, \ @@ -57,13 +60,13 @@ def mme_details(request): saved_variants = get_json_for_saved_variants( SavedVariant.objects.filter(matchmakersubmissiongenes__matchmaker_submission__guid__in=submissions_by_guid), - add_details=True, + additional_values={'genomeVersion': F('family__project__genome_version')}, ) response = { 'submissions': list(submissions_by_guid.values()), 'genesById': genes_by_id, - 'savedVariantsByGuid': {s['variantGuid']: s for s in saved_variants}, + 'savedVariantsByGuid': {s['variantGuid']: {**s, 'chrom': get_chrom_pos(s['xpos'])[0] , 'pos': get_chrom_pos(s['xpos'])[1]} for s in saved_variants}, } if user_is_analyst(request.user): response['metrics'] = get_mme_metrics() @@ -111,7 +114,12 @@ def saved_variants_page(request, tag): saved_variant_models = saved_variant_models.filter(family__project__guid__in=get_project_guids_user_can_view(request.user)) if gene: - saved_variant_models = saved_variant_models.filter(saved_variant_json__transcripts__has_key=gene) + gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( + lambda *args: Q(), + lambda *args: Q(), + _saved_variant_with_clickhouse_gene_q, + )(saved_variant_models, gene) + saved_variant_models = saved_variant_models.filter(gene_filter) elif saved_variant_models.count() > MAX_SAVED_VARIANTS: return create_json_response({'error': 'Select a gene to filter variants'}, status=400) @@ -123,6 +131,22 @@ def saved_variants_page(request, tag): return create_json_response(response_json) +def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id): + search_type_keys = saved_variant_models.filter(key__isnull=False).values( + 'dataset_type', genome_version=F('family__project__genome_version'), + ).annotate(keys=ArrayAgg('key', distinct=True)) + has_key_q = None + for agg in search_type_keys: + gene_keys = get_clickhouse_keys_for_gene(gene_id, **agg) + if gene_keys: + q = Q(dataset_type=agg['dataset_type'], family__project__genome_version=agg['genome_version'], key__in=gene_keys) + if has_key_q: + has_key_q |= q + else: + has_key_q = q + return has_key_q or Q() + + @login_and_policies_required def hpo_summary_data(request, hpo_id): data = Individual.objects.filter( @@ -214,7 +238,7 @@ def _load_aip_data(data: dict, user: User): today = datetime.now().strftime('%Y-%m-%d') num_new, num_updated = bulk_create_tagged_variants( - family_variant_data, tag_name=AIP_TAG_TYPE, user=user, load_new_variant_data=_search_new_saved_variants, + family_variant_data, tag_name=AIP_TAG_TYPE, user=user, load_new_variant_data=True, get_metadata=lambda pred: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, ) @@ -229,43 +253,6 @@ def _load_aip_data(data: dict, user: User): }) -FamilyVariantKey = tuple[int, str] - - -def _search_new_saved_variants(family_variant_ids: list[FamilyVariantKey], user: User): - family_ids = set() - variant_families = defaultdict(list) - for family_id, variant_id in family_variant_ids: - family_ids.add(family_id) - variant_families[variant_id].append(family_id) - families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} - - search_variants_by_id = { - v['variantId']: v for v in get_variants_for_variant_ids( - families=families_by_id.values(), variant_ids=variant_families.keys(), user=user, - ) - } - - new_variants = {} - missing = defaultdict(list) - for variant_id, family_ids in variant_families.items(): - variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} - for family_id in family_ids: - family = families_by_id[family_id] - if family.guid in variant['familyGuids']: - new_variants[(family_id, variant_id)] = variant - else: - missing[family.family_id].append(variant_id) - - if missing: - missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] - raise ErrorsWarningsException([ - f"Unable to find the following family's AIP variants in the search backend: {', '.join(missing_summary)}", - ]) - - return new_variants - - ALL_PROJECTS = 'all' GREGOR_CATEGORY = 'gregor' diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index efc041c5f8..5b1393ca6a 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -7,7 +7,7 @@ from seqr.views.apis.summary_data_api import mme_details, success_story, saved_variants_page, hpo_summary_data, \ bulk_update_family_external_analysis, individual_metadata, send_vlm_email -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, PARSED_VARIANTS +from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, PARSED_VARIANTS, SAVED_VARIANT_FIELDS from seqr.models import FamilyAnalysedBy, SavedVariant, VariantTag from settings import AIRTABLE_URL @@ -148,7 +148,7 @@ 'MME': 'Yes', 'family_history': 'Yes', 'genetic_findings_id-1': 'NA21234_1_248367227', - 'num_saved_variants': 1, + 'num_saved_variants': 3, 'paternal_guid': '', 'paternal_id': '', 'phenotype_description': None, @@ -181,6 +181,56 @@ 'transcript-1': None, 'analysis_groups': '', 'ClinGen_allele_ID-1': 'CA1501729', + 'alt-3': None, + 'chrom-3': '1', + 'gene_known_for_phenotype-3': 'Known', + 'phenotype_contribution-3': 'Full', + 'partial_contribution_explained-3': '', + 'pos-3': 249045487, + 'chrom_end-3': '1', + 'pos_end-3': 249045898, + 'ref-3': None, + 'copy_number-3': 1, + 'zygosity-3': 'Heterozygous', + 'variant_reference_assembly-3': 'GRCh38', + 'gene_of_interest-3': None, + 'gene_id-3': None, + 'genetic_findings_id-3': 'NA21234_1_249045487_DEL', + 'hgvsc-3': '', + 'hgvsp-3': '', + 'notes-3': '', + 'seqr_chosen_consequence-3': None, + 'sv_type-3': 'DEL', + 'sv_name-3': 'DEL:chr1:249045487-249045898', + 'validated_name-3': None, + 'variant_inheritance-3': 'unknown', + 'transcript-3': None, + 'ClinGen_allele_ID-3': None, + 'alt-2': 'C', + 'chrom-2': 'MT', + 'gene_known_for_phenotype-2': 'Candidate', + 'phenotype_contribution-2': 'Full', + 'partial_contribution_explained-2': '', + 'pos-2': 14783, + 'chrom_end-2': None, + 'pos_end-2': None, + 'ref-2': 'T', + 'copy_number-2': None, + 'zygosity-2': 'Homoplasmy', + 'variant_reference_assembly-2': 'GRCh38', + 'gene_of_interest-2': None, + 'gene_id-2': 'ENSG00000198727', + 'genetic_findings_id-2': 'NA21234_MT_14783', + 'hgvsc-2': 'c.37T>C', + 'hgvsp-2': 'p.Leu13=', + 'notes-2': '', + 'seqr_chosen_consequence-2': 'synonymous_variant', + 'sv_type-2': None, + 'sv_name-2': None, + 'validated_name-2': None, + 'variant_inheritance-2': 'unknown', + 'transcript-2': 'ENST00000361789.2', + 'ClinGen_allele_ID-2': None, } AIRTABLE_SAMPLE_RECORDS = { @@ -285,6 +335,9 @@ def test_mme_details(self, mock_datetime): self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000240361', 'ENSG00000223972', 'ENSG00000135953'}) self.assertEqual(len(response_json['submissions']), self.NUM_MANAGER_SUBMISSIONS) + self.assertSetEqual(set(next(iter(response_json['savedVariantsByGuid'].values())).keys()), { + 'chrom', 'pos', 'genomeVersion', *SAVED_VARIANT_FIELDS, + }) # Test analyst behavior self.login_analyst_user() @@ -365,7 +418,7 @@ def test_saved_variants_page(self): response = self.client.get('{}?gene=ENSG00000135953'.format(all_tag_url)) self.assertEqual(response.status_code, 200) expected_variant_guids.add('SV0000002_1248367227_r0390_100') - report_variants = {'SV0027168_191912632_r0384_rare', 'SV0027167_191912633_r0384_rare', 'SV0027166_191912634_r0384_rare'} + report_variants = {'SV0027168_191912632_r0384_rare', 'SV0027167_191912633_r0384_rare'} self.assertSetEqual(set(response.json()['savedVariantsByGuid'].keys()), {*report_variants, *expected_variant_guids}) multi_tag_url = reverse(saved_variants_page, args=['Review;Tier 1 - Novel gene and phenotype']) @@ -446,9 +499,8 @@ def test_hpo_summary_data(self): ]) @mock.patch('seqr.views.apis.summary_data_api.datetime') - @mock.patch('seqr.views.apis.summary_data_api.get_variants_for_variant_ids') @mock.patch('seqr.views.apis.summary_data_api.load_uploaded_file') - def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, mock_get_variants_for_variant_ids, mock_datetime): + def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, mock_datetime): mock_created_time = datetime(2023, 12, 5, 20, 16, 1) mock_datetime.now.return_value = mock_created_time @@ -505,7 +557,6 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc } } mock_load_uploaded_file.return_value = aip_upload - mock_get_variants_for_variant_ids.return_value = PARSED_VARIANTS body['dataType'] = 'AIP' response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) @@ -525,7 +576,7 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['errors'], [ - "Unable to find the following family's AIP variants in the search backend: 2 (12-48367227-TC-T)", + "Unable to find the following family's variants in the search backend: 2 (12-48367227-TC-T)", ]) aip_upload['results']['HG00731']['2-103343353-GAGA-G'] = aip_upload['results']['HG00731'].pop('12-48367227-TC-T') @@ -534,7 +585,7 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc self.assertDictEqual(response.json(), {'info': ['Loaded 2 new and 1 updated AIP tags for 2 families']}) new_saved_variant = SavedVariant.objects.get(variant_id='2-103343353-GAGA-G') - self.assertDictEqual(new_saved_variant.saved_variant_json, PARSED_VARIANTS[1]) + self._assert_expected_new_saved_variant(new_saved_variant) aip_tags = VariantTag.objects.filter(variant_tag_type__name='AIP').order_by('id').values( 'metadata', saved_variant_ids=ArrayAgg('saved_variants__id')) @@ -571,6 +622,12 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc self.check_no_analyst_no_access(url) + def _assert_expected_new_saved_variant(self, new_saved_variant): + self.assertEqual(new_saved_variant.xpos, 2103343353) + self.assertEqual(new_saved_variant.family_id, 2) + self.assertEqual(new_saved_variant.ref, 'GAGA') + self.assertEqual(new_saved_variant.alt, 'G') + def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False): self.assertEqual(response.status_code, 200) response_json = response.json() @@ -629,6 +686,7 @@ def test_sample_metadata_export(self): rows = response.json()['rows'] self.assertEqual(len(rows), 3) test_row = next(r for r in rows if r['participant_id'] == 'NA21234') + self.maxDiff = None self.assertDictEqual(test_row, EXPECTED_NO_GENE_SAMPLE_METADATA_ROW) # Test analyst access @@ -780,6 +838,14 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assertEqual(response.status_code, 200) self._has_expected_metadata_response(response, expected_individuals) + @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) + def test_bulk_update_family_external_analysis(self, *args, **kwargs): + super().test_bulk_update_family_external_analysis(*args, **kwargs) + + def _assert_expected_new_saved_variant(self, new_saved_variant): + super()._assert_expected_new_saved_variant(new_saved_variant) + self.assertDictEqual(new_saved_variant.saved_variant_json, PARSED_VARIANTS[1]) + def assert_has_expected_calls(self, users, skip_group_call_idxs=None): calls = [mock.call(user) for user in users] @@ -792,7 +858,7 @@ def assert_has_expected_calls(self, users, skip_group_call_idxs=None): # Test for permissions from AnVIL only class AnvilSummaryDataAPITest(AnvilAuthenticationTestCase, SummaryDataAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = [] HAS_AIRTABLE = True @@ -810,3 +876,22 @@ def test_saved_variants_page(self): ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') + + def _assert_expected_new_saved_variant(self, new_saved_variant): + super()._assert_expected_new_saved_variant(new_saved_variant) + self.assertEqual(new_saved_variant.key, 101) + self.assertEqual(new_saved_variant.dataset_type, 'SNV_INDEL') + self.assertDictEqual(new_saved_variant.genotypes, { + 'I000004_hg00731': { + 'ab': 0, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000004_hg00731', 'sampleType': 'WES', + }, + 'I000005_hg00732': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000005_hg00732', 'sampleType': 'WES', + }, + 'I000006_hg00733': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', + }, + }) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 44013ff6bd..ed699a6649 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -10,6 +10,7 @@ from django.db.models.functions import JSONObject from django.shortcuts import redirect from math import ceil +import re from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory @@ -129,9 +130,10 @@ def query_single_variant_handler(request, variant_id): """Search variants. """ families = Family.objects.filter(guid=request.GET.get('familyGuid')) - check_project_permissions(families.first().project, request.user) + family = families.first() + check_project_permissions(family.project, request.user) - variant = get_single_variant(families, variant_id, user=request.user) + variant = get_single_variant(family, variant_id, user=request.user) response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) @@ -147,7 +149,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo response_json = get_variants_response( request, saved_variants, response_variants=flat_variants, add_all_context=add_all_context, - add_locus_list_detail=add_locus_list_detail) + add_locus_list_detail=add_locus_list_detail, genome_version=families[0].project.genome_version) response_json['searchedVariants'] = variants for saved_variant in response_json['savedVariantsByGuid'].values(): @@ -274,7 +276,7 @@ def export_variants_handler(request, search_hash): variants = _flatten_variants(variants) saved_variants, variants_by_id = _get_saved_variant_models(variants, families) - json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants, add_details=True) + json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=families[0].project.genome_version) saved_variants_by_variant_family = {} for saved_variant in json_saved_variants['savedVariantsByGuid'].values(): @@ -553,9 +555,10 @@ def _parse_lookup_request(request): def variant_lookup_handler(request): parsed_variant_id, variant_id, kwargs = _parse_lookup_request(request) is_sv = not parsed_variant_id + genome_version = kwargs.get('genome_version', GENOME_VERSION_GRCh38) if is_sv: families = _all_genome_version_families( - kwargs.get('genome_version', GENOME_VERSION_GRCh38), request.user, + genome_version, request.user, ) if not families: raise PermissionDenied() @@ -572,7 +575,7 @@ def variant_lookup_handler(request): saved_variants, _ = _get_saved_variant_models(variants, None) if families else (None, None) response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, - add_all_context=True, add_locus_list_detail=True, + add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, ) response['variants'] = variants @@ -632,6 +635,10 @@ def _update_lookup_variant(variant, response): @login_and_policies_required def vlm_lookup_handler(request): parsed_variant_id, _, kwargs = _parse_lookup_request(request) + if parsed_variant_id: + invalid_alleles = [f'"{allele}"' for allele in parsed_variant_id[2:] if not re.fullmatch(r'[ATCG]+', allele)] + if invalid_alleles: + raise InvalidSearchException(f'Unable to search VLM for invalid allele(s): {", ".join(invalid_alleles)}') if not parsed_variant_id: raise InvalidSearchException('VLM lookup is not supported for SVs') return create_json_response({'vlmMatches': vlm_lookup(request.user, *parsed_variant_id, **kwargs)}) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index a076e2a2b8..721d156826 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -605,7 +605,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro expected_search_response = deepcopy(self.EXPECTED_SEARCH_RESPONSE) expected_search_response.update({ 'searchedVariants': COMP_HET_VARAINTS, - 'savedVariantsByGuid': {'SV0000002_1248367227_r0390_100': EXPECTED_SAVED_VARIANT}, + 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, 'genesById': {'ENSG00000233653': EXPECTED_GENE}, 'variantTagsByGuid': { 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, @@ -858,9 +858,8 @@ def test_query_single_variant(self, mock_get_variant): self._assert_expected_single_variant_results_context(response.json(), searchedVariants=[SINGLE_FAMILY_VARIANT]) mock_get_variant.assert_called_with(mock.ANY, '21-3343353-GAGA-G', user=self.collaborator_user) - searched_families = mock_get_variant.call_args.args[0] - self.assertEqual(searched_families.count(), 1) - self.assertEqual(searched_families.first().guid, 'F000001_1') + searched_family = mock_get_variant.call_args.args[0] + self.assertEqual(searched_family.guid, 'F000001_1') mock_get_variant.side_effect = InvalidSearchException('Variant not found') response = self.client.get(url) @@ -924,8 +923,8 @@ def test_variant_lookup(self, mock_variant_lookup): } del expected_variant['familyGenotypes'] expected_body = { - **{k: {} for k in EXPECTED_SEARCH_RESPONSE if k not in { - 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', + **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in { + 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'transcriptsById', }}, **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, 'projectsByGuid': {}, @@ -1004,7 +1003,7 @@ def test_variant_lookup(self, mock_variant_lookup): }}, **EXPECTED_TRANSCRIPTS_RESPONSE, 'omimIntervals': {}, - 'savedVariantsByGuid': {'SV0000002_1248367227_r0390_100': EXPECTED_SAVED_VARIANT}, + 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, 'variantFunctionalDataByGuid': {}, 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], 'projectsByGuid': { @@ -1028,7 +1027,6 @@ def test_variant_lookup(self, mock_variant_lookup): for k in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100']: del expected_body['variantTagsByGuid'][k] - self.maxDiff = None self.assertDictEqual(response.json(), expected_body) mock_variant_lookup.assert_called_with( self.manager_user, ('1', 10439, 'AC', 'A'), genome_version='37', @@ -1084,6 +1082,10 @@ def test_vlm_lookup(self, mock_redis): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'VLM lookup is not supported for SVs'}) + response = self.client.get(f'{base_url}?variantId=8-10439--ATGS') + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Unable to search VLM for invalid allele(s): "", "ATGS"'}) + self.reset_logs() response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -1275,12 +1277,15 @@ def assert_workspace_calls(self, group_call_count, user=None): # Test for permissions from AnVIL only class AnvilVariantSearchAPITest(AnvilAuthenticationTestCase, VariantSearchAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches', 'clickhouse_saved_variants'] EXPECTED_SEARCH_RESPONSE = { **EXPECTED_SEARCH_RESPONSE, 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}}, } + EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'] = { + k: {**v, 'key': mock.ANY, 'mainTranscriptId': mock.ANY} for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() + } def test_query_variants(self, *args): super(AnvilVariantSearchAPITest, self).test_query_variants(*args) diff --git a/seqr/views/react_app_tests.py b/seqr/views/react_app_tests.py index eaedaa7890..45e3fed6b3 100644 --- a/seqr/views/react_app_tests.py +++ b/seqr/views/react_app_tests.py @@ -10,7 +10,7 @@ @mock.patch('seqr.views.react_app.DEBUG', False) class AppPageTest(object): - databases = '__all__' + databases = ['default'] fixtures = ['users'] def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, user_email=None, user_fields=None, ga_token_id=None, anvil_loading_date=None): diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 2a9e287c00..4bc6826e4d 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,5 +1,7 @@ from collections import defaultdict from datetime import datetime + +from clickhouse_backend.models import ArrayField, StringField from django.db.models import F, Q, Value, CharField, Aggregate from django.db.models.functions import Replace from django.contrib.auth.models import User @@ -7,13 +9,15 @@ import requests from typing import Callable, Iterable +from clickhouse_search.backend.functions import ArrayMap, ArrayFilter +from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples +from seqr.utils.search.utils import get_search_samples, backend_specific_call from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY @@ -163,7 +167,7 @@ def parse_anvil_metadata( get_additional_individual_fields: Callable[[Individual, dict], dict] = None, individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, - variant_json_fields: Iterable[str] = None, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, + include_variant_id: bool = False, include_clinvar: bool = False, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False, proband_only_variants: bool = False): @@ -186,7 +190,7 @@ def parse_anvil_metadata( sample_ids.add(sample.sample_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( - list(family_data_by_id.keys()), bool(mme_value), variant_json_fields, variant_attr_fields, + list(family_data_by_id.keys()), bool(mme_value), include_variant_id, include_clinvar, variant_attr_fields, ) condition_map = _get_condition_map(family_data_by_id.values()) @@ -320,12 +324,13 @@ def _get_discovery_notes(variant, gene_variants, omit_parent_mnvs): ] def _get_parsed_saved_discovery_variants_by_family( - families: Iterable[Family], include_metadata: bool, variant_json_fields: list[str], + families: Iterable[Family], include_metadata: bool, include_variant_id: bool, include_clinvar: bool, variant_attr_fields: list[str], ): tag_types = VariantTagType.objects.filter(project__isnull=True, category=DISCOVERY_CATEGORY) annotations = dict( + genome_version=F('family__project__genome_version'), tags=ArrayAgg('varianttag__variant_tag_type__name', distinct=True), notes=ArrayAgg('variantnote__note', distinct=True, filter=Q(variantnote__report=True)), partial_hpo_terms=ArrayAgg('variantfunctionaldata__metadata', distinct=True, filter=Q(variantfunctionaldata__functional_data_tag='Partial Phenotype Contribution')), @@ -336,15 +341,20 @@ def _get_parsed_saved_discovery_variants_by_family( varianttag__variant_tag_type__in=tag_types, family__id__in=families, ).order_by('created_date').distinct().annotate(**annotations) + variant_json_fields = ['genotypes'] + if include_clinvar: + variant_json_fields.append('clinvar') variants = [] gene_ids = set() + variant_json_by_guid = backend_specific_call( + _get_variant_json_by_guid, _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, + )(project_saved_variants, include_clinvar) for variant in project_saved_variants: chrom, pos = get_chrom_pos(variant.xpos) - variant_json = variant.saved_variant_json - main_transcript = _get_variant_main_transcript(variant) - gene_id = main_transcript.get('geneId') - gene_ids.add(gene_id) + variant_json = variant_json_by_guid[variant.guid] + main_transcript = variant_json['main_transcript'] + gene_ids.add(variant_json['gene_id']) sv_type = variant_json.get('svType') variant_type = next( (variant_type for variant_type, has_type in VARIANT_TYPES if has_type(variant.ref, variant.alt)), @@ -359,23 +369,25 @@ def _get_parsed_saved_discovery_variants_by_family( parsed_variant = { 'chrom': 'MT' if chrom == 'M' else chrom, 'pos': pos, - 'variant_reference_assembly': GENOME_VERSION_LOOKUP[variant_json['genomeVersion']], - 'gene_id': gene_id, - 'gene_ids': [gene_id] if gene_id else variant_json.get('transcripts', {}).keys(), + 'variant_reference_assembly': GENOME_VERSION_LOOKUP[variant.genome_version], + 'gene_id': variant_json['gene_id'], + 'gene_ids': variant_json['gene_ids'], 'gene_known_for_phenotype': 'Known' if 'Known gene for phenotype' in variant.tags else 'Candidate', 'phenotype_contribution': phenotype_contribution, 'partial_contribution_explained': partial_hpo_terms.replace(', ', '|'), 'notes': variant.notes, 'sv_type': sv_type, - 'sv_name': (variant_json.get('svName') or '{svType}:chr{chrom}:{pos}-{end}'.format(**variant_json)) if sv_type else None, + 'sv_name': (variant_json.get('svName') or f'{sv_type}:chr{chrom}:{pos}-{variant_json["end"]}') if sv_type else None, 'variant_type': variant_type, 'validated_name': variant.validated_name[0] if variant.validated_name else None, **{k: _get_transcript_field(k, config, main_transcript) for k, config in TRANSCRIPT_FIELDS.items()}, - **{k: variant_json.get(k) for k in ['genotypes'] + (variant_json_fields or [])}, + **{k: variant_json.get(k) for k in variant_json_fields}, **{k: variant_json.get(field) if sv_type else None for k, field in [('chrom_end', 'endChrom'), ('pos_end', 'end')]}, 'ClinGen_allele_ID': variant_json.get('CAID'), **{k: getattr(variant, k) for k in ['family_id', 'ref', 'alt'] + (variant_attr_fields or [])}, } + if include_variant_id: + parsed_variant['variantId'] = variant.variant_id if include_metadata: parsed_variant.update({ 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), @@ -394,6 +406,99 @@ def _get_parsed_saved_discovery_variants_by_family( return saved_variants_by_family +def _get_variant_json_by_guid(saved_variants, *args, **kwargs): + variant_json_by_guid = {} + for v in saved_variants: + main_transcript = _get_variant_main_transcript(v) + gene_id = main_transcript.get('geneId') + gene_ids = [gene_id] if gene_id else v.saved_variant_json.get('transcripts', {}).keys() + variant_json_by_guid[v.guid] = { + **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, + } + + return variant_json_by_guid + + +def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar): + variant_json_by_guid = {} + variant_keys_by_search_type = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + no_key_variants = [] + for v in saved_variants: + if v.key: + end_chrom, end = get_chrom_pos(v.xpos_end) + variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end} + variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id)) + else: + no_key_variants.append(v) + variant_json_by_guid.update(_get_variant_json_by_guid(no_key_variants)) + + for genome_version, keys_by_dataset_type in variant_keys_by_search_type.items(): + for dataset_type, key_map in keys_by_dataset_type.items(): + set_json_func = _set_clickhouse_sv_json if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) else _set_clickhouse_snv_indel_json + set_json_func(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar) + + return variant_json_by_guid + + +def _set_clickhouse_sv_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): + annotations = get_annotations_queryset(genome_version, dataset_type, key_map.keys()).values( + 'key', svType=F('sv_type'), gene_ids=ArrayMap('sorted_gene_consequences', mapped_expression='x.geneId', output_field=ArrayField(StringField())), + ) + defaults = {'CAID': None, 'gene_id': None, 'main_transcript': {}} + if include_clinvar: + defaults['clinvar'] = None + for anns in annotations: + for guid, _ in key_map[anns['key']]: + variant_json_by_guid[guid].update({**anns, **defaults}) + + +def _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): + selected_transcripts = {st for group in key_map.values() for _, st in group if st} + annotations_qs = get_annotations_queryset(genome_version, dataset_type, key_map.keys()) + annotation_values = {field: Value(None, output_field=CharField()) for field in ['CAID', 'svType']} + transcripts_by_key = {} + if dataset_type == Sample.DATASET_TYPE_MITO_CALLS: + annotation_values['all_transcripts'] = _get_main_transcripts_expression( + 'sorted_transcript_consequences', annotations_qs, selected_transcripts, + ) + else: + annotation_values['CAID'] = F('caid') + transcripts_qs = get_transcripts_queryset(genome_version, key_map.keys()) + transcripts_by_key.update(dict(transcripts_qs.values_list( + 'key', _get_main_transcripts_expression('transcripts', transcripts_qs, selected_transcripts), + ))) + + fields = ['key'] + if include_clinvar: + fields.append('clinvar') + annotations_qs = annotations_qs.join_clinvar(key_map.keys()) + + annotations = annotations_qs.values(*fields, **annotation_values) + for anns in annotations: + for guid, selected_main_transcript_id in key_map[anns['key']]: + all_transcripts = anns.pop('all_transcripts', transcripts_by_key.get(anns['key'])) or [] + is_main = (lambda t: t['transcriptId'] == selected_main_transcript_id) if selected_main_transcript_id else ( + lambda t: t['transcriptRank'] == 0 + ) + main_transcript = next((t for t in all_transcripts if is_main(t)), {}) + gene_id = main_transcript.get('geneId') + variant_json_by_guid[guid].update({ + **anns, + 'main_transcript': main_transcript, + 'gene_id': gene_id, + 'gene_ids': [gene_id] if gene_id else [transcript['geneId'] for transcript in all_transcripts], + }) + + +def _get_main_transcripts_expression(field_name, qs, selected_transcripts): + output_field = getattr(qs.model, field_name).field.clone() + output_field.group_by_key = None + return ArrayFilter(field_name, conditions=[ + {'transcriptId': (list(selected_transcripts), 'has({value}, {field})')}, + {'transcriptRank': (0, '{value} = {field}')}, + ], output_field=output_field) + + def _get_variant_main_transcript(variant_model): variant = variant_model.saved_variant_json main_transcript_id = variant_model.selected_main_transcript_id or variant.get('mainTranscriptId') @@ -528,7 +633,7 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_ for row in parsed_rows: del row['genotypes'] - gene_variants = variants_by_gene[row[GENE_COLUMN]] + gene_variants = variants_by_gene[row[GENE_COLUMN]] if row[GENE_COLUMN] else [] notes = row['notes'] or [] if len(gene_variants) > 2: discovery_notes = _get_discovery_notes(row, gene_variants, omit_parent_mnvs) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 67a943fc8f..1f6ecb75dc 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -10,6 +10,7 @@ from guardian.shortcuts import get_users_with_perms, get_groups_with_perms import json +from clickhouse_search.search import get_clickhouse_annotations from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ @@ -411,7 +412,7 @@ def get_json_for_analysis_group(analysis_group, **kwargs): return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) -def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None): +def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None): sv_additional_values = { 'familyGuids': ArrayAgg('family__guid', distinct=True), } @@ -421,7 +422,12 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo additional_fields = [] additional_fields += additional_model_fields or [] if add_details: - additional_fields.append('saved_variant_json') + from seqr.utils.search.utils import backend_specific_call + additional_fields += ['saved_variant_json'] + backend_specific_call( + lambda x: [], + lambda x: [], + lambda gv: ['key', 'genotypes', 'dataset_type'] + ([] if gv else ['family__project__genome_version']), + )(genome_version) results = get_json_for_queryset( saved_variants, guid_key='variantGuid', additional_values=sv_additional_values, @@ -429,12 +435,36 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo ) if add_details: + from seqr.utils.search.utils import backend_specific_call + backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_annotations)(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) return results +def _add_clickhouse_annotations(results, genome_version): + results_by_genome_version_dataset_type = defaultdict(lambda: defaultdict(list)) + for result in results: + dataset_type = result.pop('datasetType') + gv = genome_version or result.pop('familyProjectGenomeVersion') + if result['key']: + results_by_genome_version_dataset_type[gv][dataset_type].append(result) + else: + result.pop('genotypes') + + for gv, grouped_results in results_by_genome_version_dataset_type.items(): + for dataset_type, gv_results in grouped_results.items(): + keys = {result['key'] for result in gv_results} + annotations_by_key = { + annotations['key']: {k: v for k, v in annotations.items() if k not in gv_results[0]} + for annotations in get_clickhouse_annotations(gv, dataset_type, keys) + } + for result in gv_results: + if result['key']: + result.update(annotations_by_key[result['key']]) + + def _format_functional_tags(tags): for tag in tags: name = tag.pop('functionalDataTag') diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index 561b126eb0..4a539a787a 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -11,7 +11,7 @@ TAG_FIELDS, VARIANT_NOTE_FIELDS class JSONUtilsTest(object): - databases = '__all__' + databases = ['default'] def test_json_for_user(self): users = User.objects.all() @@ -107,23 +107,6 @@ def test_json_for_igv_sample(self): self.assertSetEqual(set(json.keys()), IGV_SAMPLE_FIELDS) - def test_json_for_saved_variant(self): - variants = SavedVariant.objects.filter(guid='SV0000001_2103343353_r0390_100') - json = get_json_for_saved_variants(variants)[0] - - self.assertSetEqual(set(json.keys()), SAVED_VARIANT_FIELDS) - self.assertListEqual(json['familyGuids'], ["F000001_1"]) - self.assertEqual(json['variantId'], '21-3343353-GAGA-G') - - fields = set() - fields.update(SAVED_VARIANT_FIELDS) - fields.update(list(variants.first().saved_variant_json.keys())) - json = get_json_for_saved_variants(variants, add_details=True)[0] - self.assertSetEqual(set(json.keys()), fields) - self.assertListEqual(json['familyGuids'], ["F000001_1"]) - self.assertEqual(json['variantId'], '21-3343353-GAGA-G') - self.assertEqual(json['mainTranscriptId'], 'ENST00000258436') - def test_json_for_saved_variants_with_tags(self): variant_guid_1 = 'SV0000001_2103343353_r0390_100' variant_guid_2 = 'SV0000002_1248367227_r0390_100' diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index e0bf9b8cbb..9617f8734b 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -285,6 +285,7 @@ def check_mme_permissions(submission, user): check_project_permissions(project, user) if not (project.is_mme_enabled and not project.is_demo): raise PermissionDenied('Matchmaker is not enabled') + return project def has_case_review_permissions(project, user): if not project.has_case_review: diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index d176cea1a2..98319a2392 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -110,13 +110,13 @@ def add_child_ids(response): family['individualGuids'] = individual_guids_by_family[family['familyGuid']] -def families_discovery_tags(families, project=None): +def families_discovery_tags(families, genome_version, project=None): families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} discovery_tags = get_json_for_saved_variants(SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ), add_details=True) + ), add_details=True, genome_version=genome_version) gene_ids = set() for tag in discovery_tags: diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 507a6eb65c..7a2566e415 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -3,6 +3,8 @@ from copy import deepcopy from datetime import datetime from django.contrib.auth.models import User, Group +from django.core.management import call_command +from django.db import connections, transaction from django.test import TestCase from guardian.shortcuts import assign_perm from io import StringIO @@ -14,13 +16,13 @@ import responses from urllib.parse import quote_plus, urlparse -from seqr.models import Project, CAN_VIEW, CAN_EDIT +from seqr.models import Project, SavedVariant, CAN_VIEW, CAN_EDIT WINDOW_REGEX_TEMPLATE = 'window\.{key}=(?P[^)<]+)' class AuthenticationTestCase(TestCase): - databases = '__all__' + databases = ['default', 'reference_data'] SUPERUSER = 'superuser' ANALYST = 'analyst' PM = 'project_manager' @@ -70,10 +72,6 @@ def setUp(self): self._log_stream = StringIO() logging.getLogger().handlers[0].stream = self._log_stream - @classmethod - def _databases_support_transactions(cls): - return True - @classmethod def setUpTestData(cls): cls.super_user = User.objects.get(username='test_superuser') @@ -516,11 +514,43 @@ def get_group_members_side_effect(user, group, use_sa_credentials=False): return {} -class AnvilAuthenticationTestCase(AuthenticationTestCase): +class DifferentDbTransactionSupportMixin(object): + + @classmethod + def _databases_support_transactions(cls): + return True + + @classmethod + def _rollback_atomics(cls, atomics): + """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" + for db_name in reversed(cls._databases_names()): + if connections[db_name].features.supports_transactions: + transaction.set_rollback(True, using=db_name) + atomics[db_name].__exit__(None, None, None) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + for db_name in cls._databases_names(): + if not connections[db_name].features.supports_transactions: + call_command( + "flush", + verbosity=0, + interactive=False, + database=db_name, + reset_sequences=False, + allow_cascade=False, + inhibit_post_migrate=False, + ) + + +class AnvilAuthenticationTestCase(DifferentDbTransactionSupportMixin, AuthenticationTestCase): + databases = '__all__' ES_HOSTNAME = '' CLICKHOUSE_HOSTNAME = 'testhost' MOCK_AIRTABLE_KEY = 'airflow_access' + SKIP_RESET_VARIANT_JSON = False # mock the terra apis def setUp(self): @@ -563,6 +593,8 @@ def setUp(self): self.mock_get_group_members.side_effect = get_group_members_side_effect self.addCleanup(patcher.stop) super(AnvilAuthenticationTestCase, self).setUp() + if self.CLICKHOUSE_HOSTNAME and not self.SKIP_RESET_VARIANT_JSON: + SavedVariant.objects.filter(key__isnull=False).update(saved_variant_json={}) @classmethod def add_additional_user_groups(cls): @@ -847,8 +879,8 @@ def _get_list_param(call, param): SAVED_VARIANT_FIELDS = {'variantGuid', 'variantId', 'familyGuids', 'xpos', 'ref', 'alt', 'selectedMainTranscriptId', 'acmgClassification'} SAVED_VARIANT_DETAIL_FIELDS = { 'chrom', 'pos', 'genomeVersion', 'liftedOverGenomeVersion', 'liftedOverChrom', 'liftedOverPos', 'tagGuids', - 'functionalDataGuids', 'noteGuids', 'originalAltAlleles', 'genotypes', 'hgmd', 'CAID', - 'transcripts', 'populations', 'predictions', 'rsid', 'genotypeFilters', 'clinvar', 'acmgClassification' + 'functionalDataGuids', 'noteGuids', 'genotypes', 'hgmd', 'CAID', + 'transcripts', 'populations', 'predictions', 'rsid', 'clinvar', 'acmgClassification' } SAVED_VARIANT_DETAIL_FIELDS.update(SAVED_VARIANT_FIELDS) @@ -956,7 +988,6 @@ def _get_list_param(call, param): 'callset': {'af': 0.13, 'ac': 4192, 'an': '32588'}, 'gnomad_genomes': {'af': 0.007}, }, - 'genotypeFilters': 'VQSRTrancheSNP99.95to100.00', 'genotypes': { 'NA19675': { 'sampleId': 'NA19675', @@ -964,7 +995,8 @@ def _get_list_param(call, param): 'gq': 46.0, 'numAlt': 1, 'dp': '50', - 'ad': '14,33' + 'ad': '14,33', + 'filters': ['VQSRTrancheSNP99.95to100.00'], }, 'NA19679': { 'sampleId': 'NA19679', @@ -972,7 +1004,8 @@ def _get_list_param(call, param): 'gq': 99.0, 'numAlt': 0, 'dp': '45', - 'ad': '45,0' + 'ad': '45,0', + 'filters': ['VQSRTrancheSNP99.95to100.00'], } } }, @@ -1010,7 +1043,6 @@ def _get_list_param(call, param): 'variantId': '1-248367227-TC-T', 'transcripts': {'ENSG00000233653': {}}, 'familyGuids': ['F000002_2'], - 'genotypeFilters': '', 'genotypes': {} } ] diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 06584340de..6f35ba9ae1 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -1,19 +1,23 @@ from collections import defaultdict +from clickhouse_backend.models import ArrayField, StringField from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import F, Q, Count +from django.db.models import F, Q, Count, prefetch_related_objects import json import logging import redis from tqdm import tqdm import traceback +from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap +from clickhouse_search.search import get_clickhouse_key_lookup, get_annotations_queryset, get_clickhouse_genotypes from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.utils import get_variants_for_variant_ids, backend_specific_call +from seqr.utils.search.utils import get_variants_for_variant_ids, backend_specific_call, parse_variant_id from seqr.utils.gene_utils import get_genes_for_variants +from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ @@ -31,16 +35,16 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, user_email, **kwargs): +def update_projects_saved_variant_json(projects, user_email, update_function=None, **kwargs): success = {} skipped = {} error = {} updated_variants_by_id = {} logger.info(f'Reloading saved variants in {len(projects)} projects') - for project_id, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): + for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: - updated_saved_variants = update_project_saved_variant_json( - project_id, genome_version, user_email=user_email, family_guids=family_guids, **kwargs) + updated_saved_variants = (update_function or update_project_saved_variant_json)( + project_id, genome_version, user_email=user_email, family_guids=family_guids, project_guid=project_guid, **kwargs) if updated_saved_variants is None: skipped[project_name] = True else: @@ -67,7 +71,7 @@ def update_projects_saved_variant_json(projects, user_email, **kwargs): return updated_variants_by_id -def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None): +def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): saved_variants = SavedVariant.objects.filter( Q(saved_variant_json__genomeVersion__isnull=True) | Q(saved_variant_json__genomeVersion=genome_version.replace('GRCh', '')) @@ -78,10 +82,12 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, datas saved_variants = saved_variants.filter(family__guid__in=family_guids) if dataset_type: saved_variants = saved_variants.filter(**saved_variants_dataset_type_filter(dataset_type)) + elif clickhouse_dataset_type: + saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) return saved_variants -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None): +def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None, **kwargs): saved_variants = get_saved_variants(genome_version, project_id, family_guids, dataset_type).select_related('family') if not saved_variants: @@ -131,18 +137,33 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): ref = variant_json.get('ref') alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - update_json = {'saved_variant_json': variant_json} + variant_id = variant_json.get('variantId', variant_id) + if variant_json.get('key'): + update_json = { + 'key': variant_json['key'], + 'genotypes': variant_json.get('genotypes', {}), + 'dataset_type': _dataset_type(variant_id, variant_json), + } + else: + update_json = {'saved_variant_json': variant_json} return { 'xpos': xpos, 'xpos_end': xpos + var_length, 'ref': ref, 'alt': alt, 'family_id': family_id, - 'variant_id': variant_json.get('variantId', variant_id), + 'variant_id': variant_id, }, update_json -def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=None): +def _dataset_type(variant_id, variant): + if not parse_variant_id(variant_id): + sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES + return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS + + +def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=False): all_family_ids = {family_id for family_id, _ in family_variant_data.keys()} all_variant_ids = {variant_id for _, variant_id in family_variant_data.keys()} @@ -151,11 +172,13 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use for v in SavedVariant.objects.filter(family_id__in=all_family_ids, variant_id__in=all_variant_ids) } + genome_version = project.genome_version if project else Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() + new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: - new_variant_data = load_new_variant_data(new_variant_keys, user) if load_new_variant_data else { - k: v for k, v in family_variant_data.items() if k in new_variant_keys - } + new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) if load_new_variant_data else backend_specific_call( + lambda o, _: o, lambda o, _: o, _get_clickhouse_variant_keys, + )({k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): create_json, update_json = parse_saved_variant_json(variant, family_id, variant_id=variant_id) @@ -172,11 +195,15 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use ).annotate(saved_variant_ids=ArrayAgg('saved_variants__id', ordering='id')) } + variant_genes_by_id = backend_specific_call( + _get_saved_variant_genes, _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, + )(saved_variant_map.values(), genome_version) + update_tags = [] num_new = 0 for key, variant in family_variant_data.items(): updated_tag = _set_updated_tags( - key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user, + key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user, variant_genes_by_id, ) if updated_tag: update_tags.append(updated_tag) @@ -184,12 +211,13 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use num_new += 1 VariantTag.bulk_update_models(user, update_tags, ['metadata']) + return num_new, len(update_tags) def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_var_ids: list[str], saved_variant_map: dict[tuple[int, str], SavedVariant], existing_tags: dict[tuple[int, ...], VariantTag], - tag_type: VariantTagType, user: User): + tag_type: VariantTagType, user: User, variant_genes_by_id: dict[str, list[str]]): variant = saved_variant_map[key] existing_tag = existing_tags.get(tuple([variant.id])) updated_tag = None @@ -207,11 +235,11 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v VariantTag, {'variant_tag_type': tag_type, 'metadata': json.dumps(metadata)}, user) tag.saved_variants.add(variant) - variant_genes = set(variant.saved_variant_json['transcripts'].keys()) + variant_genes = variant_genes_by_id.get(key[1], set()) support_vars = [] for support_id in support_var_ids: support_v = saved_variant_map[(key[0], support_id)] - if variant_genes.intersection(set(support_v.saved_variant_json['transcripts'].keys())): + if variant_genes.intersection(variant_genes_by_id.get(support_id, set())): support_vars.append(support_v) if support_vars: variants = [variant] + support_vars @@ -224,6 +252,92 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v return updated_tag +def _get_saved_variant_genes(variant_models, *args, **kwargs): + return {v.variant_id: set(v.saved_variant_json['transcripts'].keys()) for v in variant_models} + + +def _get_clickhouse_saved_variant_genes(variant_models, genome_version): + variant_genes_by_id = {} + key_id_map = {} + for v in variant_models: + if v.key: + key_id_map[v.key] = v.variant_id + else: + variant_genes_by_id[v.variant_id] = set(v.saved_variant_json['transcripts'].keys()) + qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, key_id_map.keys()) + variant_genes_by_id.update({ + key_id_map[key]: set(gene_ids) for key, gene_ids in qs.values_list( + 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())), + ) + }) + return variant_genes_by_id + + +def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: User, genome_version: str) -> dict[tuple[int, str], dict]: + family_ids = set() + variant_families = defaultdict(list) + for family_id, variant_id in family_variant_ids: + family_ids.add(family_id) + variant_families[variant_id].append(family_id) + families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} + + search_variants_by_id = { + v['variantId']: v for v in backend_specific_call( + get_variants_for_variant_ids, get_variants_for_variant_ids, _get_clickhouse_variants, + )(families=list(families_by_id.values()), variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) + } + + new_variants = {} + missing = defaultdict(list) + for variant_id, family_ids in variant_families.items(): + variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} + for family_id in family_ids: + family = families_by_id[family_id] + if family.guid in variant['familyGuids']: + new_variants[(family_id, variant_id)] = variant + else: + missing[family.family_id].append(variant_id) + + if missing: + missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] + raise ErrorsWarningsException([ + f"Unable to find the following family's variants in the search backend: {', '.join(missing_summary)}", + ]) + + return new_variants + + +def _get_clickhouse_variants(families: list[Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: + variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_ids, reverse=True) + prefetch_related_objects(families, 'project') + families_by_project = defaultdict(list) + for family in families: + families_by_project[family.project.guid].append(family.guid) + variants = [] + samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + for project_guid, family_guids in families_by_project.items(): + genotype_keys = get_clickhouse_genotypes( + project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples, + ) + for key, genotypes in genotype_keys.items(): + variant_id = variant_key_map[key] + chrom, pos, ref, alt = variant_id.split('-') + variants.append({ + 'key': key, 'variantId': variant_id, 'chrom': chrom, 'pos': int(pos), 'ref': ref, 'alt': alt, + 'genotypes': genotypes, 'familyGuids': sorted({g['familyGuid'] for g in genotypes.values()}), + }) + return variants + + +def _get_clickhouse_variant_keys(variant_data: dict[tuple[int, str], dict], genome_version: str) -> dict[tuple[int, str], dict]: + variant_ids = {key[1] for key in variant_data} + variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, list(variant_ids)) + for (_, variant_id), variant in variant_data.items(): + if variant_id in variant_key_map: + variant['key'] = variant_key_map[variant_id] + return variant_data + + def reset_cached_search_results(project, reset_index_metadata=False): try: redis_client = redis.StrictRedis(host=REDIS_SERVICE_HOSTNAME, port=REDIS_SERVICE_PORT, socket_connect_timeout=3) @@ -291,7 +405,7 @@ def _saved_variant_genes_transcripts(variants): def get_omim_intervals_query(variants): - chroms = {v['chrom'] for v in variants if v.get('svType')} + chroms = {v['chrom'] for v in variants if v.get('svType') or v.get('hasSvType')} return Q(phenotype_mim_number__isnull=False, gene__isnull=True, chrom__in=chroms) @@ -384,8 +498,8 @@ def _add_pa_detail(locus_list_gene, locus_list_guid, gene_json): def get_variants_response(request, saved_variants, response_variants=None, add_all_context=False, include_igv=True, - add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False): - response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True) \ + add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False, genome_version=None): + response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) \ if saved_variants is not None else {'savedVariantsByGuid': {}} variants = list(response['savedVariantsByGuid'].values()) if response_variants is None else response_variants diff --git a/settings.py b/settings.py index ca6d230bd1..3762609701 100644 --- a/settings.py +++ b/settings.py @@ -253,6 +253,11 @@ 'PORT': int(os.environ.get('CLICKHOUSE_SERVICE_PORT', '9000')), 'USER': os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse'), 'PASSWORD': os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test'), + 'OPTIONS': { + 'settings': { + 'use_client_time_zone': False, + } + } } DATABASES['clickhouse'] = { **DATABASES['clickhouse_write'], diff --git a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx index 16911a20b4..9493a7936e 100644 --- a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx +++ b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx @@ -15,6 +15,7 @@ import { SNP_FREQUENCIES, MITO_FREQUENCIES, SV_FREQUENCIES, + SV_CALLSET_CRITERIA_MESSAGE, THIS_CALLSET_FREQUENCY, SPLICE_AI_FIELD, SV_IN_SILICO_GROUP, @@ -450,6 +451,16 @@ const NO_ES_SNP_FREQUENCIES = [ }, ] +const NO_ES_SV_FREQUENCIES = [ + SV_FREQUENCIES[0], + { + ...SV_FREQUENCIES[1], + label: 'seqr SV', + skipAf: true, + labelHelp: `Filter by allele count (AC) across all samples in seqr with Structural Variant (SV) calling. ${SV_CALLSET_CRITERIA_MESSAGE}`, + }, +] + export const FREQUENCY_PANEL = { name: 'freqs', headerProps: { @@ -461,11 +472,11 @@ export const FREQUENCY_PANEL = { }, }, esEnabledFields: FREQUENCIES, - fields: [...NO_ES_SNP_FREQUENCIES, ...MITO_FREQUENCIES, ...SV_FREQUENCIES], + fields: [...NO_ES_SNP_FREQUENCIES, ...MITO_FREQUENCIES, ...NO_ES_SV_FREQUENCIES], datasetTypeFields: { [DATASET_TYPE_SNV_INDEL_CALLS]: NO_ES_SNP_FREQUENCIES, [DATASET_TYPE_VARIANT_MITO]: NO_ES_SNP_FREQUENCIES.concat(MITO_FREQUENCIES), - [DATASET_TYPE_VARIANT_SV]: NO_ES_SNP_FREQUENCIES.concat(SV_FREQUENCIES), + [DATASET_TYPE_VARIANT_SV]: NO_ES_SNP_FREQUENCIES.concat(NO_ES_SV_FREQUENCIES), }, esEnabledDatasetTypeFields: { [DATASET_TYPE_SNV_INDEL_CALLS]: SNP_FREQUENCIES, diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index d43256edd5..b93f60411d 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -82,6 +82,11 @@ export const loadFamilyDetails = familyGuid => loadFamilyData( ) export const updateFamily = (values) => { + const urlBase = `/api/family/${values.familyGuid}` + if (values.nestedField) { + return updateEntity(values, RECEIVE_DATA, `${urlBase}/${values.nestedField}`, `${values.nestedField}Guid`) + } + if (values.delete) { return dispatch => new HttpRequestHelper(`/api/project/${values.projectGuid}/delete_families`, (responseJson) => { @@ -89,11 +94,6 @@ export const updateFamily = (values) => { }).post({ families: [values] }) } - const urlBase = `/api/family/${values.familyGuid}` - if (values.nestedField) { - return updateEntity(values, RECEIVE_DATA, `${urlBase}/${values.nestedField}`, `${values.nestedField}Guid`) - } - const familyField = values.familyField ? `_${values.familyField}` : '' return dispatch => new HttpRequestHelper(`${urlBase}/update${familyField}`, (responseJson) => { diff --git a/ui/shared/components/buttons/SearchResultsLink.jsx b/ui/shared/components/buttons/SearchResultsLink.jsx index 37eeea588c..13d8e986e6 100644 --- a/ui/shared/components/buttons/SearchResultsLink.jsx +++ b/ui/shared/components/buttons/SearchResultsLink.jsx @@ -3,7 +3,7 @@ import PropTypes from 'prop-types' import { connect } from 'react-redux' import { navigateSavedHashedSearch } from 'redux/rootReducer' -import { VEP_GROUP_SV, ANY_AFFECTED, FREQUENCIES, TOPMED_FREQUENCY, THIS_CALLSET_FREQUENCY } from 'shared/utils/constants' +import { VEP_GROUP_SV, ANY_AFFECTED, FREQUENCIES, TOPMED_FREQUENCY, THIS_CALLSET_FREQUENCY, SV_CALLSET_FREQUENCY } from 'shared/utils/constants' import { ButtonLink } from '../StyledComponents' const SearchResultsLink = ({ @@ -52,7 +52,8 @@ const INITIAL_GENE_SEARCH = { ...FREQUENCIES.filter(({ name }) => name !== TOPMED_FREQUENCY).reduce( (acc, { name }) => ({ ...acc, [name]: { af: 0.03 } }), {}, ), - [THIS_CALLSET_FREQUENCY]: { ac: 3000 }, // TODO use migrated value for 0.03 cutoff + [THIS_CALLSET_FREQUENCY]: { ac: 3000 }, + [SV_CALLSET_FREQUENCY]: { ac: 300 }, }, qualityFilter: { min_gq: 40, min_ab: 10 }, }