diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 2bd6461c6d..d2f166764c 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -4,6 +4,7 @@ from clickhouse_search.backend.fields import NestedField + class Array(Func): function = 'array' @@ -124,9 +125,9 @@ def as_sql(self, compiler, connection, *args, **kwargs): class ArrayIndex(IndexTransform): - def __init__(self, index, *args, **kwargs): + def __init__(self, index, *args, base_field=None, **kwargs): super().__init__(index+1, None, *args, **kwargs) - self.base_field = self.source_expressions[0].output_field.base_field + self.base_field = base_field or self.source_expressions[0].output_field.base_field @ArrayField.register_lookup diff --git a/seqr/utils/search/constants.py b/clickhouse_search/constants.py similarity index 95% rename from seqr/utils/search/constants.py rename to clickhouse_search/constants.py index ae6117128b..aaba9464b9 100644 --- a/seqr/utils/search/constants.py +++ b/clickhouse_search/constants.py @@ -1,10 +1,6 @@ from seqr.models import Individual -VCF_FILE_EXTENSIONS = ('.vcf', '.vcf.gz', '.vcf.bgz') - MAX_VARIANTS = 10000 -MAX_EXPORT_VARIANTS = 1000 -MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 XPOS_SORT_KEY = 'xpos' PATHOGENICTY_SORT_KEY = 'pathogenicity' @@ -65,7 +61,6 @@ SV_CONSEQUENCES_FIELD = 'structural_consequence' SV_TYPE_FILTER_FIELD = 'structural' SV_ANNOTATION_TYPES = {SV_CONSEQUENCES_FIELD, SV_TYPE_FILTER_FIELD, NEW_SV_FIELD} -ALL_DATA_TYPES = 'ALL' CLINVAR_KEY = 'clinvar' CLINVAR_PATH_FILTER = 'pathogenic' diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index d453214bc8..69197c9a56 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -94,6 +94,30 @@ [1, ["frameshift_variant"], "ENSG00000228198"] ] } +}, { + "model": "clickhouse_search.variantdetailsgrch37snvindel", + "pk": 113, + "fields": { + "key": 113, + "variant_id": "15-62456358-A-C", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "transcripts": [] + } +}, { + "model": "clickhouse_search.variantdetailsgrch37snvindel", + "pk": 114, + "fields": { + "key": 114, + "variant_id": "15-62456406-G-A", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "transcripts": [] + } }, { "model": "clickhouse_search.entriesgrch37snvindel", "pk": 101, diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index cb9b677255..82508b5e00 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -88,6 +88,18 @@ "sorted_motif_feature_consequences": [], "sorted_regulatory_feature_consequences": [] } +}, { + "model": "clickhouse_search.variantssnvindel", + "pk": 23, + "fields": { + "key": 23, + "sorted_transcript_consequences": [ + [null, 1, ["missense_variant"], false, null, "ENSG00000227232", false], + [null, 1, ["intron_variant"], false, null, "ENSG00000268903", false] + ], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } }, { "model": "clickhouse_search.keylookupsnvindel", "pk": "1-10439-AC-A", @@ -125,6 +137,12 @@ "key": 22 } }, { + "model": "clickhouse_search.keylookupsnvindel", + "pk": "21-3343353-GAGA-G", + "fields": { + "key": 23 + } +},{ "model": "clickhouse_search.clinvarallvariantssnvindel", "pk": "1-10439-AC-A", "fields": { @@ -860,6 +878,24 @@ ["NA21654", 0, 0, 0, 24] ] } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 23, + "fields": { + "key": 23, + "project_guid": "R0001_1kg", + "family_guid": "F000001_1", + "sample_type": "WES", + "xpos": 21003343353, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [2], + "filters": [], + "sign": 1, + "calls": [ + ["NA19678", 1, 0, 0.16, 45] + ] + } }, { "model": "clickhouse_search.variantsgrch37snvindel", "pk": 10, @@ -1418,9 +1454,9 @@ "geneId_ids": [76], "sign": 1, "calls": [ - ["NA21234", 2, 0, 99, false, true, null], ["NA21987", 1, 1, 99, false, false, 2], - ["NA21654", 1, 1, 99, false, false, 2] + ["NA21654", 1, 1, 99, false, false, 2], + ["NA21234", 2, 0, 99, false, true, null] ] } }, { @@ -1589,9 +1625,9 @@ "filters": [], "sign": 1, "calls": [ - ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false], + ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false], ["HG00732", null, null, null, null, null, null, null, [], null, null, null], - ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] + ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] ] } }, { diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json index 8955cd937a..54383fa7bd 100644 --- a/clickhouse_search/fixtures/clickhouse_transcripts.json +++ b/clickhouse_search/fixtures/clickhouse_transcripts.json @@ -235,5 +235,48 @@ ] ] } +}, { + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 23, + "fields": { + "key": 23, + "variant_id": "21-3343353-GAGA-G", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "transcripts": [ + [ + [null], "T/I", "protein_coding", 1, "aCc/aTc", ["missense_variant"], [1, 12], "ENSG00000227232", + "ENST00000257261.10:c.131C>T", "ENSP00000257261.6:p.Thr44Ile", [null, null], [null, []], "missense_variant", + null, null, "NM_001281501.1", [false], "ENST00000257261", 0, [null, null, null, [ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null + ], null] + ], + [ + [null], null, "protein_coding", null, null, ["intron_variant"], [null, null], "ENSG00000268903", + "ENST00000426137.1:c.-64+100G>A", null, [1, 5], [null, []], "intron_variant", null, null, null, [false], + "ENST00000426137", 0, [null, null, null, [ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null + ], null] + ] + ] + } +}, { + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "transcripts": [] + } } ] \ No newline at end of file diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 1e137a25ee..8f14c1a289 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -99,7 +99,7 @@ def parse_positions(classified_record_node: xml.etree.ElementTree.Element) -> di } return positions -def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element) -> [str, list[str]]: +def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element, allele_id: int, unenumerated_value_alerts: list) -> [str, list[str]]: pathogenicity_node = classified_record_node.find( 'Classifications/GermlineClassification/Description', ) @@ -133,15 +133,23 @@ def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.Element assertions = [a.strip() for a in pathogenicity_string.split(';')] enumerated_assertions = set(CLINVAR_ASSERTIONS) + filtered_assertions = [] for assertion in assertions: if assertion not in enumerated_assertions: - raise CommandError(f'Found an un-enumerated clinvar assertion: {assertion}') - - return pathogenicity, assertions + unenumerated_value_alerts.append({ + 'type': 'Assertion', + 'allele_id': allele_id, + 'value': assertion, + }) + else: + filtered_assertions.append(assertion) + return pathogenicity, filtered_assertions def parse_conflicting_pathogenicities( classified_record_node: xml.etree.ElementTree.Element, pathogenicity: str, + allele_id: int, + unenumerated_value_alerts: list, ) -> list[[str, int]]: if pathogenicity == CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY: conflicting_pathogenicities_node = classified_record_node.find( @@ -159,10 +167,17 @@ def parse_conflicting_pathogenicities( conflicting_pathogenicities_node.text ) enumerated_pathogenicities = set(CLINVAR_PATHOGENICITIES) - for (pathogenicity, _) in conflicting_pathogenicities: + filtered_pathogenicities = [] + for (pathogenicity, count) in conflicting_pathogenicities: if pathogenicity not in enumerated_pathogenicities: - raise CommandError(f'Found an un-enumerated conflicting pathogenicity: {pathogenicity}') - return conflicting_pathogenicities + unenumerated_value_alerts.append({ + 'type': 'Conflicting Pathogenicity', + 'allele_id': allele_id, + 'value': pathogenicity, + }) + else: + filtered_pathogenicities.append((pathogenicity, count)) + return filtered_pathogenicities def parse_gold_stars(classified_record_node: xml) -> Optional[int]: review_status_node = classified_record_node.find( @@ -189,7 +204,7 @@ def parse_submitters_and_conditions(classified_record_node: xml) -> [list[str], }) return submitters, conditions -def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]: +def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, unenumerated_value_alerts: Optional[list] = None) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]: # Cannot use regular bool-falseyness here, as: # "An element with no child elements (even if it exists and has text) will be falsey." classified_record_node = elem.find('ClassifiedRecord') @@ -202,8 +217,8 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str) if not positions: return None, None, None - pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node) - conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity) + pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node, allele_id, unenumerated_value_alerts) + conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity, allele_id, unenumerated_value_alerts) gold_stars = parse_gold_stars(classified_record_node) submitters, conditions = parse_submitters_and_conditions(classified_record_node) # Note: this manipulation to an enumerated pathogenicty happens after we parse conflicting pathogenicities. @@ -247,6 +262,7 @@ def handle(self, *args, **options): ClinvarAllVariantsSnvIndel: [], ClinvarAllVariantsMito: [], } + unenumerated_value_alerts = [] new_version = None with tempfile.TemporaryDirectory() as tmpdir: with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \ @@ -276,7 +292,7 @@ def handle(self, *args, **options): ) # Handle parsing variants if event == 'end' and elem.tag == 'VariationArchive' and new_version: - for obj in extract_variant_info(elem, new_version): + for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): for model, batch in model_to_batch.items(): if isinstance(obj, model): batch.append(obj) @@ -316,4 +332,10 @@ def handle(self, *args, **options): version=new_version ) slack_message = f'Successfully updated Clinvar ClickHouse tables to {new_version}.' + if unenumerated_value_alerts: + slack_message += f'\n\nFound {len(unenumerated_value_alerts)} unenumerated value(s) during parsing:' + for alert in unenumerated_value_alerts[:10]: + slack_message += f"\n- {alert['type']}: '{alert['value']}' (Allele ID: {alert['allele_id']})" + if len(unenumerated_value_alerts) > 10: + slack_message += f"\n• ... and {len(unenumerated_value_alerts) - 10} more" safe_post_to_slack(SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, slack_message) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 49a6d0c0d8..2569dfc3a8 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -14,156 +14,10 @@ BATCH_SIZE = 10000 -SV_DROPPED_IDS = { - 'cluster_6_last_call_cnv_17479_DUP', 'cluster_1_last_call_cnv_30127_DEL', 'cluster_19_COHORT_cnv_23176_DEL', - 'phase2_DEL_chrX_1149', 'prefix_121357_DUP', 'prefix_73945_DEL', 'phase2_DUP_chr9_1663', 'phase2_CPX_chr20_4', - 'phase2_INV_chr19_1', 'cohort_2911.chr2.final_cleanup_BND_chr2_3805', 'prefix_136453_DEL', 'prefix_283065_DEL', - 'phase2_CPX_chr1_27', 'cohort_2911.chr1.final_cleanup_BND_chr1_3716', 'cohort_2911.chr1.final_cleanup_DEL_chr1_8598', - 'cohort_2911.chr10.final_cleanup_BND_chr10_174', 'cohort_2911.chr10.final_cleanup_DUP_chr10_3475','prefix_100035_DEL', - 'cohort_2911.chr11.final_cleanup_DEL_chr11_885', 'cohort_2911.chr11.final_cleanup_DUP_chr11_2292', - 'cohort_2911.chr12.final_cleanup_DEL_chr12_4527', 'cohort_2911.chr13.final_cleanup_DEL_chr13_2215', 'prefix_100069_DEL', - 'cohort_2911.chr11.final_cleanup_DEL_chr11_8856', 'cohort_2911.chr14.final_cleanup_DEL_chr14_5833', - 'cohort_2911.chr16.final_cleanup_DEL_chr16_3', 'cohort_2911.chr2.final_cleanup_BND_chr2_197', - 'cohort_2911.chr2.final_cleanup_BND_chr2_88', 'cohort_2911.chr4.final_cleanup_BND_chr4_1988', 'phase2_BND_chr4_1503', - 'prefix_187707_DUP', 'phase2_DEL_chrX_1156', 'phase2_INV_chr13_1', 'prefix_104037_DUP', 'prefix_101570_DEL', - 'prefix_107442_DEL', 'prefix_110663_DUP', 'prefix_116075_DEL', 'prefix_116353_DEL', 'prefix_131840_DEL', - 'prefix_158993_DUP', 'prefix_15755_DEL', 'prefix_176502_DUP', 'prefix_186412_DUP', 'prefix_186413_DUP', - 'prefix_232873_DUP', 'prefix_232874_DUP', 'prefix_232877_DUP', 'prefix_28289_DUP', 'prefix_31304_DUP', - 'prefix_51597_DUP', 'prefix_190102_DEL', 'prefix_51608_DUP', 'prefix_221407_DEL', 'prefix_221408_DEL', - 'prefix_255688_DUP', 'prefix_37290_DEL', 'prefix_260433_DEL', 'prefix_72862_DEL', 'prefix_98188_DEL', - 'prefix_90352_DUP', 'prefix_107497_DUP', 'phase2_DEL_chr1_6031', 'prefix_121381_DEL', 'prefix_202121_DUP', - 'cohort_2911.chr2.final_cleanup_BND_chr2_889', 'cohort_2911.chr2.final_cleanup_DUP_chr2_3932', - 'cohort_2911.chr2.final_cleanup_DUP_chr2_7490', 'cohort_2911.chr2.final_cleanup_INS_chr2_1130', - 'cohort_2911.chr22.final_cleanup_DUP_chr22_1511', 'cohort_2911.chr3.final_cleanup_BND_chr3_268', - 'cohort_2911.chr3.final_cleanup_DEL_chr3_10060', 'cohort_2911.chr3.final_cleanup_DUP_chr3_2896', - 'cohort_2911.chr5.final_cleanup_BND_chr5_884', 'cohort_2911.chr5.final_cleanup_DEL_chr5_5718', - 'cohort_2911.chr6.final_cleanup_BND_chr6_2309', 'cohort_2911.chr6.final_cleanup_CPX_chr6_38', - 'cohort_2911.chr7.final_cleanup_BND_chr7_1589', 'cohort_2911.chr7.final_cleanup_BND_chr7_1731', - 'cohort_2911.chr7.final_cleanup_BND_chr7_1751', 'cohort_2911.chr7.final_cleanup_DEL_chr7_9736', - 'cohort_2911.chr7.final_cleanup_DUP_chr7_5963', 'cohort_2911.chr9.final_cleanup_BND_chr9_1042', - 'cohort_2911.chrX.final_cleanup_BND_chrX_236', 'cohort_2911.chrX.final_cleanup_BND_chrX_370', - 'cohort_2911.chrX.final_cleanup_BND_chrX_482', 'cohort_2911.chrX.final_cleanup_BND_chrX_501', - 'cohort_2911.chrX.final_cleanup_BND_chrX_569', 'cohort_2911.chrX.final_cleanup_DEL_chrX_1911', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1912', 'cohort_2911.chrX.final_cleanup_DEL_chrX_1916', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1958', 'cohort_2911.chrX.final_cleanup_DEL_chrX_4003', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_4372', 'cohort_2911.chrX.final_cleanup_DUP_chrX_2709', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_631', 'phase2_BND_chr11_47', 'phase2_BND_chr11_810', 'phase2_BND_chr1_1064', - 'phase2_BND_chr1_143', 'phase2_BND_chr1_28', 'phase2_BND_chr2_2776', 'phase2_BND_chr3_2305', 'phase2_BND_chr4_1271', - 'phase2_BND_chr7_987', 'phase2_BND_chrX_310', 'phase2_CPX_chr11_67', 'phase2_CPX_chr1_191', 'phase2_CPX_chr20_50', - 'phase2_CPX_chr2_178', 'phase2_CPX_chr3_12', 'phase2_CPX_chrX_49', 'phase2_CTX_chr3_1', 'phase2_DEL_chr11_6013', - 'phase2_DEL_chr11_61', 'phase2_DEL_chr11_64', 'phase2_DEL_chr11_6470', 'phase2_DEL_chr11_70', 'phase2_DEL_chr11_72', - 'phase2_DEL_chr12_1092', 'phase2_DEL_chr12_5745', 'phase2_DEL_chr14_304', 'phase2_DEL_chr14_4456', - 'phase2_DEL_chr16_3480', 'phase2_DEL_chr17_2166', 'phase2_DEL_chr19_3', 'phase2_DEL_chr19_3801', 'phase2_DEL_chr19_4089', - 'phase2_DEL_chr1_10289', 'phase2_DEL_chr1_1859', 'phase2_DEL_chr1_564', 'phase2_DEL_chr1_6442', 'phase2_DEL_chr1_702', - 'phase2_DEL_chr1_825', 'phase2_DEL_chr1_8985', 'phase2_DEL_chr20_3055', 'phase2_DEL_chr2_11622', 'phase2_DEL_chr2_246', - 'phase2_DEL_chr2_6560', 'phase2_DEL_chr2_8969', 'phase2_DEL_chr3_3865', 'phase2_DEL_chr5_3479', 'phase2_DEL_chr5_3841', - 'phase2_DEL_chr5_5773', 'phase2_DEL_chr5_837', 'phase2_DEL_chr5_8866', 'phase2_DEL_chr5_9284', 'phase2_DEL_chr5_940', - 'phase2_DEL_chr6_3310', 'phase2_DEL_chr6_3783', 'phase2_DEL_chr7_3994', 'phase2_DEL_chr7_7308', 'phase2_DEL_chr7_8285', - 'phase2_DEL_chr8_7050', 'phase2_DEL_chrX_1692', 'phase2_DEL_chrX_1711', 'phase2_DEL_chrX_2006', 'phase2_DEL_chrX_2570', - 'phase2_DEL_chrX_4225', 'phase2_DUP_chr11_2031', 'phase2_DUP_chr13_2894', 'phase2_DUP_chr15_1044', 'phase2_DUP_chr16_56', - 'phase2_DUP_chr1_4', 'phase2_DUP_chr1_4156', 'phase2_DUP_chr2_8878', 'phase2_DUP_chr4_2375', 'phase2_DUP_chr4_486', - 'phase2_DUP_chr5_1272', 'phase2_DUP_chr5_6239', 'phase2_DUP_chr6_12', 'phase2_DUP_chr7_235', 'phase2_DUP_chr7_2872', - 'phase2_DUP_chr7_6068', 'phase2_DUP_chr7_6226', 'phase2_DUP_chrX_1009', 'phase2_DUP_chrX_1597', 'phase2_DUP_chrX_3449', - 'phase2_INS_chr11_1083', 'phase2_INS_chr14_995', 'phase2_INS_chr15_462', 'phase2_INS_chr17_887', 'phase2_INS_chr18_112', - 'phase2_INS_chr4_8', 'phase2_INS_chr5_14', 'phase2_INS_chr7_8', 'phase2_INS_chr7_985', 'phase2_INS_chrX_634', - 'prefix_101656_DUP', 'prefix_103772_DUP', 'prefix_104848_DUP', 'prefix_104955_DEL', 'prefix_105106_DEL', - 'prefix_10566_DEL', 'prefix_105762_DEL', 'prefix_105767_DUP', 'prefix_106133_DEL', 'prefix_106350_DUP', - 'prefix_107375_DUP', 'prefix_107449_DEL', 'prefix_107998_DEL', 'prefix_108257_DEL', 'prefix_109835_DEL', - 'prefix_11007_DEL', 'prefix_110195_DUP', 'prefix_112228_DEL', 'prefix_113169_DUP', 'prefix_114542_DEL', - 'prefix_116109_DUP', 'prefix_116684_DEL', 'prefix_11729_DUP', 'prefix_118876_DUP', 'prefix_120303_DEL', - 'prefix_120655_DEL', 'prefix_120715_DUP', 'prefix_120732_DUP', 'prefix_120770_DEL', 'prefix_121356_DEL', - 'prefix_122104_DUP', 'prefix_122891_DEL', 'prefix_123043_DEL', 'prefix_123043_DEL', 'prefix_123043_DEL', - 'prefix_12347_DEL', 'prefix_1243_DUP', 'prefix_124627_DEL', 'prefix_1250_DUP', 'prefix_125972_DEL', 'prefix_127583_DEL', - 'prefix_127702_DEL', 'prefix_13002_DEL', 'prefix_131133_DUP', 'prefix_131619_DUP', 'prefix_131832_DEL', - 'prefix_132292_DUP', 'prefix_132327_DEL', 'prefix_132903_DUP', 'prefix_133314_DEL', 'prefix_133771_DEL', - 'prefix_133965_DEL', 'prefix_133974_DEL', 'prefix_134970_DUP', 'prefix_134975_DUP', 'prefix_134982_DUP', - 'prefix_137070_DEL', 'prefix_137352_DUP', 'prefix_137515_DUP', 'prefix_137824_DEL', 'prefix_141384_DUP', - 'prefix_141673_DEL', 'prefix_142235_DUP', 'prefix_14236_DUP', 'prefix_142_DUP', 'prefix_143105_DUP', 'prefix_144640_DEL', - 'prefix_145910_DEL', 'prefix_146365_DEL', 'prefix_146774_DUP', 'prefix_146964_DUP', 'prefix_147003_DUP', - 'prefix_150029_DUP', 'prefix_15016_DEL', 'prefix_151274_DEL', 'prefix_153605_DEL', 'prefix_155212_DUP', - 'prefix_156514_DUP', 'prefix_158384_DUP', 'prefix_158650_DEL', 'prefix_158664_DEL', 'prefix_160277_DUP', - 'prefix_161748_DEL', 'prefix_162169_DEL', 'prefix_162239_DEL', 'prefix_162270_DEL', 'prefix_162738_DEL', - 'prefix_162951_DUP', 'prefix_163088_DEL', 'prefix_163102_DEL', 'prefix_163246_DEL', 'prefix_163355_DEL', - 'prefix_163854_DEL', 'prefix_164063_DUP', 'prefix_164549_DEL', 'prefix_164556_DEL', 'prefix_164633_DUP', - 'prefix_165128_DUP', 'prefix_168109_DUP', 'prefix_168895_DUP', 'prefix_168897_DEL', 'prefix_169791_DUP', - 'prefix_170374_DUP', 'prefix_170705_DEL', 'prefix_172571_DUP', 'prefix_172946_DEL', 'prefix_17321_DUP', - 'prefix_173394_DEL', 'prefix_173845_DUP', 'prefix_174288_DUP', 'prefix_174291_DUP', 'prefix_174292_DUP', - 'prefix_17444_DEL', 'prefix_17444_DEL', 'prefix_174517_DUP', 'prefix_175277_DUP', 'prefix_177361_DEL', 'prefix_17764_DEL', - 'prefix_177659_DEL', 'prefix_177950_DEL', 'prefix_178465_DEL', 'prefix_178540_DUP', 'prefix_179099_DUP', - 'prefix_180040_DEL', 'prefix_180738_DEL', 'prefix_180819_DEL', 'prefix_181154_DUP', 'prefix_182745_DEL', - 'prefix_182778_DUP', 'prefix_183768_DUP', 'prefix_184019_DEL', 'prefix_184139_DUP', 'prefix_184140_DUP', - 'prefix_184914_DUP', 'prefix_185204_DEL', 'prefix_185484_DUP', 'prefix_185763_DUP', 'prefix_190154_DUP', - 'prefix_191534_DUP', 'prefix_191897_DEL', 'prefix_191899_DEL', 'prefix_192723_DEL', 'prefix_192753_DEL', - 'prefix_192789_DEL', 'prefix_192810_DUP', 'prefix_192814_DEL', 'prefix_192815_DUP', 'prefix_192893_DEL', - 'prefix_193101_DUP', 'prefix_19341_DUP', 'prefix_19359_DUP', 'prefix_19494_DEL', 'prefix_19514_DEL', 'prefix_195407_DUP', - 'prefix_195410_DUP', 'prefix_196172_DUP', 'prefix_196462_DEL', 'prefix_196930_DUP', 'prefix_198944_DEL', - 'prefix_199367_DEL', 'prefix_200326_DEL', 'prefix_201581_DUP', 'prefix_202196_DEL', 'prefix_203149_DUP', - 'prefix_204824_DUP', 'prefix_206046_DUP', 'prefix_20681_DUP', 'prefix_207500_DEL', 'prefix_210512_DEL', - 'prefix_21173_DUP', 'prefix_212735_DUP', 'prefix_215162_DUP', 'prefix_21548_DUP', 'prefix_215773_DEL', 'prefix_2163_DUP', - 'prefix_217013_DEL', 'prefix_217024_DUP', 'prefix_217326_DUP', 'prefix_217327_DUP', 'prefix_21741_DEL', - 'prefix_222096_DUP', 'prefix_22259_DUP', 'prefix_223605_DUP', 'prefix_225452_DUP', 'prefix_22721_DEL', 'prefix_228404_DEL', - 'prefix_22880_DEL', 'prefix_230711_DEL', 'prefix_232671_DUP', 'prefix_232878_DUP', 'prefix_232948_DEL', 'prefix_234881_DEL', - 'prefix_235365_DUP', 'prefix_236208_DUP', 'prefix_237157_DEL', 'prefix_237619_DUP', 'prefix_237805_DEL', - 'prefix_237917_DUP', 'prefix_23794_DEL', 'prefix_238329_DUP', 'prefix_23844_DEL', 'prefix_238575_DEL', 'prefix_238946_DEL', - 'prefix_238949_DEL', 'prefix_239129_DEL', 'prefix_23925_DEL', 'prefix_240064_DUP', 'prefix_240065_DUP', 'prefix_240066_DUP', - 'prefix_240067_DUP', 'prefix_240075_DUP', 'prefix_240077_DUP', 'prefix_240079_DUP', 'prefix_240080_DUP', 'prefix_240082_DUP', - 'prefix_240084_DUP', 'prefix_240087_DUP', 'prefix_240088_DUP', 'prefix_240090_DUP', 'prefix_240091_DUP', 'prefix_240098_DUP', - 'prefix_240102_DUP', 'prefix_240103_DUP', 'prefix_240104_DUP', 'prefix_240107_DUP', 'prefix_240110_DUP', 'prefix_240116_DUP', - 'prefix_240168_DEL', 'prefix_240306_DUP', 'prefix_242363_DUP', 'prefix_242364_DUP', 'prefix_242485_DUP', 'prefix_24445_DEL', - 'prefix_244719_DEL', 'prefix_244925_DUP', 'prefix_245163_DUP', 'prefix_245178_DEL', 'prefix_245675_DUP', 'prefix_245708_DUP', - 'prefix_245785_DUP', 'prefix_246638_DEL', 'prefix_246946_DUP', 'prefix_247619_DUP', 'prefix_247835_DEL', 'prefix_249326_DUP', - 'prefix_250124_DEL', 'prefix_251066_DEL', 'prefix_251476_DUP', 'prefix_251879_DEL', 'prefix_252006_DEL', 'prefix_25227_DUP', - 'prefix_253446_DEL', 'prefix_254368_DEL', 'prefix_254594_DUP', 'prefix_25463_DEL', 'prefix_254654_DUP', 'prefix_255337_DEL', - 'prefix_255338_DEL', 'prefix_255699_DEL', 'prefix_255947_DEL', 'prefix_257636_DEL', 'prefix_260980_DEL', 'prefix_261010_DEL', - 'prefix_262712_DEL', 'prefix_263578_DUP', 'prefix_263619_DEL', 'prefix_263637_DUP', 'prefix_263641_DUP', 'prefix_263645_DUP', - 'prefix_263647_DUP', 'prefix_264278_DUP', 'prefix_26438_DEL', 'prefix_264706_DUP', 'prefix_264786_DEL', 'prefix_264797_DEL', - 'prefix_266509_DEL', 'prefix_266705_DEL', 'prefix_26790_DEL', 'prefix_27178_DEL', 'prefix_272072_DUP', 'prefix_27418_DUP', - 'prefix_277330_DEL', 'prefix_277330_DEL', 'prefix_278495_DUP', 'prefix_281404_DEL', 'prefix_28173_DUP', 'prefix_283002_DUP', - 'prefix_28329_DUP', 'prefix_283636_DUP', 'prefix_284375_DEL', 'prefix_284482_DEL', 'prefix_284499_DEL', 'prefix_284666_DUP', - 'prefix_284724_DEL', 'prefix_28564_DEL', 'prefix_28677_DEL', 'prefix_287359_DEL', 'prefix_28768_DEL', 'prefix_28815_DEL', - 'prefix_28955_DEL', 'prefix_29050_DEL', 'prefix_29052_DEL', 'prefix_290_DUP', 'prefix_29115_DEL', 'prefix_29132_DEL', - 'prefix_291814_DUP', 'prefix_29319_DEL', 'prefix_29359_DEL', 'prefix_29409_DEL', 'prefix_29465_DEL', 'prefix_296068_DEL', - 'prefix_296070_DEL', 'prefix_296071_DEL', 'prefix_296072_DEL', 'prefix_297161_DUP', 'prefix_297163_DUP', 'prefix_297193_DUP', - 'prefix_297203_DUP', 'prefix_297244_DEL', 'prefix_297325_DUP', 'prefix_298246_DEL', 'prefix_299171_DEL', 'prefix_299729_DUP', - 'prefix_30029_DEL', 'prefix_301500_DEL', 'prefix_301521_DEL', 'prefix_303270_DEL', 'prefix_31042_DEL', 'prefix_31146_DUP', - 'prefix_31636_DUP', 'prefix_32411_DEL', 'prefix_32572_DEL', 'prefix_32817_DEL', 'prefix_32938_DEL', 'prefix_32943_DEL', - 'prefix_33036_DEL', 'prefix_33259_DEL', 'prefix_33269_DEL', 'prefix_35408_DUP', 'prefix_36013_DUP', 'prefix_3660_DEL', - 'prefix_37114_DUP', 'prefix_37120_DUP', 'prefix_37284_DEL', 'prefix_37542_DEL', 'prefix_37710_DUP', 'prefix_37751_DEL', - 'prefix_38251_DUP', 'prefix_38525_DEL', 'prefix_40157_DEL', 'prefix_40443_DEL', 'prefix_41131_DEL', 'prefix_42483_DUP', - 'prefix_42564_DUP', 'prefix_43104_DEL', 'prefix_46281_DUP', 'prefix_46357_DEL', 'prefix_46467_DUP', 'prefix_46483_DEL', - 'prefix_46566_DEL', 'prefix_46681_DEL', 'prefix_47968_DUP', 'prefix_49538_DEL', 'prefix_4996_DEL', 'prefix_50673_DUP', - 'prefix_50678_DEL', 'prefix_518_DEL', 'prefix_52282_DUP', 'prefix_5289_DEL', 'prefix_54347_DUP', 'prefix_54569_DUP', - 'prefix_55426_DEL', 'prefix_56509_DUP', 'prefix_56903_DUP', 'prefix_58485_DEL', 'prefix_5932_DUP', 'prefix_6262_DEL', - 'prefix_63135_DUP', 'prefix_6344_DEL', 'prefix_64090_DEL', 'prefix_64813_DUP', 'prefix_65486_DEL', 'prefix_66447_DEL', - 'prefix_67003_DEL', 'prefix_7120_DEL', 'prefix_71708_DEL', 'prefix_72647_DUP', 'prefix_75175_DEL', 'prefix_75176_DEL', - 'prefix_75177_DEL', 'prefix_75180_DEL', 'prefix_75506_DUP', 'prefix_75843_DUP', 'prefix_76309_DUP', 'prefix_7665_DUP', - 'prefix_76693_DEL', 'prefix_7802_DUP', 'prefix_78417_DEL', 'prefix_78921_DUP', 'prefix_78943_DUP', 'prefix_78963_DUP', - 'prefix_78971_DUP', 'prefix_78976_DUP', 'prefix_78978_DUP', 'prefix_78980_DUP', 'prefix_78982_DUP', 'prefix_78988_DUP', - 'prefix_78994_DUP', 'prefix_79001_DUP', 'prefix_7962_DEL', 'prefix_796_DUP', 'prefix_79989_DUP', 'prefix_82092_DEL', - 'prefix_83579_DEL', 'prefix_83600_DUP', 'prefix_83786_DEL', 'prefix_83974_DEL', 'prefix_84263_DUP', 'prefix_84718_DEL', - 'prefix_86064_DUP', 'prefix_86065_DUP', 'prefix_89136_DEL', 'prefix_895_DUP', 'prefix_90181_DUP', 'prefix_91282_DEL', - 'prefix_91373_DUP', 'prefix_91952_DEL', 'prefix_91952_DEL', 'prefix_92457_DUP', 'prefix_92756_DUP', 'prefix_93796_DEL', - 'prefix_94231_DUP', 'prefix_943_DUP', 'prefix_94660_DEL', 'prefix_96293_DEL', 'prefix_97207_DUP', 'prefix_9762_DUP', - 'prefix_98217_DUP', 'prefix_98339_DUP', 'prefix_98578_DUP', 'prefix_98763_DUP', 'prefix_98910_DEL', 'prefix_99458_DEL', - 'suffix_20012_DEL_2', 'suffix_231565_DUP_2', 'suffix_264599_DEL_2', 'suffix_54380_DEL_2', 'suffix_69178_DEL_2', - 'CMG.phase1_CMG_DEL_chr15_535', 'CMG.phase1_CMG_DEL_chr15_748', 'CMG.phase1_CMG_DEL_chr16_1524', - 'CMG.phase1_CMG_DEL_chr16_1529', 'CMG.phase1_CMG_DEL_chr16_1842', 'CMG.phase1_CMG_DEL_chr17_398', - 'CMG.phase1_CMG_DEL_chr17_908', 'CMG.phase1_CMG_DEL_chr1_3107', 'CMG.phase1_CMG_DEL_chr1_4610', - 'CMG.phase1_CMG_DEL_chr2_5843', 'CMG.phase1_CMG_DEL_chr3_168', 'CMG.phase1_CMG_DEL_chr3_174', 'CMG.phase1_CMG_DEL_chr3_175', - 'CMG.phase1_CMG_DEL_chr3_744', 'CMG.phase1_CMG_DEL_chr5_1661', 'CMG.phase1_CMG_DEL_chr5_1936', - 'CMG.phase1_CMG_DEL_chr7_3298', 'CMG.phase1_CMG_DEL_chrX_1942', 'CMG.phase1_CMG_DEL_chrX_521', 'CMG.phase1_CMG_DEL_chrX_524', - 'CMG.phase1_CMG_DEL_chrX_544', 'phase2_DEL_chr16_4247', -} - class Command(BaseCommand): def handle(self, *args, **options): - num_updated = SavedVariant.objects.filter( - genotypes={}, saved_variant_json__genotypes__isnull=False, - ).exclude(saved_variant_json__genotypes={}).update(genotypes=F('saved_variant_json__genotypes')) - logger.info(f'Updated genotypes for {num_updated} variants') - variant_ids = SavedVariant.objects.filter( key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh38, saved_variant_json__populations__isnull=False, # Omit manual variants @@ -225,7 +79,6 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ mapped_variant_ids = list(variant_key_map.keys()) - update_fields = ['key', 'dataset_type'] total_num_updated = 0 for i in range(0, len(mapped_variant_ids), BATCH_SIZE): batch_ids = mapped_variant_ids[i:i + BATCH_SIZE] @@ -234,8 +87,7 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ ) for variant in saved_variants: variant.key = variant_key_map[variant.variant_id] - variant.dataset_type = dataset_type - num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields) + num_updated = SavedVariant.objects.bulk_update(saved_variants, ['key']) logger.info(f'Updated batch of {num_updated}') total_num_updated += num_updated @@ -276,12 +128,11 @@ def _resolve_missing_variants(cls, variant_ids, genome_version): @classmethod def _resolve_reloaded_svs(cls, variant_ids): - num_known_dropped = len(SV_DROPPED_IDS.intersection(variant_ids)) missing_with_search_data, num_missing = cls._query_missing_variants( - list(set(variant_ids) - SV_DROPPED_IDS), ['family__individual__sample__sample_type'], + list(variant_ids), ['family__individual__sample__sample_type'], ) logger.info( - f'{num_missing + num_known_dropped} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data, {num_known_dropped} of which are known to have dropped out of the callset.' + f'{num_missing} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data' ) if not missing_with_search_data: return diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index 6462ba065a..c22bff5763 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -200,12 +200,10 @@ def test_batching(self, mock_logger, mock_safe_post_to_slack): @responses.activate def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): + # Test errors that should still raise (not unenumerated values) for description, review_status, conflicting_pathogenicities, error_message in [ - ("Pathogenic-ey", None, None, 'Found an un-enumerated clinvar assertion: Pathogenic-ey'), - ("Pathogenic; but unknown assertion", None, None, 'Found an un-enumerated clinvar assertion: but unknown assertion'), ("Pathogenic", "unhandled", None, 'Found unexpected review status unhandled'), ("Conflicting classifications of pathogenicity", None, "Pathogenic;", 'Failed to correctly parse conflicting pathogenicity counts: Pathogenic;'), - ("Conflicting classifications of pathogenicity", None, "Pathogenic(18); unhandled(1)", 'Found an un-enumerated conflicting pathogenicity: unhandled'), ("Conflicting classifications of pathogenicity", None, None, 'Failed to find the conflicting pathogenicities node'), ]: data = f''' @@ -239,6 +237,66 @@ def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): with self.assertRaisesMessage(CommandError, error_message): call_command('reload_clinvar_all_variants') + # Test that unenumerated values are now collected instead of raising + unenumerated_test_cases = [ + { + 'allele_id': 1, + 'description': 'Pathogenic-ey', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'Pathogenic-ey', + }, + { + 'allele_id': 2, + 'description': 'Pathogenic; but unknown assertion', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'but unknown assertion', + }, + { + 'allele_id': 3, + 'description': 'Conflicting classifications of pathogenicity', + 'explanation': 'Pathogenic(18); unhandled(1)', + 'alert_type': 'Conflicting Pathogenicity', + 'alert_value': 'unhandled', + }, + ] + data = WEEKLY_XML_RELEASE_HEADER + for case in unenumerated_test_cases: + data += f''' + + + + + + + + + + {case['description']} + {f"{case['explanation']}" if case['explanation'] else ""} + + + + + ''' + data += '' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(data.encode()), + stream=True, + ) + DataVersions.objects.all().delete() + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + call_command('reload_clinvar_all_variants') + # Command should succeed and collect all unenumerated values in single slack message + slack_message = mock_safe_post_to_slack.call_args[0][1] + self.assertIn(f'Found {len(unenumerated_test_cases)} unenumerated value(s) during parsing:', slack_message) + for case in unenumerated_test_cases: + self.assertIn(f"- {case['alert_type']}: '{case['alert_value']}' (Allele ID: {case['allele_id']})", slack_message) + # Variants with missing/equivalent alleles and positions are skipped for simple_allele_attrs, sequence_location_attrs in [ # Case 1: Missing AlleleId in diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 9084105a3d..e53bdf1a2b 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -13,14 +13,21 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] - SKIP_RESET_VARIANT_JSON = True MOCK_GCNV_DATA = MOCK_GCNV_DATA @classmethod def setUpTestData(cls): Project.objects.filter(id=3).update(genome_version='38') Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) + for sv in SavedVariant.objects.filter(key__isnull=False): + sv.saved_variant_json = { + 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, + } + if sv.guid == 'SV0000009_25000014783_r0004_no': + sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} + sv.save() SavedVariant.objects.update(key=None) + SavedVariant.objects.filter(guid='SV0000007_prefix_19107_DEL_r00').update(variant_id='prefix_19107_DEL') @mock.patch('clickhouse_search.management.commands.set_saved_variant_key.BATCH_SIZE', 2) @mock.patch('seqr.utils.file_utils.subprocess.Popen') @@ -30,7 +37,6 @@ def test_command(self, mock_subprocess): call_command('set_saved_variant_key') self.assert_json_logs(user=None, expected=[ - ('Updated genotypes for 7 variants', None), ('Finding keys for 1 MITO (GRCh38) variant ids', None), ('Found 1 keys', None), ('Updated batch of 1', None), @@ -58,29 +64,27 @@ def test_command(self, mock_subprocess): ('Done', None), ]) - saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'dataset_type', 'genotypes', 'saved_variant_json')) + saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')) expected_saved_variants = [ - {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, - {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000002_na19675': mock.ANY, 'I000017_na20889': mock.ANY}, 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000018_na21234': {'sampleId': 'NA20885', 'ab': 0.0, 'gq': 99.0, 'numAlt': 1}}, 'saved_variant_json': {}}, - {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}}, - {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'dataset_type': 'MITO', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, - {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, - {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'saved_variant_json': {}}, + {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'saved_variant_json': {}}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'saved_variant_json': {}}, + {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'saved_variant_json': mock.ANY}, ] self.assertListEqual(saved_variants, expected_saved_variants) - self.assertTrue(all(v['genotypes'] == v['saved_variant_json']['genotypes'] for v in saved_variants[:1] + saved_variants[7:])) # Reloading is a no-op self.reset_logs() call_command('set_saved_variant_key') self.assert_json_logs(user=None, expected=[ - ('Updated genotypes for 0 variants', None), ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), @@ -88,7 +92,7 @@ def test_command(self, mock_subprocess): ('Done', None), ]) self.assertListEqual( - list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'dataset_type', 'genotypes', 'saved_variant_json')), + list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')), expected_saved_variants, ) @@ -104,7 +108,6 @@ def test_command(self, mock_subprocess): call_command('set_saved_variant_key') self.assert_json_logs(user=None, expected=[ - ('Updated genotypes for 7 variants', None), ('Finding keys for 1 MITO (GRCh38) variant ids', None), ('Found 0 keys', None), ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), @@ -115,7 +118,7 @@ def test_command(self, mock_subprocess): ('Found 0 keys', None), ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), ('Found 0 keys', None), - ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None), + ('2 SV variants have no key, 0 of which have no search data', None), ('1 remaining SV WES variants prefix_19107_DEL - 12', None), ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 93c0f38df5..af5d5c4d28 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -12,7 +12,7 @@ GroupArrayIntersect, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat, Untuple, \ IntDiv, Modulo, SplitByString, ArrayIndex, Multiply, IndexOf from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict -from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ +from clickhouse_search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, CLINVAR_LIKELY_PATH_FILTER, \ @@ -39,10 +39,6 @@ def _pathogenicity_tuple(model, field_prefix, **kwargs): output_field=NamedTupleField(list(fields.values()), null_if_empty=True, null_empty_arrays=True, **kwargs), ) - @classmethod - def _clinvar_path_q(cls, pathogenicity): - return cls._clinvar_filter_q(pathogenicity, allowed_filters=CLINVAR_PATH_SIGNIFICANCES) - @classmethod def _clinvar_filter_q(cls, pathogenicity, allowed_filters=None): clinvar_filters = (pathogenicity or {}).get(CLINVAR_KEY) @@ -291,7 +287,7 @@ def annotation_fields(self): @property def variant_detail_field(self): - return next(obj.name for obj in self.model._meta.related_objects if obj.name.startswith('variantdetails')) + return next((obj.name for obj in self.model._meta.related_objects if obj.name.startswith('variantdetails')), None) @property def entry_field(self): @@ -692,14 +688,15 @@ def _consequence_term_filter(consequences, **kwargs): def _annotate_filtered_transcripts(self, results, consequence_field, transcript_filters, *args, require_mane_canonical=False, **kwargs): if require_mane_canonical: - filtered_expr = ArrayFilter(consequence_field, conditions=[{'canonical': (0, '{field} > {value}')}]) if 'isManeSelect' in self.sorted_transcript_consequence_fields: - filtered_expr = If( - ArrayFilter(consequence_field, conditions=[{'isManeSelect': (True, '{field}')}]), - filtered_expr, - condition='arrayExists(x -> x.isManeSelect, sortedTranscriptConsequences), ', - ) - results = results.annotate(**{self.FILTERED_CONSEQUENCE_FIELD: filtered_expr}) + mane_genes_expression = 'arrayMap(a -> a.geneId, arrayFilter(b -> b.isManeSelect, sortedTranscriptConsequences))' + conditions=[ + {'isManeSelect': (True, '{field}')}, + {'canonical': (mane_genes_expression, 'and({field} > 0, not has({value}, x.geneId))')}, + ] + else: + conditions = [{'canonical': (0, '{field} > {value}')}] + results = results.annotate(**{self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=conditions)}) consequence_field = self.FILTERED_CONSEQUENCE_FIELD return super()._annotate_filtered_transcripts(results, consequence_field, transcript_filters, **kwargs) @@ -761,10 +758,6 @@ def annotation_values(self): annotations['transcripts'] = annotations.pop(self.model.sorted_gene_consequences.field.db_column) return annotations - @classmethod - def _clinvar_path_q(cls, pathogenicity): - return None - @staticmethod def _genotype_override_expression(index, col, cn_index): expressions = [f'x.{index}', f'sample_{col}'] @@ -1018,6 +1011,10 @@ def genome_version(self): def callset_filter_field(self): return 'callset' + @classmethod + def _clinvar_path_q(cls, pathogenicity): + return cls._clinvar_filter_q(pathogenicity, allowed_filters=CLINVAR_PATH_SIGNIFICANCES) + def search(self, sample_data, exclude_keys=None, **kwargs): entries = self.filter_locus(**kwargs) @@ -1300,7 +1297,7 @@ def annotation_fields(cls, entries): fields.append('seqrPop') return fields - def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, annotate_num_families=False, override_annotations=None, **kwargs): + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, **kwargs): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -1308,10 +1305,8 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu if multi_sample_type_families or sample_data is None or sample_data['num_families'] > 1: entries = entries.values(*fields) - if annotate_num_families: + if skip_entry_fields: entries = entries.annotate(numFamilies=Count('family_guid')) - elif skip_entry_fields: - entries = entries.distinct('key') else: gt_field, gt_expression = self.genotype_expression(sample_data) entries = entries.annotate( diff --git a/clickhouse_search/models/reference_data_models.py b/clickhouse_search/models/reference_data_models.py index 5a7b26a041..901fbafb61 100644 --- a/clickhouse_search/models/reference_data_models.py +++ b/clickhouse_search/models/reference_data_models.py @@ -506,11 +506,6 @@ class HgmdMvMeta(RefreshableMaterializedViewMeta): HGMD_INFO_STRUCTURE = 'CHROM String, POS UInt32, ID String, REF String, ALT String, QUAL String, FILTER String, INFO String' -def _hgmd_source(url): - if not url: - return f"null('{HGMD_INFO_STRUCTURE}')" - return f"gcs(pipeline_data_access, url='{url}', format='TSV', structure='{HGMD_INFO_STRUCTURE}')" - class HgmdAllMv(RefreshableMaterializedView): variant_id = models.StringField(db_column='variantId', primary_key=True) diff --git a/clickhouse_search/models/search_models.py b/clickhouse_search/models/search_models.py index 8e1ebf8cd6..d3c609df9d 100644 --- a/clickhouse_search/models/search_models.py +++ b/clickhouse_search/models/search_models.py @@ -11,7 +11,6 @@ ScreenDict from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample -from seqr.utils.search.constants import SPLICE_AI_FIELD from seqr.utils.xpos_utils import CHROMOSOME_CHOICES from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 99041a3b1e..faa14f7a81 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -9,21 +9,24 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ - ArrayMap, Modulo, ArrayObjectSort + ArrayMap, Modulo from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP -from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization, Individual +from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP +from seqr.models import Sample, PhenotypePrioritization, Family, Individual from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ +from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets logger = SeqrLogger(__name__) +class InvalidSearchException(Exception): + pass + BATCH_SIZE = 10000 TRANSCRIPT_CONSEQUENCES_FIELD = 'sortedTranscriptConsequences' @@ -31,21 +34,26 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version,page=1, num_results=100, sort=None, **kwargs): +def get_clickhouse_variants(families, dataset_types, search, user, genome_version, sort=None, secondary_dataset_types=None, sample_data_by_dataset_type=None, encode_genotypes_json=False): inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) - sample_data_by_dataset_type = _get_sample_data( - samples, + sample_data_by_dataset_type = sample_data_by_dataset_type or _get_sample_data( + families, + dataset_types, + secondary_dataset_types=secondary_dataset_types, skip_multi_project_individual_guid=True, annotate_affected_males=has_x_chrom_comp_het or has_x_linked, + has_comp_het=has_comp_het, + inheritance_mode=inheritance_mode, + inheritance_filter=search.get('inheritance_filter'), + allow_no_samples=bool(search.get('no_access_project_genome_version')), ) results = [] family_guid = None exclude_keys = search.pop('exclude_keys', None) or {} exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} - search.pop('dataset_type', None) for dataset_type, sample_data in sample_data_by_dataset_type.items(): logger.info(f'Loading {dataset_type} data for {sample_data["num_families"]} families', user) @@ -73,52 +81,43 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno comp_het_sample_data = sample_data if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: comp_het_sample_data = _no_affected_male_families(sample_data, user) - result_q = get_data_type_comp_het_results_queryset(genome_version, dataset_type, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), **search) + result_q = _get_data_type_comp_het_results_queryset(genome_version, dataset_type, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), **search) dataset_results += _evaluate_results(result_q, is_comp_het=True) if 'samples' not in sample_data: - add_individual_guids(dataset_results, samples) + _add_individual_guids(dataset_results, encode_genotypes_json=encode_genotypes_json) results += dataset_results if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_by_dataset_type, user, exclude_key_pairs, **search) + results += _get_multi_data_type_comp_het_results(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, **search) if search.get('no_access_project_genome_version'): logger.info('Looking up variants in projects with no user access', user) results += _get_search_results( - genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, **search, sample_data=None, skip_entry_fields=True, annotate_num_families=True, + genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, **search, sample_data=None, skip_entry_fields=True, exclude_projects=sample_data_by_dataset_type.get(Sample.DATASET_TYPE_VARIANT_CALLS, {}).get('project_guids'), ) - cache_results = get_clickhouse_cache_results(results, sort, family_guid) - previous_search_results.update(cache_results) - - logger.info(f'Total results: {cache_results["total_results"]}', user) + logger.info(f'Total results: {len(results)}', user) + return get_sorted_search_results(results, sort, family_guid) - return format_clickhouse_results(cache_results['all_results'][(page-1)*num_results:page*num_results], genome_version) - -def get_search_queryset(genome_version, dataset_type, sample_data, **search_kwargs): +def _get_search_results(genome_version, dataset_type, sample_data, skip_entry_fields=False, **search_kwargs): entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] variants_cls = VARIANTS_CLASS_MAP[genome_version][dataset_type] - entries = entry_cls.objects.search(sample_data, **search_kwargs) - return variants_cls.objects.subquery_join(entries).search(**search_kwargs) - - -def _get_search_results(*args, skip_entry_fields=False, **search_kwargs): - results = get_search_queryset(*args, skip_entry_fields=skip_entry_fields, **search_kwargs) + entries = entry_cls.objects.search(sample_data, **search_kwargs, skip_entry_fields=skip_entry_fields) + results = variants_cls.objects.subquery_join(entries).search(**search_kwargs, skip_entry_fields=skip_entry_fields) return _evaluate_results(results.result_values(skip_entry_fields=skip_entry_fields)) def _evaluate_results(result_q, is_comp_het=False): results = [list(result[1:]) if is_comp_het else result for result in result_q[:MAX_VARIANTS + 1]] if len(results) > MAX_VARIANTS: - from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -139,7 +138,7 @@ def _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_b logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) snv_indel_sample_type_families = { - sample_type: familes.intersection(sv_families) + sample_type: set(familes).intersection(sv_families) for sample_type, familes in snv_indel_sample_data['sample_type_families'].items() } snv_indel_sample_type_families = {k: v for k, v in snv_indel_sample_type_families.items() if v} @@ -157,37 +156,31 @@ def _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_b 'samples': [s for s in sv_sample_data['samples'] if s['family_guid'] in families] if 'samples' in sv_sample_data else None, } - result_q = get_multi_data_type_comp_het_results_queryset( - genome_version, sv_dataset_type, sv_sample_data, type_snv_indel_sample_data, num_families=len(families), - exclude_key_pairs=exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), - annotations=annotations, **search_kwargs, + entries = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects.search( + type_snv_indel_sample_data, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, + annotate_hom_alts=True, annotations=annotations, **search_kwargs, + ) + variants_cls = VARIANTS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] + snv_indel_q = variants_cls.objects.subquery_join(entries).search(annotations=annotations, **search_kwargs) + + sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search( + sv_sample_data, annotations=annotations, **search_kwargs, inheritance_mode=COMPOUND_HET, annotate_carriers=True, + ) + sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][sv_dataset_type] + sv_q = sv_variants_cls.objects.subquery_join(sv_entries).search(annotations=annotations, **search_kwargs) + + result_q = _get_comp_het_results_queryset( + variants_cls, snv_indel_q, sv_q, len(families), exclude_key_pairs=exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), ) dataset_results = _evaluate_results(result_q, is_comp_het=True) if not sv_sample_data['samples']: - add_individual_guids(dataset_results, samples) + _add_individual_guids(dataset_results) results += dataset_results return results -def get_multi_data_type_comp_het_results_queryset(genome_version, sv_dataset_type, sv_sample_data, snv_indel_sample_data, num_families, exclude_key_pairs=None, **kwargs): - entries = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects.search( - snv_indel_sample_data, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, - annotate_hom_alts=True, **kwargs, - ) - variants_cls = VARIANTS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - snv_indel_q = variants_cls.objects.subquery_join(entries).search(**kwargs) - - sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search( - sv_sample_data, **kwargs, inheritance_mode=COMPOUND_HET, annotate_carriers=True, - ) - sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][sv_dataset_type] - sv_q = sv_variants_cls.objects.subquery_join(sv_entries).search(**kwargs) - - return _get_comp_het_results_queryset(variants_cls, snv_indel_q, sv_q, num_families, exclude_key_pairs) - - -def get_data_type_comp_het_results_queryset(genome_version, dataset_type, sample_data, annotations=None, annotations_secondary=None, pathogenicity=None, inheritance_mode=None, exclude_key_pairs=None, split_pathogenicity_annotations=False, **search_kwargs): +def _get_data_type_comp_het_results_queryset(genome_version, dataset_type, sample_data, annotations=None, annotations_secondary=None, pathogenicity=None, inheritance_mode=None, exclude_key_pairs=None, split_pathogenicity_annotations=False, **search_kwargs): entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] variants_cls = VARIANTS_CLASS_MAP[genome_version][dataset_type] entries = entry_cls.objects.search( @@ -290,14 +283,14 @@ def _result_as_tuple(results, field_prefix): return Tuple(*fields.keys(), output_field=NamedTupleField(list(fields.values()))) -def add_individual_guids(results, samples, encode_genotypes_json=False): +def _add_individual_guids(results, encode_genotypes_json=False): families = set() for result in results: for r in (result if isinstance(result, list) else [result]): families.update(r.get('familyGenotypes', {}).keys()) sample_map = { - (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in samples.filter( - individual__family__guid__in=families, + (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Sample.objects.filter( + individual__family__guid__in=families, is_active=True, ).values_list('individual__family__guid', 'individual__guid', 'sample_id') } for result in results: @@ -323,14 +316,41 @@ def _set_individual_guids(result, sample_map, encode_genotypes_json): result['genotypes'] = genotypes -def get_clickhouse_cache_results(results, sort, family_guid): +def get_sorted_search_results(results, sort, family_guid): sort_metadata = _get_sort_gene_metadata(sort, results, family_guid) sort_key = _get_sort_key(sort, sort_metadata) sorted_results = sorted([ sorted(result, key=sort_key) if isinstance(result, list) else result for result in results ], key=sort_key) - total_results = len(sorted_results) - return {'all_results': sorted_results, 'total_results': total_results} + return sorted_results + + +def format_clickhouse_export_results(results, genome_version): + formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] + keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} + detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) + details_by_key = { + detail['key']: detail for detail in detail_qs.values( + 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), + **detail_qs.split_variant_id_annotations(), + ) + } + + gene_ids = set() + for result in formatted_results: + if 'transcripts' in result: + result['mainTranscript'] = next((gene_transcripts[0] for gene_transcripts in result['transcripts'].values()), {}) + else: + result.update(details_by_key.get(result['key'], {})) + if result.get('mainTranscript'): + gene_ids.add(result['mainTranscript']['geneId']) + + gene_id_map = dict(GeneInfo.objects.filter(gene_id__in=gene_ids).values_list('gene_id', 'gene_symbol')) + for result in formatted_results: + if result.get('mainTranscript'): + result['geneSymbol'] = gene_id_map.get(result['mainTranscript']['geneId']) + + return formatted_results def format_clickhouse_results(results, genome_version): @@ -339,7 +359,7 @@ def format_clickhouse_results(results, genome_version): } details_by_key = { detail['key']: detail for detail in - _get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() } formatted_results = [] @@ -392,24 +412,43 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): and transcript.get('spliceregion', {}).get('extended_intronic_splice_region_variant') == minimal_transcript.get('extendedIntronicSpliceRegionVariant')) -def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate_affected_males=False): +def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples): + samples = Sample.objects.filter(individual__family__in=families, is_active=True) + if not samples.exists(): + if allow_no_samples: + return None + raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}') + + if dataset_types: + samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) + if not samples.exists(): + if allow_no_samples: + return None + raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') + mismatch_affected_samples = samples.values('sample_id', 'dataset_type').annotate( projects=ArrayAgg('individual__family__project__name', distinct=True), affected=ArrayAgg('individual__affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: - from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), ) + return samples + + +def _get_grouped_samples(samples, skip_multi_project_individual_guid, affected_family_only, annotate_affected_males): skip_individual_guid = ( skip_multi_project_individual_guid and samples.values('individual__family__project_id').distinct().count() > 1 ) + family_array_kwargs = {'distinct': True} + if affected_family_only: + family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED) annotations = { 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True), - 'family_guids': ArrayAgg('individual__family__guid', distinct=True), + 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs), } if skip_individual_guid: annotations['num_unaffected'] = Count( @@ -421,10 +460,26 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate )) else: annotations['samples'] = ArrayAgg(JSONObject( - affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), + affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', + family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), )) sample_data = samples.values('dataset_type', 'sample_type').annotate(**annotations) + if affected_family_only: + sample_data = sample_data.filter(family_guids__len__gt=0) + return sample_data + + +def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): + samples = _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples) + if not samples: + return {} + + individual_affected_status = (inheritance_filter or {}).get('affected') + genotype_filter = (inheritance_filter or {}).get('genotype') + affected_family_only = inheritance_mode and not individual_affected_status + sample_data = _get_grouped_samples(samples, skip_multi_project_individual_guid, affected_family_only, annotate_affected_males) + samples_by_dataset_type = {} for data in sample_data: dataset_type = data.pop('dataset_type') @@ -432,6 +487,13 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate if dataset_type == Sample.DATASET_TYPE_SV_CALLS: dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + if genotype_filter and all(s['individual_guid'] not in genotype_filter for s in data.get('samples', [])): + continue + if inheritance_mode and individual_affected_status and all( + (individual_affected_status.get(s['individual_guid']) or s['affected']) != AFFECTED for s in data.get('samples', []) + ): + continue + family_guids = set(data.pop('family_guids')) if dataset_type in samples_by_dataset_type: other_type_data = samples_by_dataset_type[dataset_type] @@ -463,11 +525,35 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate 'num_families': len(set().union(*data['sample_type_families'].values())), } + _validate_dataset_types( + samples_by_dataset_type, dataset_types, secondary_dataset_types, has_comp_het, genotype_filter, + ) _add_missing_multi_type_samples(samples, samples_by_dataset_type) return samples_by_dataset_type +def _validate_dataset_types(samples_by_dataset_type, dataset_types, secondary_dataset_types, has_comp_het, genotype_filter): + if not samples_by_dataset_type: + raise InvalidSearchException( + 'Invalid custom inheritance' if genotype_filter else + 'Inheritance based search is disabled in families with no data loaded for affected individuals' + ) + + if has_comp_het and secondary_dataset_types: + sample_dataset_types = { + Sample.DATASET_TYPE_SV_CALLS if dt.startswith(Sample.DATASET_TYPE_SV_CALLS) else dt + for dt in samples_by_dataset_type + } + invalid_type = next(( + dts[0] for dts in [dataset_types, secondary_dataset_types] if dts and not sample_dataset_types.intersection(dts) + ), None) + if invalid_type: + raise InvalidSearchException( + f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' + ) + + def _add_missing_multi_type_samples(samples, samples_by_dataset_type): for dataset_type, data in samples_by_dataset_type.items(): if data['sample_type_families'].get('multi') and 'samples' not in data: @@ -656,17 +742,15 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, samples=None, affected_only=False, hom_only=False): +def _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, affected_only=False, hom_only=False): entry_cls = ENTRY_CLASS_MAP[genome_version][data_type] variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] - sample_data = _get_sample_data(samples)[data_type] if samples else None - entries = entry_cls.objects.filter_locus( variant_ids=[variant_id], parsed_variant_ids=[parsed_variant_id] if parsed_variant_id else [], ) entries = _filter_lookup_entries(entries, affected_only, hom_only) - entries = entries.result_values(sample_data) + entries = entries.result_values() results = variants_cls.objects.subquery_join(entries) if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) @@ -730,7 +814,7 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, dataset_type, def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only): lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) - if not lifted_entry_cls: + if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return lifted_id = (variant['liftedOverChrom'], variant['liftedOverPos'], *variant_id[2:]) lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) @@ -742,30 +826,15 @@ def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_o variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) -def get_clickhouse_variant_by_id(variant_id, parsed_variant_id, samples, genome_version, dataset_type): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - data_types = [ - f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - for sample_type in samples.values_list('sample_type', flat=True).distinct() - ] - else: - data_types = [dataset_type] - for data_type in data_types: - variant = _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, samples) - if variant: - return variant - return None - - -def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): - sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] +def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, additional_fields=None): + sample_data = _get_sample_data(Family.objects.filter(guid__in=family_guids), None)[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( project_guid=project_guid, family_guid__in=family_guids, key__in=keys, ) gt_field, gt_expr = entries.genotype_expression(sample_data) return { - key: _clickhouse_genotypes_json(genotypes) for key, genotypes in - entries.annotate(**{gt_field: gt_expr}).values_list('key', 'genotypes') + e['key']: {**e, 'genotypes': _clickhouse_genotypes_json(e['genotypes'])} for e in + entries.annotate(**{gt_field: gt_expr}).values('key', 'genotypes', *(additional_fields or [])) } @@ -780,14 +849,14 @@ def get_variants_queryset(genome_version, dataset_type, keys, variant_ids=None): return variants_cls.objects.filter(key__in=keys) -def _get_variant_details_queryset(genome_version, dataset_type, keys): +def get_variant_details_queryset(genome_version, dataset_type, keys): if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: return VARIANT_DETAILS_CLASS_MAP[genome_version].objects.filter(key__in=keys) return get_variants_queryset(genome_version, dataset_type, keys) def get_clickhouse_variant_annotations(genome_version, dataset_type, keys): - qs = _get_variant_details_queryset(genome_version, dataset_type, keys) + qs = get_variant_details_queryset(genome_version, dataset_type, keys) return qs.join_annotations().result_values(skip_entry_fields=True) @@ -801,38 +870,6 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids): return lookup -def get_variant_main_transcripts_by_key(genome_version, dataset_type, selected_transcripts_by_key, include_clinvar=False, additional_values=None): - qs = _get_variant_details_queryset(genome_version, dataset_type, selected_transcripts_by_key.keys()) - if hasattr(qs.model, 'transcripts'): - transcript_field = 'transcripts' - expr = ArrayObjectSort - expr_kwargs = {'sort_field': 'transcriptRank'} - else: - transcript_field = qs.TRANSCRIPT_FIELD - expr = ArrayMap - expr_kwargs = {'mapped_expression': 'x'} - output_field = getattr(qs.model, transcript_field).field.clone() - output_field.group_by_key = None - qs = qs.annotate(sorted_transcripts=expr(transcript_field, output_field=output_field, **expr_kwargs)) - - fields = ['key', 'sorted_transcripts'] - if include_clinvar: - fields.append('clinvar') - qs = qs.join_clinvar() - - variants_by_key = {} - for variant in qs.values(*fields, **(additional_values or {})): - key = variant['key'] - transcripts = variant.pop('sorted_transcripts') - variant['main_transcripts'] = { - selected_transcript_id: _main_transcript(selected_transcript_id, transcripts) - for selected_transcript_id in selected_transcripts_by_key[key] - } - variants_by_key[key] = variant - - return variants_by_key - - def _main_transcript(selected_transcript_id, sorted_transcripts): if not sorted_transcripts: return {} @@ -842,7 +879,8 @@ def _main_transcript(selected_transcript_id, sorted_transcripts): def delete_clickhouse_project(project, dataset_type, sample_type=None): - dataset_type = _clickhouse_dataset_type(dataset_type, sample_type) + if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: + dataset_type = 'GCNV' table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) @@ -851,13 +889,3 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh() ENTRY_CLASS_MAP[project.genome_version][dataset_type].gt_stats.rel.related_model.reload() return f'Deleted all {dataset_type} search data for project {project.name}' - - -SV_DATASET_TYPES = { - Sample.SAMPLE_TYPE_WGS: Sample.DATASET_TYPE_SV_CALLS, - Sample.SAMPLE_TYPE_WES: 'GCNV', -} -def _clickhouse_dataset_type(dataset_type, sample_type): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - dataset_type = SV_DATASET_TYPES[sample_type] - return dataset_type \ No newline at end of file diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6ed311915b..bd6ad73664 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,9 +1,12 @@ -from django.core.exceptions import ObjectDoesNotExist +import re +from collections import defaultdict +from datetime import timedelta from django.core.management import call_command from django.db import connections from django.urls.base import reverse import json import mock +import random import responses from clickhouse_search.models.gt_stats_models import ProjectGtStatsSnvIndel, \ @@ -20,7 +23,7 @@ TopmedGRCh37Dict, GnomadmitoMv, GnomadmitoDict, GnomadmitoheteroplasmyMv, GnomadmitoheteroplasmyDict, HelixmitoMv, \ HelixmitoDict, HelixmitoheteroplasmyMv, HelixmitoheteroplasmyDict, ScreenDict, MitomapMv, MitomapDict, Absplice2Mv, \ Absplice2Dict, PromoterAIMv, PromoterAIDict, PextSnvIndelMv, PextSnvIndelDict, PextMitoMv, PextMitoDict -from clickhouse_search.models.search_models import EntriesSnvIndel, VariantsSnvIndel +from clickhouse_search.models.search_models import EntriesSnvIndel, VariantsSnvIndel, EntriesSv, EntriesGcnv from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, CACHED_CONSEQUENCES_BY_KEY, \ VARIANT_ID_SEARCH, VARIANT_IDS, LOCATION_SEARCH, GENE_IDS, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT, \ SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, COMP_HET_ALL_PASS_FILTERS, \ @@ -30,17 +33,24 @@ VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ - MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ - MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ + MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, \ + MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, \ + DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.utils.search.search_utils_tests import SearchTestHelper -from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException +from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual from seqr.views.apis.data_manager_api import trigger_delete_project -from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.views.apis.variant_search_api import query_variants_handler +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ + SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ + FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS +from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler, \ + query_single_variant_handler, variant_lookup_handler + + +SEARCH_RESPONSE_KEYS = { + 'genesById', 'locusListsByGuid', 'mmeSubmissionsByGuid', 'phenotypeGeneScores', 'rnaSeqData', 'savedVariantsByGuid', + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', 'totalSampleCounts', +} class ClickhouseSearchTestCase(AnvilAuthenticationTestCase): @@ -96,45 +106,127 @@ def setUpTestData(cls): super().setUpTestData() -class ClickhouseSearchTests(SearchTestHelper, ClickhouseSearchTestCase): +class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' - fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): - super().set_up() + self.MOCK_CACHE = {} + patcher = mock.patch('seqr.utils.redis_utils.redis.StrictRedis') + self.mock_redis = patcher.start().return_value + self.mock_redis.get.side_effect = self.MOCK_CACHE.get + self.addCleanup(patcher.stop) + + patcher = mock.patch('seqr.models.VariantSearchResults._compute_guid') + self.mock_results_guid = patcher.start() + self.mock_results_guid.return_value = random.randint(1000, 10000) # nosec + self.addCleanup(patcher.stop) + super().setUp() - self.mock_redis.get.return_value = None - def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', results_model=None, **search_kwargs): - results_model = results_model or self.results_model - self.search_model.search.update(search_kwargs or {}) - self.search_model.search['qualityFilter'] = quality_filter - self.search_model.search['inheritance']['mode'] = inheritance_mode + def set_cache(self, cache_key, cached): + self.MOCK_CACHE[cache_key] = json.dumps(cached) + self.mock_redis.keys.side_effect = lambda pattern: [ + key for key in list(self.MOCK_CACHE.keys()) if re.match(pattern, key) + ] + + def assert_cached_results(self, expected_results, cache_key): + self.mock_redis.set.assert_any_call(cache_key, mock.ANY) + call_index = next(i for i, call in enumerate(self.mock_redis.set.call_args_list) if call.args[0] == cache_key) + self.assertEqual(json.loads(self.mock_redis.set.call_args_list[call_index].args[1]), expected_results) + self.mock_redis.expire.assert_any_call(cache_key, timedelta(weeks=2)) + self.mock_redis.set.reset_mock() + self.mock_redis.expire.reset_mock() + + def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs): + search_hash = search_hash or random.randint(1000, 10000) # nosec + self.mock_results_guid.return_value = f'VRS{search_hash:07d}' + url = reverse(query_variants_handler, args=[search_hash]) + + search_body = { + 'inheritance': {'mode': inheritance_mode}, + 'freqs': {'callset': {'ac': 1000}}, + 'qualityFilter': quality_filter, + **(search_kwargs or {}), + } if inheritance_filter is not None: - self.search_model.search['inheritance']['filter'] = inheritance_filter + search_body['inheritance']['filter'] = inheritance_filter + + request_data = { + **(request_body or {'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families }), 'search': search_body, + } + + if check_login: + check_login(url, request_data=request_data) - variants, total = query_variants(results_model, user=self.user, sort=sort) - encoded_variants = self._assert_expected_variants(variants, expected_results) + query_string = '&'.join([f'{key}={value}' for key, value in {'sort': sort, **(query_params or {})}.items()]) + response = self.client.post(f'{url}?{query_string}', content_type='application/json', data=json.dumps(request_data)) - self.assertEqual(total, len(expected_results)) - self._assert_expected_search_cache(encoded_variants, total, cached_variant_fields, sort, results_model) + return response, search_hash, search_body + + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): + response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) + self.assertEqual(response.status_code, 200) + expected_response = { + 'searchedVariants': results_page or expected_results, + 'search': { + 'search': {**search_body, **(response_search or {})}, + 'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families, + 'totalResults': len(expected_results), + }, + + } + if expected_results: + expected_response.update({ + **{key: mock.ANY for key in SEARCH_RESPONSE_KEYS}, + **(additional_response or {}), + }) + if not is_37: + expected_response['omimIntervals'] = mock.ANY + self.assertDictEqual(response.json(), expected_response) + + if not skip_cache_check: + cache_key = f'search_results__VRS{search_hash:07d}__{cache_sort or sort}' + cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) + self.assert_cached_results(cached_variants, cache_key) if gene_counts: - gene_counts_json = get_variant_query_gene_counts(results_model, self.user) - self.assertDictEqual(gene_counts_json, gene_counts) + self._assert_expected_gene_counts(search_hash, gene_counts) + + if export_data: + self._assert_expected_export_data(search_hash, export_data) - def _assert_expected_variants(self, variants, expected_results): - encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) - self.assertListEqual(encoded_variants, expected_results) - return encoded_variants + return response.json() - def _assert_expected_search_cache(self, variants, total, cached_variant_fields, sort, results_model): - cached_variants = [ + def _assert_expected_gene_counts(self, search_hash, gene_counts): + url = reverse(get_variant_gene_breakdown, args=[search_hash]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + response_json = response.json() + self.assertDictEqual(response_json, { + 'searchGeneBreakdown': {str(search_hash): gene_counts}, + 'genesById': mock.ANY, + }) + for gene in response_json['genesById'].values(): + self.assertSetEqual(set(gene.keys()), GENE_VARIANT_DISPLAY_FIELDS) + + def _assert_expected_export_data(self, search_hash, export_data): + url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], export_data) + + def _assert_expected_search_error(self, error, **kwargs): + response, search_hash, _ = self._execute_search(**kwargs) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': error}) + return search_hash + + def _format_cached_variants(self, variants, cached_variant_fields=None): + return [ self._get_cached_variant(variant, (cached_variant_fields[i] if cached_variant_fields else None)) for i, variant in enumerate(variants) ] - results_cache = {'all_results': cached_variants, 'total_results': total} - self.assert_cached_results(results_cache, sort=sort, cache_key=f'search_results__{results_model.guid}__{sort}') @classmethod def _get_cached_variant(cls, variant, cached_variant_fields): @@ -148,26 +240,12 @@ def _get_cached_variant(cls, variant, cached_variant_fields): **(cached_variant_fields or {}), } - def _set_multi_project_search(self): - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11'])) - - def _set_single_family_search(self): - self.results_model.families.set(self.families.filter(guid='F000002_2')) - - def _set_sv_family_search(self): - self.results_model.families.set(Family.objects.filter(guid__in=['F000014_14'])) - - def _reset_search_families(self): - self.results_model.families.set(self.families) - def _set_grch37_search(self): Project.objects.filter(id=1).update(genome_version='37') Sample.objects.filter(sample_id='HG00732').update(is_active=False) Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False) - self._set_single_family_search() def test_single_family_search(self): - self._set_single_family_search() variant_gene_counts = { 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, @@ -175,63 +253,57 @@ def test_single_family_search(self): } self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4], gene_counts=variant_gene_counts, locus={'rawItems': '1:1-100000000'}, - **ALL_SNV_INDEL_PASS_FILTERS, + exclude_svs=True, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) self._assert_expected_search( [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=MITO_GENE_COUNTS, locus={'rawItems': 'M:1-100000000'}, - annotations=None, pathogenicity=None, + exclude_svs=False, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts=GCNV_GENE_COUNTS, locus=None, annotations={'structural': COMP_HET_ALL_PASS_FILTERS['annotations']['structural']}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, annotations=None, + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, + project_families=SV_PROJECT_FAMILIES, ) - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000014_14'])) self._assert_expected_search( - [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, - GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, + [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, + GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT]) + self._assert_expected_search([GRCH37_VARIANT], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, is_37=True) def test_standard_searches(self): - results_model = self._saved_search_results_model('De Novo/Dominant Restrictive') self._assert_expected_search( - [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': None}, {}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='De Novo/Dominant Restrictive').search, + check_login=self.check_collaborator_login, ) - results_model = self._saved_search_results_model('De Novo/Dominant Permissive') self._assert_expected_search( - [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': None}, {}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='De Novo/Dominant Permissive').search, ) - results_model = self._saved_search_results_model('Recessive Restrictive') - self._assert_expected_search([VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + self._assert_expected_search([VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][0]}, {}, - ]) + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='Recessive Restrictive').search) - results_model = self._saved_search_results_model('Recessive Permissive') - self._assert_expected_search([VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + self._assert_expected_search([VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][0]}, {}, - ]) - - def _saved_search_results_model(self, name): - results_model = VariantSearchResults.objects.create(variant_search=VariantSearch.objects.get(name=name), search_hash=name) - results_model.families.set(self.families.filter(guid='F000002_2')) - return results_model + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='Recessive Permissive').search) def test_single_project_search(self): variant_gene_counts = { @@ -242,7 +314,7 @@ def test_single_project_search(self): } self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, - GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], check_login=self.check_collaborator_login, gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} ) @@ -264,38 +336,78 @@ def test_single_project_search(self): ) def test_multi_project_search(self): - self._set_multi_project_search() gene_counts = { **GENE_COUNTS, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2, 'F000011_11': 1}}, } + locus = {'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'} self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts=gene_counts, locus={'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + gene_counts=gene_counts, locus=locus, check_login=self.check_collaborator_login, project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) + self.login_manager() self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, - MITO_VARIANT2, MITO_VARIANT3], gene_counts={**gene_counts, **SV_GENE_COUNTS}, + MITO_VARIANT2, MITO_VARIANT3], gene_counts={**gene_counts, **SV_GENE_COUNTS}, locus=locus, project_families=[ + *MULTI_PROJECT_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES, + ] + ) + + def test_all_project_search(self): + self.login_collaborator() + request_body = {'allGenomeProjectFamilies': '38'} + project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': [ + 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', + 'F000009_9', 'F000010_10', 'F000013_13', + ]}] + additional_response = {'familiesByGuid': {'F000001_1': mock.ANY}} + self._assert_expected_search( + [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, + GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + request_body=request_body, project_families=project_families, additional_response=additional_response, + ) + + self.login_manager() + locus = {'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, chr21:1-100000000, M:1-100000000'} + project_families += SV_PROJECT_FAMILIES + results = [ + VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, + GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, + ] + self._assert_expected_search( + results, request_body=request_body, project_families=project_families, additional_response=additional_response, locus=locus, + ) + + request_body['unsolvedFamiliesOnly'] = True + project_families[0]['familyGuids'].remove('F000007_7') + project_families[0]['familyGuids'].remove('F000010_10') + self._assert_expected_search( + results, request_body=request_body, project_families=project_families, additional_response=additional_response, locus=locus, + ) + + request_body['trioFamiliesOnly'] = True + self._assert_expected_search( + [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, + GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + request_body=request_body, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, locus=locus, ) def test_both_sample_types_search(self): Sample.objects.filter(dataset_type='MITO').update(is_active=False) # One family (F000011_11) in a multi-project search has identical exome and genome data. - self._set_multi_project_search() self._add_sample_type_samples('WES', individual__family__guid='F000011_11') self._assert_expected_search( MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) - self._set_single_family_search() self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731']) # Variant 1 is de novo in exome but inherited and homozygous in genome. @@ -305,34 +417,34 @@ def test_both_sample_types_search(self): self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode='any_affected', locus=None, + inheritance_mode='any_affected', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4], - inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, + inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', quality_filter=None, + inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode='any_affected', + inheritance_mode='any_affected', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', + inheritance_mode='de_novo', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', quality_filter={'min_gq': 40} + inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self.maxDiff = None @@ -340,7 +452,7 @@ def test_both_sample_types_search(self): [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4], [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], + GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, @@ -362,19 +474,19 @@ def test_inheritance_filter(self): self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - inheritance_mode=inheritance_mode, + inheritance_mode=inheritance_mode, check_login=self.check_collaborator_login, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], inheritance_mode=inheritance_mode, + project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, + [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT3], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, ) @@ -385,14 +497,12 @@ def test_inheritance_filter(self): inheritance_mode=inheritance_mode, annotations=None, ) - self._set_sv_family_search() sv_affected = {'affected': {'I000019_na21987': 'N'}} self._assert_expected_search( - [SV_VARIANT1], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, + [SV_VARIANT1], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, project_families=SV_PROJECT_FAMILIES, ) inheritance_mode = 'x_linked_recessive' - self._reset_search_families() self._assert_expected_search([], inheritance_mode=inheritance_mode) # self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) @@ -402,29 +512,25 @@ def test_inheritance_filter(self): inheritance_mode=inheritance_mode, ) - self._set_multi_project_search() self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2, GCNV_VARIANT3, MITO_VARIANT3], + [PROJECT_2_VARIANT1, VARIANT2, GCNV_VARIANT3, MITO_VARIANT3], project_families=MULTI_PROJECT_PROJECT_FAMILIES, inheritance_mode=inheritance_mode, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4], inheritance_mode=inheritance_mode, locus=None, + [SV_VARIANT4], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, project_families=SV_PROJECT_FAMILIES, ) gt_inheritance_filter = {'genotype': {'I000006_hg00733': 'ref_ref', 'I000005_hg00732': 'has_alt'}} - self._set_single_family_search() - self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_filter=gt_inheritance_filter) + self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_filter=gt_inheritance_filter, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( [VARIANT2, GCNV_VARIANT2, GCNV_VARIANT3], inheritance_mode='any_affected', inheritance_filter={'affected': { 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'U', - }}, + }}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) inheritance_mode = 'compound_het' - self._reset_search_families() self._assert_expected_search( [[VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, inheritance_filter={}, gene_counts={ 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, @@ -459,17 +565,15 @@ def test_inheritance_filter(self): ] ) - self._set_sv_family_search() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, gene_counts={'ENSG00000171621': {'total': 2, 'families': {'F000014_14': 2}}}, inheritance_filter=sv_affected, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], - ], + ], project_families=SV_PROJECT_FAMILIES, ) inheritance_mode = 'recessive' - self._set_multi_project_search() self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, gene_counts={ 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, @@ -480,16 +584,15 @@ def test_inheritance_filter(self): {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, ], **ALL_SNV_INDEL_PASS_FILTERS, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) - self._set_single_family_search() self._assert_expected_search( [VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], locus=None, + ], locus=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **ALL_SNV_INDEL_PASS_FILTERS, ) - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, gene_counts={ @@ -521,7 +624,6 @@ def test_inheritance_filter(self): ], ) - self._set_sv_family_search() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, gene_counts={ @@ -529,7 +631,7 @@ def test_inheritance_filter(self): 'ENSG00000184986': {'total': 1, 'families': {'F000014_14': 1}}, }, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, - ], + ], project_families=SV_PROJECT_FAMILIES, inheritance_filter=sv_affected, ) # Test deletion in trans with hom alt snp/indel @@ -539,44 +641,48 @@ def test_inheritance_filter(self): sample.save() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4], - inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, + inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, {}, - ], + ], project_families=SV_PROJECT_FAMILIES, ) self._set_grch37_search() - self._assert_expected_search([], inheritance_mode=inheritance_mode) + self._assert_expected_search([], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( [GRCH37_VARIANT], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}, + **COMP_HET_ALL_PASS_FILTERS, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, is_37=True, ) def test_exclude_previous_search_results(self): - VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') - self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + self.mock_results_guid.return_value = 'VRS00079516' + vsr = VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') + cache_key = f'search_results__{vsr.guid}__gnomad' + self.set_cache(cache_key, [ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], - ]})] - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] + ]) + request_body = {'projectFamilies': DEFAULT_PROJECT_FAMILIES, 'previousSearchHash': 'abc1234'} + exclude = {'previousSearch': True} + response_search = {'exclude': {**exclude, 'previousSearchHash': 'abc1234'}} self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - exclude={'previousSearch': True, 'previousSearchHash': 'abc1234'}, cached_variant_fields=[ + request_body=request_body, response_search=response_search, exclude=exclude, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], check_login=self.check_collaborator_login, ) - self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + self.set_cache(cache_key, [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, - ]})] - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] + ]) self._assert_expected_search( - [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', cached_variant_fields=[ + [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, + request_body=request_body, response_search=response_search, inheritance_mode='recessive', cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], ], ) @@ -585,16 +691,15 @@ def test_quality_filter(self): quality_filter = {'vcf_filter': 'pass'} self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], - quality_filter=quality_filter + quality_filter=quality_filter, check_login=self.check_collaborator_login, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT4], quality_filter=quality_filter, + [SV_VARIANT4], quality_filter=quality_filter, project_families=SV_PROJECT_FAMILIES, ) gcnv_quality_filter = {'min_gq': 40, 'min_qs': 20, 'min_hl': 5} - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT3], quality_filter=gcnv_quality_filter, ) @@ -604,16 +709,14 @@ def test_quality_filter(self): ) sv_quality_filter = {'min_gq_sv': 40} - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, annotations=None, + [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, annotations=None, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, + [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, MITO_VARIANT1, MITO_VARIANT2], quality_filter={'min_gq': 40, 'min_qs': 30, 'vcf_filter': 'pass'}, annotations=None, @@ -624,12 +727,10 @@ def test_quality_filter(self): quality_filter={'min_gq': 60, 'min_qs': 10, 'affected_only': True}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, + [SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT1, VARIANT2, FAMILY_3_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4,MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], quality_filter={'min_ab': 50}, ) @@ -683,14 +784,15 @@ def test_quality_filter(self): quality_filter = {'min_gq': 1, 'min_ab': 10} self._assert_expected_search([], quality_filter=quality_filter, annotations=None, pathogenicity=None) self._assert_expected_search( - [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, + [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, is_37=True, ) + @mock.patch('seqr.views.apis.variant_search_api.MAX_FAMILIES_PER_ROW', 1) def test_location_search(self): self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4], **LOCATION_SEARCH, cached_variant_fields=[ {'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'} - ], + ], check_login=self.check_collaborator_login, export_data=SPLIT_FAMILY_EXPORT_DATA, ) sv_locus = {'rawItems': 'ENSG00000275023, ENSG00000171621'} @@ -698,28 +800,25 @@ def test_location_search(self): [GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, ) - self._set_sv_family_search() + self.login_manager() # For gene search, return SVs annotated in gene even if they fall outside the gene interval self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, + [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'} + [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT1, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], exclude={ 'rawItems': ','.join([LOCATION_SEARCH['locus']['rawItems'], sv_locus['rawItems']]) }, locus=None, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT2, SV_VARIANT3], exclude=sv_locus, + [SV_VARIANT2, SV_VARIANT3], exclude=sv_locus, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT, VARIANT4], locus={'rawItems': f'{GENE_IDS[1]}\n1:91500851-91525764'}, exclude=None, cached_variant_fields=[ @@ -728,16 +827,16 @@ def test_location_search(self): ) self._add_sample_type_samples('WES', individual__family__guid='F000014_14') - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000014_14'])) self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, + project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT], locus={'rawItems': '7:143268894-143271480'}) + self._assert_expected_search([GRCH37_VARIANT], locus={'rawItems': '7:143268894-143271480'}, is_37=True) def test_variant_id_search(self): - self._assert_expected_search([VARIANT1], **VARIANT_ID_SEARCH) + self._assert_expected_search([VARIANT1], **VARIANT_ID_SEARCH, check_login=self.check_collaborator_login) self._assert_expected_search( [VARIANT1], locus={'rawVariantItems': VARIANT_IDS[0]}, @@ -747,140 +846,565 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) + @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) + @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) + @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([]) - self.assertEqual(str(cm.exception),'This search returned too many results') + url = reverse(query_variants_handler, args=['not_a_hash']) + self.check_require_login(url) + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Invalid search hash: not_a_hash'}) + self.login_collaborator() + + self._assert_expected_search_error('Invalid search: no projects/ families specified', project_families=[]) + + search_hash = self._assert_expected_search_error( + 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, + ) + + self.set_cache(f'search_results__VRS{search_hash:07d}__xpos', [VARIANT1, VARIANT2, VARIANT3, VARIANT4]) + export_url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(export_url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Unable to export more than 2 variants (4 requested)'}) + + self._assert_expected_search_error('Invalid variants: rs9876', locus={'rawVariantItems': 'rs9876'}) + + self._assert_expected_search_error( + 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345', + locus={'rawItems': 'chr27:1234-5678,2:40-400000000, ENSG00012345'}, + ) + + self._assert_expected_search_error('Too many genes/intervals', locus={'rawItems': '1:1-1000, 2:2000-3000, 3:4000-5000'}) + + build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' + self._assert_expected_search_error('Invalid genes/intervals: OR4F29, ENSG00000256186', locus={'rawItems': build_specific_genes}) + + self._assert_expected_search_error( + 'Cannot specify both Location and Excluded Genes/Intervals', + locus={'rawItems': build_specific_genes}, exclude={'rawItems': build_specific_genes}, + ) + + self._assert_expected_search_error('Invalid genes/intervals: OR4F29, ENSG00000256186', exclude={'rawItems': build_specific_genes}) + + self._assert_expected_search_error('This search returned too many results') + + self._assert_expected_search_error( + 'ClinVar pathogenicity vus is both included and excluded', + pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}, + ) + + self._assert_expected_search_error('Annotations must be specified to search for compound heterozygous variants', inheritance_mode='recessive') + + annotations = {'frameshift': ['frameshift_variant']} + self._assert_expected_search_error( + 'Location must be specified to search for compound heterozygous variants across many families', + annotations=annotations, inheritance_mode='recessive', + ) + + self._assert_expected_search_error( + 'Phenotype sort is only supported for single-family search.', + sort='prioritized_gene', annotations={'frameshift': ['frameshift_variant']}, + ) + + self._assert_expected_search_error('seqr AC frequency of at least 5000 must be specified to search across multiple families', freqs={}) + + self._assert_expected_search_error( + 'Inheritance based search is disabled in families with no data loaded for affected individuals', + inheritance_mode='recessive', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000005_5']}], + annotations=annotations, + ) + + no_sv_project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000003_3']}] + self._assert_expected_search_error( + 'Unable to search against dataset type "SV"', pathogenicity={}, annotations={'structural': ['DEL']}, + project_families=no_sv_project_families, + ) + + self._assert_expected_search_error( + 'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals', + inheritance_mode='recessive', annotations={'structural': ['DEL']}, annotations_secondary={'frameshift': ['frameshift_variant']}, project_families=no_sv_project_families, + ) + + self._assert_expected_search_error( + 'Inheritance based search is disabled in families with no data loaded for affected individuals', + inheritance_mode='recessive', inheritance_filter={'affected': {'I000007_na20870': 'N'}}, project_families=no_sv_project_families, + annotations=annotations, + ) + + self._assert_expected_search_error('Inheritance must be specified if custom affected status is set', inheritance_filter={'affected': {'I000007_na20870': 'N'}}) + + self._assert_expected_search_error( + 'Invalid custom inheritance', project_families=no_sv_project_families, inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref'}}, + ) + + self._assert_expected_search_error( + 'No search data found for families no_individuals', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000013_13']}], + ) + + self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES) Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') - self._set_multi_project_search() - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}) - self.assertEqual( - str(cm.exception), + self._assert_expected_search_error( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', + locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}, project_families=MULTI_PROJECT_PROJECT_FAMILIES, + ) + + self._set_grch37_search() + self._assert_expected_search_error('Invalid genes/intervals: DDX11L1, ENSG00000223972', locus={'rawItems': build_specific_genes}) + + search_hash = self._assert_expected_search_error( + 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project', + project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) + # Test export disabled in demo projects + export_url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(export_url) + self.assertEqual(response.status_code, 403) + @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] + EntriesSv.objects.using('clickhouse_write').filter(family_guid='F000002_2_x').delete() + EntriesGcnv.objects.using('clickhouse_write').filter(family_guid='F000002_2_x').delete() + + self.check_require_login(reverse(variant_lookup_handler)) + + lookup_variant = { + **VARIANT1, + 'liftedFamilyGuids': ['F000014_14'], + 'familyGuids': [VARIANT1['familyGuids'][0], 'F000011_11'], + 'genotypes': { + **VARIANT1_BOTH_SAMPLE_TYPES['genotypes'], + 'I000015_na20885': [ + {**PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], 'sampleType': 'WES'}, + PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], + ], + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531, 'filters': [], 'individualGuid': 'I000018_na21234', + }, + }, + } + expected_variant = { + **VARIANT1, + 'familyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A'], + 'liftedFamilyGuids': ['F2_1-10439-AC-A'], + 'genotypes': { + 'I0_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I1_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I2_F0_1-10439-AC-A': [ + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F1_1-10439-AC-A': [ + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], + 'sampleType': 'WGS'}, + }, + } + expected_individuals = { + 'I0_F0_1-10439-AC-A': { + 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], + 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I0_F1_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F1_1-10439-AC-A', 'individualGuid': 'I0_F1_1-10439-AC-A', 'sex': 'M', + 'features': [{'category': 'HP:0001626', 'label': '1 terms'}, {'category': 'Other', 'label': '1 terms'}], + 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, + 'I0_F2_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], + 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_1-10439-AC-A': { + 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], + 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, + {'category': 'HP:0001626', 'label': '1 terms'}], + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + cache_key = 'variant_lookup_results__1-10439-AC-A__38' + self._assert_expected_lookup( + '1-10439-AC-A', expected_variant, cache_key, cached_variants=[lookup_variant], skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, expected_individuals=expected_individuals, locusListsByGuid={}, + ) - variants = variant_lookup(self.user, '1-10439-AC-A', '38') - self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT]) - - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, '1-91511686-TCA-G', '38') - self.assertEqual(str(cm.exception), 'Variant not present in seqr') + expected_gcnv_variant = { + **GCNV_VARIANT4, + 'familyGuids': ['F0_suffix_140608_DUP'], + 'genotypes': { + 'I0_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_variant = { + **SV_VARIANT4, + 'familyGuids': ['F0_phase2_DEL_chr14_4640'], + 'genotypes': { + 'I0_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_individuals = { + 'I0_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I0_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', + 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I1_F0_phase2_DEL_chr14_4640': { + 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', + 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I2_F0_suffix_140608_DUP': { + 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, + {'category': 'HP:0001626', 'label': '1 terms'}], + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + sv_genes = { + 'ENSG00000184986': mock.ANY, 'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY, + } + self._assert_expected_lookup( + 'phase2_DEL_chr14_4640', expected_sv_variant, 'variant_lookup_results__phase2_DEL_chr14_4640__38', + additional_variant=expected_gcnv_variant, cached_variants=[SV_VARIANT4, GCNV_VARIANT4], skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById=sv_genes, + ) + + self.login_manager() + individual_guid_map = { + 'I000006_hg00733': 'I0_F0_1-10439-AC-A', + 'I000005_hg00732': 'I1_F0_1-10439-AC-A', + 'I000004_hg00731': 'I2_F0_1-10439-AC-A', + 'I000015_na20885': 'I0_F1_1-10439-AC-A', + 'I000018_na21234': 'I0_F2_1-10439-AC-A', + } + expected_individuals = { + individual_guid: { + **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, + **{k: v for k, v in expected_individuals[anon_individual_guid].items() + if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, + } for individual_guid, anon_individual_guid in individual_guid_map.items() + } + expected_individuals.update( + {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) + self._assert_expected_lookup( + '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, + project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], + ) - variants = variant_lookup(self.user, '7-143270172-A-G', '37') - grch37_lookup_variant = { - **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() - ], key=lambda x: x['sampleId'], reverse=True)}, + hom_only_lookup_variant = { + **lookup_variant, + 'genotypes': { + **lookup_variant['genotypes'], + **{guid: VARIANT1_BOTH_SAMPLE_TYPES['genotypes'][guid][1] for guid in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', + ]}, + }, } - self._assert_expected_variants(variants, [grch37_lookup_variant]) + del hom_only_lookup_variant['liftedFamilyGuids'] + del hom_only_lookup_variant['genotypes']['I000018_na21234'] + self._assert_expected_lookup( + '1-10439-AC-A', hom_only_lookup_variant, f'{cache_key}__hom', hom_only=True, + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], + ) + + url = self._assert_expected_lookup( + '21-3343353-GAGA-G', FAMILY_1_VARIANT, 'variant_lookup_results__21-3343353-GAGA-G__38', project_guids=['R0001_1kg'], + family_guids=['F000001_1'], individual_guids=['I000001_na19675', 'I000002_na19678', 'I000003_na19679'], + familyNotesByGuid={'FAN000001_1': mock.ANY, 'FAN000001_2': mock.ANY, 'FAN000001_3': mock.ANY}, + genesById={'ENSG00000227232': mock.ANY, 'ENSG00000268903': mock.ANY}, + igvSamplesByGuid={'S000145_na19675': mock.ANY}, + mmeSubmissionsByGuid={'MS000001_na19675': mock.ANY}, + phenotypeGeneScores={'I000001_na19675': mock.ANY, 'I000002_na19678': mock.ANY}, + rnaSeqData={'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}}, + savedVariantsByGuid={'SV0000001_2103343353_r0390_100': mock.ANY }, + variantTagsByGuid={'VT1708633_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0390_100': mock.ANY}, + variantFunctionalDataByGuid={ + 'VFD0000023_1248367227_r0390_10': mock.ANY, 'VFD0000024_1248367227_r0390_10': mock.ANY, + 'VFD0000025_1248367227_r0390_10': mock.ANY, 'VFD0000026_1248367227_r0390_10': mock.ANY, + }, + ) + + response = self.client.get(url + '&affectedOnly=true') + self.assertEqual(response.status_code, 404) + self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) + + url = url.replace('21-3343353-GAGA-G', '1-91511686-TCA-G') + response = self.client.get(url) + self.assertEqual(response.status_code, 404) + self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) + + self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [{ + **VARIANT1, + 'familyGenotypes': {'F000002_2': list(VARIANT1['genotypes'].values())}, + }]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json()['variants'][0], { + **VARIANT1, + 'familyGuids': [], + 'lookupFamilyGuids': VARIANT1['familyGuids'], + }) - variants = variant_lookup(self.user, '7-143270172-A-G', '37', hom_only=True, affected_only=True) - self._assert_expected_variants(variants, [grch37_lookup_variant]) + cache_key = 'variant_lookup_results__7-143270172-A-G__37' + self._assert_expected_lookup( + '7-143270172-A-G', GRCH37_VARIANT, cache_key, genome_version='37', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) + + self._assert_expected_lookup( + '7-143270172-A-G', GRCH37_VARIANT, f'{cache_key}__affected__hom', genome_version='37', + hom_only=True, affected_only=True, project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) # Lookup works if variant is only present on a different build - variants = variant_lookup(self.user, '7-143260172-A-G', '38') - self._assert_expected_variants(variants, [grch37_lookup_variant]) + self._assert_expected_lookup( + '7-143260172-A-G', GRCH37_VARIANT, 'variant_lookup_results__7-143260172-A-G__38', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'], genesById={ + 'ENSG00000176227': mock.ANY, 'ENSG00000271079': mock.ANY, + }, + ) mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) - liftover_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGenotypes': { - family_guid: gts - for family_guid, gts in VARIANT_LOOKUP_VARIANT['familyGenotypes'].items() if family_guid != 'F000014_14' - }, - } - del liftover_variant['liftedFamilyGuids'] - variants = variant_lookup(self.user, '1-439-AC-A', '37') - self._assert_expected_variants(variants, [liftover_variant]) + del lookup_variant['liftedFamilyGuids'] + del lookup_variant['genotypes']['I000018_na21234'] + self._assert_expected_lookup( + '1-439-AC-A', lookup_variant, 'variant_lookup_results__1-439-AC-A__37', genome_version='37', + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], + ) mock_liftover.assert_called_with('hg19', 'hg38') mock_convert_coordinate.assert_called_with('chr1', 439) - hom_only_lookup_variant = { - **liftover_variant, - 'familyGenotypes': { - **liftover_variant['familyGenotypes'], - 'F000002_2': [gt for gt in liftover_variant['familyGenotypes']['F000002_2'] if gt['sampleType'] == 'WGS'], - }, - } - variants = variant_lookup(self.user, '1-10439-AC-A', '38', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant]) - variants = variant_lookup(self.user, '1-439-AC-A', '37', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant]) - - variants = variant_lookup(self.user, 'M-4429-G-A', '38') - self._assert_expected_variants(variants, [{ - **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() - ]}, - }]) + self._assert_expected_lookup( + '1-439-AC-A', { + **lookup_variant, 'genotypes': hom_only_lookup_variant['genotypes'], + }, 'variant_lookup_results__1-439-AC-A__37__hom', genome_version='37', hom_only=True, + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], + ) - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, 'M-4429-G-A', '38', hom_only=True) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') + url = self._assert_expected_lookup( + 'M-4429-G-A', MITO_VARIANT1, 'variant_lookup_results__M-4429-G-A__38', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], + ) - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) + url = url.replace('38', '37') + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'MITO variants are not available for GRCh37'}) - affected_only_lookup_variant = { - **GCNV_LOOKUP_VARIANT, - 'familyGenotypes': { - family_guid: gts for family_guid, gts in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if family_guid != 'F000002_2_x' - }, - } - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', affected_only=True) - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, affected_only_lookup_variant]) + url = url.replace('M-4429-G-A', 'phase2_DEL_chr14_4640') + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'SV variants are not available for GRCh37'}) + + url = url.replace('37', '38') + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'Sample type must be specified to look up a structural variant'}) + + self._assert_expected_lookup( + 'phase2_DEL_chr14_4640', SV_VARIANT4, 'variant_lookup_results__phase2_DEL_chr14_4640__38', + additional_variant=GCNV_VARIANT4, sample_type='WGS', genesById=sv_genes, + project_guids=['R0001_1kg', 'R0004_non_analyst_project'], family_guids=['F000002_2', 'F000014_14'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654'], + ) # reciprocal overlap does not meet the threshold for smaller events - variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT]) + self._assert_expected_lookup( + 'suffix_140608_DUP', GCNV_VARIANT4, 'variant_lookup_results__suffix_140608_DUP__38', + sample_type='WES', genesById={'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY}, + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], + ) + + # Test error handling when the ClickHouse sampleId cannot be mapped to any Postgres Individual + self.reset_logs() + Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') + missing_gt_variant = { + **GRCH37_VARIANT, 'genotypes': {k: v for k, v in GRCH37_VARIANT['genotypes'].items() if k != 'I000006_hg00733'}, + } + self._assert_expected_lookup( + '7-143270172-A-G', missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) + self.assert_json_logs(self.manager_user, [ + ('Looking up variant 7-143270172-A-G with data type SNV_INDEL', None), + ('Unable to map sample HG00733 in family F000002_2 to an individual for variant 7-143270172-A-G', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) - variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES', affected_only=True) - self._assert_expected_variants(variants, [affected_only_lookup_variant]) + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): + url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' + if hom_only: + url += '&homOnly=true' + if affected_only: + url += '&affectedOnly=true' + if sample_type: + url += f'&sampleType={sample_type}' - variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + context_fields = {*SEARCH_RESPONSE_KEYS, 'igvSamplesByGuid', 'locusListsByGuid', 'familyNotesByGuid'} + if skip_fields: + context_fields -= skip_fields + if genome_version == '38': + context_fields.add('omimIntervals') + variants = [variant, *([additional_variant] if additional_variant else [])] + expected_body = { + **{k: {} for k in context_fields}, + 'projectsByGuid': {p: mock.ANY for p in project_guids or []}, + 'familiesByGuid': {f: mock.ANY for f in family_guids or []}, + 'individualsByGuid': expected_individuals or {i: mock.ANY for i in individual_guids or []}, + 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, + 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, + 'variants': [{ + **v, + 'familyGuids': [], + 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []), + } for v in variants], + **kwargs, + } + self.assertDictEqual(response.json(), expected_body) + + parsed_cached_variants = [] + for v in (cached_variants or variants): + family_genotypes = defaultdict(list) + for individual_guid, gts in v['genotypes'].items(): + if not isinstance(gts, list): + gts = [gts] + for gt in gts: + family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] + family_genotypes[family_guid].append({k: v for k, v in gt.items() if k != 'individualGuid'}) + parsed_cached_variants.append({ + **{k: v for k, v in v.items() if k not in {'familyGuids', 'genotypes'}}, + 'familyGenotypes': { + family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) + for family_guid, gts in family_genotypes.items() + }, + }) + self.assert_cached_results(parsed_cached_variants, cache_key) + return url def test_get_single_variant(self): - self._set_single_family_search() - variant = get_single_variant(self.results_model.families.first(), VARIANT_IDS[0]) - self._assert_expected_variants([variant], [VARIANT1]) + url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') + url = url_template.format('21-3343353-GAGA-G', 'F000001_1') + self.check_collaborator_login(url) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'searchedVariants': [FAMILY_1_VARIANT], + 'familiesByGuid': {'F000001_1': mock.ANY}, + 'projectsByGuid': {'R0001_1kg': mock.ANY}, + 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': { + key: mock.ANY for key in ['paLocusList', 'intervals', *LOCUS_LIST_FIELDS] + }}, + **{key: mock.ANY for key in [ + 'omimIntervals', 'individualsByGuid', 'igvSamplesByGuid', 'familyNotesByGuid', *SEARCH_RESPONSE_KEYS, + ]}, + }) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.results_model.families.first(), VARIANT_IDS[1]) - self.assertEqual(str(cm.exception), 'Variant 1-91511686-TCA-G not found') + response = self.client.get(url_template.format(VARIANT_IDS[1], 'F000002_2')) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Variant 1-91511686-TCA-G not found'}) - variant = get_single_variant(self.results_model.families.first(), 'M-4429-G-A') - self._assert_expected_variants([variant], [MITO_VARIANT1]) + response = self.client.get(url_template.format('M-4429-G-A', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [MITO_VARIANT1]) - variant = get_single_variant(self.results_model.families.first(), 'suffix_140608_DUP') - self._assert_expected_variants([variant], [GCNV_VARIANT4]) + response = self.client.get(url_template.format('suffix_140608_DUP', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [GCNV_VARIANT4]) - self._set_sv_family_search() - variant = get_single_variant(self.results_model.families.first(), 'phase2_DEL_chr14_4640') - self._assert_expected_variants([variant], [SV_VARIANT4]) + self.login_manager() + response = self.client.get(url_template.format('phase2_DEL_chr14_4640', 'F000014_14')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [SV_VARIANT4]) self._set_grch37_search() - variant = get_single_variant(self.results_model.families.first(), '7-143270172-A-G') - self._assert_expected_variants([variant], [GRCH37_VARIANT]) + response = self.client.get(url_template.format('7-143270172-A-G', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [GRCH37_VARIANT]) + self.mock_redis.get.assert_not_called() + self.mock_redis.set.assert_not_called() def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {'ac': 7}, **sv_callset_filter}, + freqs={'callset': {'ac': 7}, **sv_callset_filter}, check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -891,12 +1415,11 @@ def test_frequency_filter(self): [MULTI_FAMILY_VARIANT, GCNV_VARIANT3, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], freqs={'callset': {'ac': 7, 'hh': 0}, 'sv_callset': {'ac': 50}}, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT1], freqs={'sv_callset': {'ac': 1}}, + [SV_VARIANT1], freqs={'sv_callset': {'ac': 1}}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], freqs={'callset': {'ac': 1000}, 'gnomad_genomes': {'af': 0.03}, 'gnomad_mito': {'af': 0.005}}, ) @@ -910,29 +1433,27 @@ def test_frequency_filter(self): freqs={'callset': {'ac': 1000}, 'topmed': {'af': 0.05, 'hh': 1}, 'sv_callset': {'ac': 50}}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, + [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'ac': 4000}}, + [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'ac': 4000}}, project_families=SV_PROJECT_FAMILIES, ) # seqr af filter is ignored for SNV_INDEL - self._set_single_family_search() self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {'af': 0.2}, **sv_callset_filter}, + freqs={'callset': {'af': 0.2}, **sv_callset_filter}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], - freqs={'callset': {'ac': 6}, 'gnomad_genomes': {'ac': 50}, 'gnomad_mito': {'ac': 10}}, + freqs={'callset': {'ac': 6}, 'gnomad_genomes': {'ac': 50}, 'gnomad_mito': {'ac': 10}}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {}, 'gnomad_genomes': {'af': None}}, + freqs={'callset': {}, 'gnomad_genomes': {'af': None}}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters @@ -940,15 +1461,16 @@ def test_frequency_filter(self): [VARIANT1, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.002, 'hh': 10}, 'gnomad_mito': {'ac': 1000}}, annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus']}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.002}, 'gnomad_mito': {'ac': 1000}}, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus']}, + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus']}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) def test_annotations_filter(self): - self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}) + self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}, check_login=self.check_collaborator_login) self._assert_expected_search([], pathogenicity={'hgmd': ['disease_causing', 'likely_disease_causing']}) clinvar_paths = ['likely_pathogenic', 'conflicting_p_lp', 'conflicting_no_p', 'vus', 'benign'] @@ -1010,12 +1532,11 @@ def test_annotations_filter(self): ], ) - self._set_sv_family_search() - self._assert_expected_search([SV_VARIANT1], annotations=annotations) + self.login_manager() + self._assert_expected_search([SV_VARIANT1], annotations=annotations, project_families=SV_PROJECT_FAMILIES) annotations['splice_ai'] = '0.005' annotations['structural'] = ['gCNV_DUP', 'DEL'] - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], annotations=annotations, cached_variant_fields=[ @@ -1026,11 +1547,16 @@ def test_annotations_filter(self): ] ) - self._set_sv_family_search() - self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations) + annotations['missense'] = [] + annotations['in_frame'] = [] + del annotations['splice_ai'] + self._assert_expected_search( + [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], annotations=annotations, + ) + + self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations, project_families=SV_PROJECT_FAMILIES) annotations = {'other': ['non_coding_transcript_exon_variant__canonical', 'non_coding_transcript_exon_variant']} - self._set_single_family_search() self._assert_expected_search( [VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, cached_variant_fields=[ @@ -1038,13 +1564,13 @@ def test_annotations_filter(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._reset_search_families() + locus = {'rawItems': f'{GENE_IDS[1]}\n1:11785723-91525764'} self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2], - locus={'rawItems': f'{GENE_IDS[1]}\n1:11785723-91525764'}, pathogenicity=None, annotations=annotations, + locus=locus, pathogenicity=None, annotations=annotations, cached_variant_fields=[{ 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5], }], @@ -1053,7 +1579,7 @@ def test_annotations_filter(self): annotations['other'].append('intron_variant') self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], - annotations=annotations, cached_variant_fields=[ + annotations=annotations, locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][1]}, ], @@ -1061,12 +1587,11 @@ def test_annotations_filter(self): annotations['other'] = annotations['other'][:1] annotations['splice_ai'] = '0.005' - self._set_single_family_search() self._assert_expected_search( [VARIANT1, VARIANT3, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, locus=None, cached_variant_fields=[ {'selectedTranscript': None}, {'selectedTranscript': None}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations['extended_splice_site'] = ['extended_intronic_splice_region_variant'] @@ -1077,14 +1602,14 @@ def test_annotations_filter(self): {'selectedTranscript': None}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][0]}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'extended_splice_site': ['5_prime_UTR_variant']} self._assert_expected_search( [selected_transcript_variant_2], pathogenicity=None, annotations=annotations, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][1]}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations['extended_splice_site'].append('extended_intronic_splice_region_variant') @@ -1092,12 +1617,12 @@ def test_annotations_filter(self): [selected_transcript_variant_2, VARIANT4], annotations=annotations, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][1]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][0]}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'motif_feature': ['TF_binding_site_variant'], 'regulatory_feature': ['regulatory_region_variant']} self._assert_expected_search( - [VARIANT3, VARIANT4], annotations=annotations, pathogenicity=None, + [VARIANT3, VARIANT4], annotations=annotations, pathogenicity=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_grch37_search() @@ -1105,7 +1630,7 @@ def test_annotations_filter(self): annotations['missense'] = ['missense_variant'] self._assert_expected_search( [GRCH37_VARIANT], pathogenicity=None, annotations=annotations, - cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[11][0]}], + cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[11][0]}], is_37=True, ) def test_secondary_annotations_filter(self): @@ -1120,6 +1645,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='compound_het', annotations=annotations_1, annotations_secondary=annotations_2, cached_variant_fields=[comp_het_cached_fields], + check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1186,12 +1712,13 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}]], ) + locus = {'rawItems': 'ENSG00000277258,ENSG00000275023'} self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, 'structural': ['gCNV_DEL'], 'structural_consequence': ['INTRONIC']}, annotations_secondary={**annotations_2, **gcnv_annotations_1}, - locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, cached_variant_fields=[ + locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, @@ -1201,7 +1728,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, 'structural': [], 'structural_consequence': []}, - annotations_secondary=gcnv_annotations_2, cached_variant_fields=[ + annotations_secondary=gcnv_annotations_2, locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, @@ -1211,12 +1738,12 @@ def test_secondary_annotations_filter(self): sv_annotations_1 = {'structural': ['INS', 'LOF']} sv_annotations_2 = {'structural': ['DEL', 'gCNV_DUP'], 'structural_consequence': ['INTRONIC']} - self._set_sv_family_search() + self.login_manager() + inheritance_filter = {'affected': {'I000019_na21987': 'N'}} self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode='compound_het', locus=None, - annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, inheritance_filter={'affected': { - 'I000019_na21987': 'N', - }}, cached_variant_fields=[ + annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, inheritance_filter=inheritance_filter, + project_families=SV_PROJECT_FAMILIES, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], ], ) @@ -1225,11 +1752,10 @@ def test_secondary_annotations_filter(self): [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode='recessive', annotations=sv_annotations_2, annotations_secondary=sv_annotations_1, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, - ], + ], inheritance_filter=inheritance_filter, project_families=SV_PROJECT_FAMILIES, ) pathogenicity = {'clinvar': ['likely_pathogenic', 'conflicting_p_lp', 'conflicting_no_p', 'vus']} - self._reset_search_families() self._assert_expected_search( [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4], MITO_VARIANT3], inheritance_mode='recessive', annotations=annotations_2, annotations_secondary=annotations_1, pathogenicity=pathogenicity, cached_variant_fields=[ @@ -1300,7 +1826,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [VARIANT3, VARIANT4], MITO_VARIANT3], annotations={**selected_transcript_annotations, **screen_annotations}, annotations_secondary=annotations_2, - inheritance_mode='recessive', cached_variant_fields=[ + inheritance_mode='recessive', pathogenicity=pathogenicity, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, [ {'selectedGeneId': 'ENSG00000097046', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][0]}, {'selectedGeneId': 'ENSG00000097046', 'selectedTranscript': None}, @@ -1309,20 +1835,19 @@ def test_secondary_annotations_filter(self): ) self._add_sample_type_samples('WES', individual__family__guid='F000014_14') - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, cached_variant_fields=[ + locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, pathogenicity=pathogenicity, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}]], + project_families=[*MULTI_PROJECT_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], ) # Search works with a different number of samples within the family - self._reset_search_families() missing_gt_gcnv_variant = { **GCNV_VARIANT4, 'familyGuids': ['F000002_2_x'], @@ -1341,13 +1866,14 @@ def test_secondary_annotations_filter(self): annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, cached_variant_fields=[[ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': None}, {'selectedGeneId': 'ENSG00000277258'}, - ]], + ]], project_families=[{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000002_2_x', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids'][1:]]}], ) def test_in_silico_filter(self): main_in_silico = {'eigen': '3.5', 'mut_taster': 'N', 'vest': 0.5} self._assert_expected_search( [VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], in_silico=main_in_silico, + check_login=self.check_collaborator_login, ) in_silico = {**main_in_silico, 'requireScore': True} @@ -1371,29 +1897,26 @@ def test_in_silico_filter(self): [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], in_silico=sv_in_silico, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT4], in_silico=sv_in_silico, + [SV_VARIANT4], in_silico=sv_in_silico, project_families=SV_PROJECT_FAMILIES, ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico) + self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico, is_37=True) def test_sort(self): - self._set_single_family_search() self._assert_expected_search( [VARIANT4, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1, VARIANT2, MITO_VARIANT2, MITO_VARIANT3, MITO_VARIANT1, VARIANT3, VARIANT1], - sort='protein_consequence', + sort='protein_consequence', check_login=self.check_collaborator_login, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._reset_search_families() - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], - sort='protein_consequence', + sort='protein_consequence', project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT4, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, MITO_VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT], sort='protein_consequence', @@ -1409,18 +1932,22 @@ def test_sort(self): [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], sort='pathogenicity', annotations=None, ) - self._set_single_family_search() + self.login_analyst_user() self._assert_expected_search( [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], - sort='pathogenicity_hgmd', + sort='pathogenicity', cache_sort='pathogenicity_hgmd', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) + sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] + self._assert_expected_search(sorted_variants, sort='gnomad', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) + + self._assert_expected_search( + sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', query_params={'page': 2, 'per_page': 4}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + ) self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - sort='gnomad', + sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', query_params={'page': 3, 'per_page': 4}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='gnomad_exomes', @@ -1463,47 +1990,44 @@ def test_sort(self): [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT3, GCNV_VARIANT4, VARIANT4, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort=sort, ) - self._set_single_family_search() - self._assert_expected_search([VARIANT2, VARIANT3, VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='prioritized_gene') + self._assert_expected_search([VARIANT2, VARIANT3, VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='prioritized_gene', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self._set_multi_project_search() self._assert_expected_search( [MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, PROJECT_2_VARIANT], sort='family_guid', locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) # size sort only applies to SVs, so has no impact on other variant - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT1, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='size', locus=None, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT4, SV_VARIANT1, SV_VARIANT3, SV_VARIANT2], sort='size', + [SV_VARIANT4, SV_VARIANT1, SV_VARIANT3, SV_VARIANT2], sort='size', project_families=SV_PROJECT_FAMILIES, ) # sort applies to compound hets - self._set_single_family_search() self._assert_expected_search( [[VARIANT4, VARIANT3], VARIANT2, MITO_VARIANT3], sort='revel', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [[VARIANT3, VARIANT4], VARIANT2, MITO_VARIANT3], sort='splice_ai', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [MITO_VARIANT3, [VARIANT4, VARIANT3], VARIANT2], sort='callset_af', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) def test_multi_data_type_comp_het_sort(self): @@ -1513,7 +2037,8 @@ def test_multi_data_type_comp_het_sort(self): sort='protein_consequence', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], - [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], {}, {}] + [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], {}, {}], + check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1588,143 +2113,303 @@ def test_multi_data_type_comp_het_sort(self): ) def test_gene_variant_lookup(self): - url = reverse(query_variants_handler, args=['abc123']) - self.check_require_login(url) - - body = { + request_body = { 'allGenomeProjectFamilies': '38', 'includeNoAccessProjects': True, - 'search': { - 'annotations': { - 'missense': ['missense_variant'], - 'other': ['non_coding_transcript_exon_variant'], - }, - 'freqs': { - 'callset': {'ac': 3000}, - 'gnomad_genomes': {'af': 0.003}, - 'gnomad_exomes': {'af': 0.003}, - }, - } } - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'error': 'Including external projects is only available when searching for a single gene', - }) + self._assert_expected_search_error( + 'Including external projects is only available when searching for a single gene', request_body=request_body, + check_login=self.check_require_login, + ) - body['search']['locus'] = {'rawItems': 'ENSG00000097046'} - response = self.client.post(url+'1', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) + annotations = { + 'missense': ['missense_variant'], + 'other': ['non_coding_transcript_exon_variant'], + } + freqs = { + 'callset': {'ac': 3000}, + 'gnomad_genomes': {'af': 0.003}, + 'gnomad_exomes': {'af': 0.003}, + } + locus = {'rawItems': 'ENSG00000097046'} + response_search = {'no_access_project_genome_version': '38'} variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} del variant4['familyGuids'] del variant4['genotypes'] - expected_response = { - 'searchedVariants': [variant4], - 'genesById': {'ENSG00000097046': mock.ANY}, - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': [], - 'totalResults': 1, - }, - 'totalSampleCounts': { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - }, - 'locusListsByGuid': {}, - 'mmeSubmissionsByGuid': {}, - 'omimIntervals': {}, - 'phenotypeGeneScores': {}, - 'rnaSeqData': {}, - 'savedVariantsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'variantNotesByGuid': {}, - 'variantTagsByGuid': {}, - } - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [variant4], request_body=request_body, response_search=response_search, + cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], + annotations=annotations, freqs=freqs, locus=locus, project_families=[], + ) - body['search']['freqs'] = {'callset': body['search']['freqs']['callset']} - response = self.client.post(url+'2', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) + freqs = {'callset': freqs['callset']} variant3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611', 'numFamilies': 4} del variant3['familyGuids'] del variant3['genotypes'] - expected_response['searchedVariants'].insert(0, variant3) - expected_response['genesById']['ENSG00000177000'] = mock.ANY - expected_response['search']['search'].update(body['search']) - expected_response['search']['totalResults'] = 2 - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [variant3, variant4], request_body=request_body, response_search=response_search, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, + ], + annotations=annotations, freqs=freqs, locus=locus, project_families=[], + ) - body['search']['inheritance'] = {'mode': 'recessive'} - response = self.client.post(url + '3', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'error': 'Compound heterozygous search is not supported when including external projects', - }) + self._assert_expected_search_error( + 'Compound heterozygous search is not supported when including external projects', + request_body=request_body, locus=locus, annotations=annotations, inheritance_mode='recessive', + ) - body['search']['inheritance']['mode'] = 'homozygous_recessive' - response = self.client.post(url + '4', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': [], - 'totalResults': 0, - }, - 'searchedVariants': [], - }) + self._assert_expected_search( + [], request_body=request_body, response_search=response_search, project_families=[], + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='homozygous_recessive', + ) - body['search']['inheritance']['mode'] = 'de_novo' - response = self.client.post(url + '5', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - expected_response['search']['search'].update(body['search']) variant3['numFamilies'] = 1 variant4['numFamilies'] = 1 - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [variant3, variant4], request_body=request_body, response_search=response_search, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, + ], + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=[], + ) self.login_collaborator() - response = self.client.post(url + '6', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': mock.ANY}] - expected_response['search']['projectFamilies'] = project_families - expected_response.update({ - 'searchedVariants': [ + self._assert_expected_search([ {**FAMILY_3_VARIANT, 'selectedMainTranscriptId': 'ENST00000497611'}, {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997'}, + ], request_body=request_body, response_search=response_search, project_families=project_families, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY}, - 'phenotypeGeneScores': {'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}, - }) - self.assertDictEqual(response.json(), expected_response) + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) - body['search']['locus']['rawItems'] = 'ENSG00000171621' - response = self.client.post(url + '7', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - expected_response['search']['search'].update(body['search']) - expected_response['search']['totalResults'] = 1 + locus['rawItems'] = 'ENSG00000171621' other_project_variant = {**PROJECT_4_COMP_HET_VARIANT, 'numFamilies': 1} del other_project_variant['familyGuids'] del other_project_variant['genotypes'] - expected_response.update({ - 'searchedVariants': [other_project_variant], - 'genesById': {'ENSG00000171621': mock.ANY}, - 'locusListsByGuid': {}, - 'phenotypeGeneScores': {}, + self._assert_expected_search( + [other_project_variant], request_body=request_body, response_search=response_search, + project_families=project_families, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[22][0]}], + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) + + locus['rawItems'] = 'ENSG00000229905' + self._assert_expected_search( + [], request_body=request_body, response_search=response_search, + project_families=project_families, annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) + + def test_search_context(self): + sv = SavedVariant.objects.get(guid='SV0000001_2103343353_r0390_100') + sv.saved_variant_json['genomeVersion'] = '38' + sv.save() + + expected_results = [ + VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, + GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, + ] + project_families = [{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000001_1', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids']]}] + additional_response = {'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}} + response_json = self._assert_expected_search( + expected_results, project_families=project_families, additional_response=additional_response, + check_login=self.check_collaborator_login, export_data=EXPORT_DATA, + ) + + expected_gene = {k: mock.ANY for k in GENE_VARIANT_FIELDS} + expected_gene['locusListGuids'] = [] + expected_pa_gene = {**expected_gene, 'locusListGuids': ['LL00049_pid_genes_autosomal_do'], 'panelAppDetail': mock.ANY} + self.assertDictEqual(response_json['genesById'], { + **{gene_id: expected_gene for gene_id in [ + 'ENSG00000097046', 'ENSG00000277258', 'ENSG00000177000', 'ENSG00000275023', 'ENSG00000268903', + ]}, + 'ENSG00000227232': expected_pa_gene, + }) + self.assertDictEqual( + response_json['genesById']['ENSG00000227232']['panelAppDetail'], + {'LL00049_pid_genes_autosomal_do': {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}}, + ) + self.assertDictEqual(response_json['locusListsByGuid'], {'LL00049_pid_genes_autosomal_do': {'intervals': [mock.ANY, mock.ANY]}}) + self.assertSetEqual( + set(response_json['locusListsByGuid']['LL00049_pid_genes_autosomal_do']['intervals'][0].keys()), + {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} + ) + self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}) + self.assertDictEqual(response_json['omimIntervals'], {}) + self.assertDictEqual(response_json['phenotypeGeneScores'],{ + 'I000001_na19675': {'ENSG00000268903': {'exomiser': [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, 'scores': { + 'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, 'variant_score': 0.999200702, + }}, + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, + 'I000002_na19678': {'ENSG00000268903': {'lirical': [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, + 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, + ]}}, + 'I000004_hg00731': {'ENSG00000177000': {'exomiser': [ + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 5, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, + 'I000005_hg00732': {'ENSG00000177000': {'exomiser': [ + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 3, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, + }) + self.assertDictEqual(response_json['rnaSeqData'], { + 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, + }) + self.assertDictEqual( + response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], + {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, + 'tissueType': 'M', 'zScore': 7.08} + ) + self.assertListEqual( + sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), + [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, + 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, + {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, + 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, + {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, + 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] + ) + self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { + **{k: mock.ANY for k in [ + *SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId', 'screenRegionType', + 'sortedMotifFeatureConsequences', 'sortedRegulatoryFeatureConsequences', + ]}, + 'mmeSubmissions': [{'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}], + }}) + self.assertDictEqual(response_json['totalSampleCounts'], { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + }) + expected_functional_tag = {k: mock.ANY for k in FUNCTIONAL_FIELDS} + self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {guid: expected_functional_tag for guid in [ + 'VFD0000023_1248367227_r0390_10', 'VFD0000024_1248367227_r0390_10', 'VFD0000025_1248367227_r0390_10', 'VFD0000026_1248367227_r0390_10', + ]}) + self.assertDictEqual(response_json['variantNotesByGuid'], {}) + expected_tag = {k: mock.ANY for k in TAG_FIELDS} + self.assertDictEqual(response_json['variantTagsByGuid'], { + 'VT1708633_2103343353_r0390_100': expected_tag, 'VT1726961_2103343353_r0390_100': expected_tag, }) - self.assertDictEqual(response.json(), expected_response) - body['search']['locus']['rawItems'] = 'ENSG00000229905' - response = self.client.post(url+'8', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': project_families, - 'totalResults': 0, + # test project context info + self._assert_expected_search( + expected_results, project_families=project_families, query_params={'loadProjectTagTypes': 'true'}, additional_response={ + **additional_response, + 'projectsByGuid': {'R0001_1kg': {field: mock.ANY for field in { + 'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes', + }}}, + }, + ) + + # test family context info + expected_family = {field: mock.ANY for field in [*FAMILY_FIELDS, 'individualGuids']} + expected_individual = {field: mock.ANY for field in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} + expected_igv = {field: mock.ANY for field in IGV_SAMPLE_FIELDS} + expected_note = {field: mock.ANY for field in FAMILY_NOTE_FIELDS} + self._assert_expected_search( + expected_results, project_families=project_families, query_params={'loadFamilyContext': 'true'}, + additional_response={ + **additional_response, + 'familiesByGuid': { + 'F000001_1': {**expected_family, 'tpmGenes': ['ENSG00000227232']}, + 'F000002_2': expected_family, + 'F000003_3': expected_family, + }, + 'individualsByGuid': {guid: expected_individual for guid in [ + 'I000001_na19675', 'I000002_na19678', 'I000003_na19679', 'I000004_hg00731', 'I000005_hg00732', + 'I000006_hg00733', 'I000007_na20870', + ]}, + 'igvSamplesByGuid': {guid: expected_igv for guid in ['S000145_na19675', 'S000146_na20870', 'S000147_na20870']}, + 'familyNotesByGuid': {guid: expected_note for guid in ['FAN000001_1', 'FAN000001_2', 'FAN000001_3']}, }, - 'searchedVariants': [], + ) + + # Test OMIM intervals + self.login_manager() + response_json = self._assert_expected_search( + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, + project_families=SV_PROJECT_FAMILIES, + ) + self.assertDictEqual(response_json['omimIntervals'], { + '3': {'chrom': '1', 'end': 249055991, 'mimNumber': 600315, 'phenotypeDescription': '?Immunodeficiency 16', 'phenotypeInheritance': 'Autosomal recessive', 'phenotypeMimNumber': 615120, 'start': 249044482}, }) + # Test cross-project discovery for analyst users + self.set_cache('search_results__VRS0009876__xpos',[{ + 'key': 100, 'familyGuids': ['F000002_2'], 'xpos': 1248367227, 'genomeVersion': '38', 'sortedTranscriptConsequences': [], + }]) + response, _, _ = self._execute_search(search_hash=9876) + self.assertEqual(response.status_code, 200) + variants = response.json()['searchedVariants'] + self.assertEqual(len(variants), 1) + self.assertFalse('discoveryTags' in variants[0]) + + self.login_analyst_user() + response, _, _ = self._execute_search(search_hash=9876) + self.assertEqual(response.status_code, 200) + variants = response.json()['searchedVariants'] + self.assertEqual(len(variants), 1) + self.assertListEqual(variants[0]['discoveryTags'], [{ + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0003_tes', + 'familyGuid': 'F000012_12', + 'projectGuid': 'R0003_test', + }, + 'tagGuid': 'VT1726961_2103343353_r0003_tes', + 'name': 'Tier 1 - Novel gene and phenotype', + 'category': 'CMG Discovery Tags', + 'color': '#03441E', + 'searchHash': None, + 'metadata': None, + 'lastModifiedDate': '2018-05-29T16:32:51.449Z', + 'createdBy': None, + }]) + self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) + + def test_cached_query_variants(self): + search_hash = 987 + cache_key_prefix = 'search_results__VRS0000987' + cached_variants = [VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1, VARIANT2] + cache_result = self._format_cached_variants(cached_variants) + self.set_cache(f'{cache_key_prefix}__xpos', cache_result) + + self._assert_expected_search( + cached_variants, search_hash=search_hash, skip_cache_check=True, check_login=self.check_collaborator_login, + ) + + self._assert_expected_search( + cached_variants, search_hash=search_hash, skip_cache_check=True, + results_page=[GCNV_VARIANT1, MITO_VARIANT1], query_params={'page': 2, 'per_page': 2}, gene_counts={ + 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, + }, + ) + + self._assert_expected_search( + [VARIANT2, VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1], search_hash=search_hash, sort='cadd', + ) + class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): databases = '__all__' fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] @@ -1735,8 +2420,9 @@ def test_trigger_delete_project(self): self.check_data_manager_login(url) Project.objects.filter(guid='R0001_1kg').update(genome_version='38') + body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} response = self.client.post( - url, content_type='application/json', data=json.dumps({'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'}) + url, content_type='application/json', data=json.dumps(body) ) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { @@ -1758,10 +2444,23 @@ def test_trigger_delete_project(self): 5: {'ac_wes': 1, 'ac_wgs': 1, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 2, 'hom_affected': 0}, 6: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, 22: {'ac_wes': 0, 'ac_wgs': 3, 'hom_wes': 0, 'hom_wgs': 1, 'ac_affected': 3, 'hom_affected': 1}, + 23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, }) project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True) self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0) self.assertEqual(project_samples.count(), 4) + body['datasetType'] = 'SV' + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': [ + 'Deactivated search for 3 individuals', + 'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de', + ], + }) + self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0) + self.assertEqual(project_samples.count(), 1) + diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 4364279d52..91f9ced017 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -383,6 +383,64 @@ 'CAID': 'CA520798130', } +FAMILY_1_VARIANT = { + 'key': 23, + 'variantId': '21-3343353-GAGA-G', + 'chrom': '21', + 'pos': 3343353, + 'ref': 'GAGA', + 'alt': 'G', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'xpos': 21003343353, + 'rsid': None, + 'familyGuids': ['F000001_1'], + 'genotypes': {'I000002_na19678': { + 'ab': 0.16, 'dp': 45, 'familyGuid': 'F000001_1', 'filters': [], 'gq': 0, 'individualGuid': 'I000002_na19678', + 'numAlt': 1, 'sampleId': 'NA19678', 'sampleType': 'WES', + }}, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 1, 'hom': 0, 'ac_wes': 1, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0}, + 'seqr_affected': {'ac': 0, 'hom': 0}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + }, + 'predictions': { + 'cadd': None, + 'eigen': None, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + 'absplice': None, + 'pext': None, + 'promoter_ai': None, + }, + 'transcripts': { + 'ENSG00000227232': [{'alphamissense': {'pathogenicity': None}, 'aminoAcids': 'T/I','biotype': 'protein_coding', 'canonical': 1, 'codons': 'aCc/aTc', 'consequenceTerms': ['missense_variant'], 'exon': {'index': 1, 'total': 12}, 'geneId': 'ENSG00000227232', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'intron': None, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'majorConsequence': 'missense_variant', 'manePlusClinical': None, 'maneSelect': None, 'refseqTranscriptId': 'NM_001281501.1', 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'transcriptId': 'ENST00000257261', 'transcriptRank': 0, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}}], + 'ENSG00000268903': [{'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000268903', 'hgvsc': 'ENST00000426137.1:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000426137', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 5}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}], + }, + 'mainTranscriptId': 'ENST00000257261', + 'selectedMainTranscriptId': None, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'CAID': None, +} + GRCH37_VARIANT = { 'key': 11, 'variantId': '7-143270172-A-G', @@ -1074,56 +1132,6 @@ }, } -LOOKUP_GENOTYPE = {k: v for k, v in PROJECT_2_VARIANT1['genotypes']['I000015_na20885'].items() if k != 'individualGuid'} -VARIANT_LOOKUP_VARIANT = { - **VARIANT1, - 'liftedFamilyGuids': ['F000014_14'], - 'familyGenotypes': { - VARIANT1['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for gs in VARIANT1_BOTH_SAMPLE_TYPES['genotypes'].values() for g in gs - ], key=lambda x: (x['sampleType'] == 'WES', x['sampleId']), reverse=True), - 'F000011_11': [{**LOOKUP_GENOTYPE, 'sampleType': 'WES'}, LOOKUP_GENOTYPE], - 'F000014_14': [{ - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', - 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531, 'filters': [], - }], - } -} -SV_LOOKUP_VARIANT = { - **SV_VARIANT4, - 'familyGenotypes': { - SV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in SV_VARIANT4['genotypes'].values()], - }, -} -GCNV_LOOKUP_VARIANT = { - **GCNV_VARIANT4, - 'familyGenotypes': { - GCNV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in GCNV_VARIANT4['genotypes'].values()], - 'F000002_2_x': [ - {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} - for individual_guid, g in GCNV_VARIANT4['genotypes'].items() if individual_guid != 'I000005_hg00732' - ], - }, -} -GCNV_LOOKUP_VARIANT_3 = { - **MULTI_PROJECT_GCNV_VARIANT3, - 'familyGenotypes': { - **{family_guid: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MULTI_PROJECT_GCNV_VARIANT3['genotypes'].values() - if g['familyGuid'] == family_guid] for family_guid in MULTI_PROJECT_GCNV_VARIANT3['familyGuids'] - }, - 'F000002_2_x': [ - {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} - for individual_guid, g in GCNV_VARIANT3['genotypes'].items() if individual_guid != 'I000005_hg00732' - ], - }, -} -for k in {'familyGuids', 'genotypes'}: - VARIANT_LOOKUP_VARIANT.pop(k) - SV_LOOKUP_VARIANT.pop(k) - GCNV_LOOKUP_VARIANT.pop(k) - GCNV_LOOKUP_VARIANT_3.pop(k) - PROJECT_4_COMP_HET_VARIANT = { 'key': 22, 'variantId': '1-9310123-T-C', @@ -1331,6 +1339,23 @@ 'geneId': 'ENSG00000171621', 'isManeSelect': False, }], +23: [{ + 'alphamissensePathogenicity': None, + 'canonical': 1, + 'consequenceTerms': ['missense_variant'], + 'extendedIntronicSpliceRegionVariant': False, + 'fiveutrConsequence': None, + 'geneId': 'ENSG00000227232', + 'isManeSelect': False, +}, { + 'alphamissensePathogenicity': None, + 'canonical': 1, + 'consequenceTerms': ['intron_variant'], + 'extendedIntronicSpliceRegionVariant': False, + 'fiveutrConsequence': None, + 'geneId': 'ENSG00000268903', + 'isManeSelect': False, +}], } def format_cached_variant(variant): @@ -1393,3 +1418,76 @@ def format_cached_variant(variant): 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, } + +EXPORT_DATA = [ + ['chrom', 'pos', 'ref', 'alt', 'gene', 'gene_symbol', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', + 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', + 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', + 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', + 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3', + 'sample_4', 'num_alt_alleles_4', 'filters_4', 'gq_4', 'ab_4'], + ['1', '10439', 'AC', 'A', '', '', '', '8', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', + 'rs112766696', '', '', 'Likely_pathogenic', '', '2', '', '', '', '', '', 'HG00731', '1', '', '99', '0.5', 'HG00732', + '0', '', '0', '0', 'HG00733', '0', '', '20', '0', '', '', '', '', ''], + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'MTHFR', 'missense_variant', '10', '0.00344493', '0.00288997', '0.246152', + '25.9', '0.197', '2.001', '0', '0.1', '0.05', 'polymorphism', '0', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', + 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1', 'HG00732', + '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0', '', '', '', '', ''], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '3', '', + '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741', + 'NA20870', '1', '', '99', '0.67857'], + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'CDC7', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', + '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', + 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', + '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', '', '', '', '', ''], + ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', + '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], + ['16', '29809156', '', '', 'ENSG00000103495', '', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '1', '', '', + '', '', '', '', '', ''], + ['17', '38717327', '', '', 'ENSG00000275023', 'CDC9', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '2', '', '', '', '', '', 'HG00731', '2', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '0', '', '', '', + '', '', '', '', ''], + ['17', '38721781', '', '', 'ENSG00000275023', 'CDC9', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '1', '', '', '', + '', '', '', '', ''], + ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'WASH7P', 'missense_variant', '1', '0', '0', '0', '', '', '', '', '', '', + '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', + 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19678', '1', '', '0', '0.16', '', '', + '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '4429', 'G', 'A', 'ENSG00000210112', '', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', + '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', + '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '11597', 'A', 'G', 'ENSG00000198886', '', 'missense_variant', '0', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', + '', 'ENST00000361381.2:c.838A>G', 'ENSP00000354961.2:p.Thr280Ala', '', '', '2', '', '', '', '', '', 'HG00731', '1', + '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '14783', 'T', 'C', 'ENSG00000198727', '', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', + 'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '', + '', '', 'HG00731', '2', 'artifact_prone_site'], +] + +SPLIT_FAMILY_EXPORT_DATA = [ + ['chrom', 'pos', 'ref', 'alt', 'gene', 'gene_symbol', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', + 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', + 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', + 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', + 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', + '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', '', '', '', '', ''], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '3', '', '', 'HG00733', + '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857', '', '', '', '', ''], + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'CDC7', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', + '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', 'HG00731', '1', + 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', + '0', 'VQSRTrancheSNP99.95to100.00', '30', '0'], +] + +DEFAULT_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2', 'F000003_3', 'F000005_5']}] +SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] +MULTI_PROJECT_PROJECT_FAMILIES = [ + *SINGLE_FAMILY_PROJECT_FAMILIES, {'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']} +] +SV_PROJECT_FAMILIES = [{'projectGuid': 'R0004_non_analyst_project', 'familyGuids': ['F000014_14']}] diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 40056b7e64..a1791115fd 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -11,7 +11,7 @@ get_submission_json_for_external_match, parse_mme_features, get_submission_gene_variants, get_mme_matches, \ get_gene_ids_for_feature, validate_patient_data, get_hpo_terms_by_id, MME_DISCLAIMER from seqr.models import Individual, SavedVariant -from seqr.utils.communication_utils import safe_post_to_slack +from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_chrom_pos @@ -24,7 +24,7 @@ has_project_permissions, login_and_policies_required, get_project_and_check_permissions from settings import BASE_URL, MME_ACCEPT_HEADER, MME_NODES, MME_DEFAULT_CONTACT_EMAIL, \ - MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL + MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL logger = SeqrLogger(__name__) @@ -559,3 +559,25 @@ def _generate_notification_for_seqr_match(submission, results): from_email=MME_DEFAULT_CONTACT_EMAIL, ) email_message.send() + + +@login_and_policies_required +def send_vlm_email(request): + request_json = json.loads(request.body) + email_message = EmailMessage( + subject=request_json['subject'], + body=request_json['body'], + bcc=[s.strip() for s in request_json['to'].split(',')], + cc=[request.user.email], + reply_to=[request.user.email], + to=[VLM_SEND_EMAIL], + from_email=VLM_SEND_EMAIL, + ) + set_email_message_stream(email_message, 'vlm') + + try: + email_message.send() + except Exception as e: + logger.error(f'VLM Email Error: {e}', request.user, detail=request_json) + + return create_json_response({'success': True}) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index 845e85411e..0bbe9e53d7 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -10,7 +10,7 @@ from matchmaker.matchmaker_utils import MME_DISCLAIMER from matchmaker.views.matchmaker_api import get_individual_mme_matches, search_individual_mme_matches, \ update_mme_submission, delete_mme_submission, update_mme_result_status, send_mme_contact_email, \ - get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, \ + get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, send_vlm_email, \ update_mme_contact_note, update_mme_project_contact from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_FIELDS @@ -871,3 +871,48 @@ def test_update_mme_project_contact(self): {'name': 'Baylor UDN Clinical Site', 'email': 'UDNCC@hms.harvard.edu'}, {'name': '', 'email': 'matchmaker@phenomecentral.org'}, ]) + + + @mock.patch('matchmaker.views.matchmaker_api.EmailMessage') + def test_send_vlm_email(self, mock_email): + url = reverse(send_vlm_email) + self.check_require_login(url) + + self.reset_logs() + body = { + 'to': 'test@test.com , other_test@gmail.com', + 'body': 'some email content', + 'subject': 'some email subject' + } + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self._assert_expected_vlm_email(response, mock_email) + + self.reset_logs() + mock_email.return_value.send.side_effect = Exception('Send failed') + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self._assert_expected_vlm_email(response, mock_email, additional_logs=[ + ('VLM Email Error: Send failed', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + 'detail': body, + }), + ]) + + def _assert_expected_vlm_email(self, response, mock_email, additional_logs=None): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'success': True}) + + mock_email.assert_called_with( + subject='some email subject', + body='some email content', + bcc=['test@test.com', 'other_test@gmail.com'], + cc=['test_user_no_access@test.com'], + reply_to=['test_user_no_access@test.com'], + to=['vlm-noreply@broadinstitute.org'], + from_email='vlm-noreply@broadinstitute.org') + self.assertDictEqual(mock_email.return_value.esp_extra, {'MessageStream': 'vlm'}) + mock_email.return_value.send.assert_called() + + self.assert_json_logs(self.no_access_user, (additional_logs or []) + [ + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}) + ]) diff --git a/requirements.in b/requirements.in index b416c0bd3b..0039bcce8c 100644 --- a/requirements.in +++ b/requirements.in @@ -9,8 +9,6 @@ django-cors-headers # allows CORS requests for client-side develop django-storages[google] # alternative GCS storage backend for the django media_root social-auth-app-django>5.0.0 # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django -elasticsearch==7.9.1 # elasticsearch client -elasticsearch-dsl==7.2.1 # elasticsearch query utilities gunicorn # web server jmespath openpyxl # library for reading/writing Excel files diff --git a/requirements.txt b/requirements.txt index 53b74e6f0f..f9e7afb329 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ cachetools==5.3.0 # via google-auth certifi==2024.7.4 # via - # elasticsearch # requests cffi==1.15.1 # via cryptography @@ -60,12 +59,6 @@ django-notifications-hq==1.8.3 # via -r requirements.in django-storages[google]==1.11.1 # via -r requirements.in -elasticsearch==7.9.1 - # via - # -r requirements.in - # elasticsearch-dsl -elasticsearch-dsl==7.2.1 - # via -r requirements.in et-xmlfile==1.1.0 # via openpyxl feedparser==6.0.10 @@ -123,12 +116,10 @@ pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 # via cffi -pyjwt==2.8.0 +pyjwt==2.12.0 # via social-auth-core pyliftover==0.4 # via -r requirements.in -python-dateutil==2.8.2 - # via elasticsearch-dsl python3-openid==3.2.0 # via social-auth-core pytz==2022.7.1 @@ -157,10 +148,8 @@ sgmllib3k==1.0.0 # via feedparser six==1.16.0 # via - # elasticsearch-dsl # google-auth # markdownify - # python-dateutil slacker==0.14.0 # via -r requirements.in slugify==0.0.1 @@ -187,7 +176,6 @@ tzlocal==5.3.1 # via clickhouse-driver urllib3==2.6.3 # via - # elasticsearch # requests whitenoise==6.3.0 # via -r requirements.in diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 8eab981a31..9957c85223 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1869,7 +1869,10 @@ "ref": "GAGA", "alt": "G", "variant_id": "21-3343353-GAGA-G", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, + "genotypes": {"I000003_na19679": {"sampleId": "NA19679", "ab": 0.0, "ad": "45,0", "gq": 99.0, "dp": "45", "pl": "0,135,1525", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000002_na19678": {"sampleId": "NA19678", "ab": 0.0, "ad": "42,0", "gq": 99.0, "dp": "43", "pl": "0,126,1479", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000001_na19675": {"sampleId": "NA19675_1", "ab": 0.7021276595744681, "ad": "14,33", "gq": 46.0, "dp": "50", "pl": "46,0,686", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 1}}, "saved_variant_json": { "variantId": "21-3343353-GAGA-G", "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, @@ -1922,100 +1925,8 @@ } }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": { - "clinvar": { - "pathogenicity": "Uncertain_significance", - "alleleId": 12345, - "assertions": null, - "conditions": null, - "conflictingPathogenicities": null, - "submitters": null, - "goldStars": null - }, - "liftedOverGenomeVersion": "38", - "familyGuids": [ - "F000001_1" - ], - "liftedOverPos": "", - "populations": { - "callset": { - "ac": null, - "an": null, - "af": null - }, - "g1k": { - "ac": null, - "an": null, - "af": 0.0 - }, - "gnomad_genomes": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 0.00012925741614425127 - }, - "gnomad_exomes": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 6.505916317651364e-05 - }, - "exac": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 0.0006726888333653661 - }, - "topmed": { - "ac": null, - "an": null, - "af": null - } - }, - "genomeVersion": "37", - "pos": 248367227, - "predictions": { - "eigen": null, - "revel": null, - "sift": null, - "cadd": "27.2", - "metasvm": "", - "mpc": null, - "splice_ai": null, - "phastcons_100_vert": null, - "mut_taster": null, - "fathmm": null, - "polyphen": null, - "dann": null, - "primate_ai": null, - "gerp_rs": null - }, - "hgmd": { - "accession": null, - "class": null - }, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000135953"} - ], - "ENSG00000240361": [] - }, - "chrom": "1", - "genotypes": { - "I000004_hg00731": { - "numAlt": 2 - }, - "I000005_hg00732": { - "numAlt": 1 - } - }, - "CAID": "CA1501729" - }, + "saved_variant_json": {}, + "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": null, "consequenceTerms": [], "geneId": "ENSG00000135953", "hgvsc": null, "hgvsp": null, "loftee": [null, []], "majorConsequence": null, "transcriptId": "ENST00000371839", "transcriptRank": 1}, "selected_main_transcript_id": "ENST00000371839", "family": 2 } @@ -2033,7 +1944,14 @@ "ref": "G", "alt": "A", "variant_id": "1-46859832-G-A", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000197530"], + "main_transcript": {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "CELSR1", "majorConsequence": "splice_region_variant", "canonical": "YES", "hgvs": "ENST00000505820.2:c.1067-4G>A", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000505820.2:c.1067-4G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"}, + "genotypes": { + "I000003_na19679": {"sampleId": "NA19679", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 0}, + "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 6, "dp": 2, "pl": null, "numAlt": 0}, + "I000002_na19675": {"sampleId": "NA19675", "ab": 0.125, "ad": null, "gq": 7, "dp": 8, "pl": null, "numAlt": 1} + }, "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -2074,7 +1992,14 @@ "ref": "G", "alt": "CA", "variant_id": "1-1562437-G-CA", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000197530"], + "main_transcript": {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"}, + "genotypes": { + "I000003_na19679": {"sampleId": "NA19679", "ab": 0.71428573, "ad": null, "gq": 58, "dp": 7, "pl": null, "numAlt": 1}, + "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 30, "dp": 10, "pl": null, "numAlt": 0}, + "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1} + }, "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -2166,44 +2091,8 @@ "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": { - "clinvar": { - "pathogenicity": "Uncertain_significance", - "alleleId": 12345, - "assertions": null, - "conditions": null, - "conflictingPathogenicities": null, - "submitters": null, - "goldStars": null - }, - "liftedOverGenomeVersion": "38", "liftedOverPos": "", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, - "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.00012925741614425127}, - "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 6.505916317651364e-05}, - "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0006726888333653661}, - "topmed": {"ac": null, "an": null, "af": null}}, - "genomeVersion": "37", "pos": 248367227, "predictions": { - "eigen": null, "revel": null, "sift": null, "cadd": "27.2", "metasvm": "", "mpc": null, - "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, - "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", - "mainTranscriptId": "ENST00000505820", "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000228198"} - ], - "ENSG00000240361": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", - "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, - "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", - "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000240361", - "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, "chrom": "1", "genotypes": { - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, - "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} - }, - "CAID": "CA1501729" - }, + "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": "Gtg/Atg", "consequenceTerms": ["intron_variant"], "geneId": "ENSG00000240361", "hgvsc": "ENST00000262738.3:c.3955G>A", "hgvsp": "ENST00000505820.2:c.1586-17C>G", "loftee": [null, []], "majorConsequence": "intron_variant", "transcriptId": "ENST00000505820", "transcriptRank": 0}, + "saved_variant_json": {}, "family": 12 } }, @@ -2219,8 +2108,13 @@ "xpos_end": 1249045898, "ref": null, "alt": null, - "variant_id": "prefix_19107_DEL", + "variant_id": "DEL:chr1:249045487-249045898", + "dataset_type": "SV", "gene_ids": ["ENSG00000135953", "ENSG00000223972", "ENSG00000240361"], + "sv_type": "DEL", + "genotypes": { + "I000017_na20889": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} + }, "saved_variant_json": { "liftedOverGenomeVersion": null, "pos": 249045487, @@ -2264,13 +2158,7 @@ "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", "liftedOverPos": "", "genomeVersion": "38", "pos": 248367227, - "transcripts": {}, "chrom": "1", "genotypes": { - "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} - }, - "CAID": "CA1501729" - }, + "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 0d64249d1c..3571f6b9c2 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -12,7 +12,11 @@ "ref": "C", "alt": "T", "variant_id": "19-1912634-C-T", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000240361"], + "genotypes": { + "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} + }, "saved_variant_json": { "liftedOverGenomeVersion": "37", "pos": 1912634, @@ -58,7 +62,12 @@ "ref": "G", "alt": "T", "variant_id": "19-1912633-G-T", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361"}, + "genotypes": { + "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} + }, "saved_variant_json": { "liftedOverGenomeVersion": "37", "pos": 1912633, @@ -107,7 +116,12 @@ "ref": "G", "alt": "C", "variant_id": "19-1912632-G-C", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361", "hgvsc": "c.586_587delinsTT", "hgvsp": "p.Ala196Leu"}, + "genotypes": { + "I000004_hg00731": {"numAlt": 1} + }, "saved_variant_json": { "pos": 1912632, "end": 1912632, @@ -135,7 +149,7 @@ "pk": 12, "fields": { "guid": "SV0000009_25000014783_r0004_no", - "created_date": "2018-05-15T14:51:56.943Z", + "created_date": "2018-05-16T14:51:56.943Z", "created_by": null, "last_modified_date": "2018-05-31T16:36:02.805Z", "xpos": 25000014783, @@ -149,19 +163,8 @@ "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} }, "gene_ids": ["ENSG00000198727"], - "saved_variant_json": { - "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, - "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, - "rsid": "rs193302982", "familyGuids": ["F000002_2"], "genotypes": { - "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} - }, "populations": {"seqr": {"af": 0.019480518996715546, "ac": 3, "an": 154}, "seqr_heteroplasmy": {"af": 0.006493506487458944, "ac": 1, "an": 154}, - "gnomad_mito": {"af": 0.05534649267792702, "ac": 3118, "an": 56336}, "gnomad_mito_heteroplasmy": {"af": 5.3251918870955706e-05, "ac": 3, "an": 56336, "max_hl": 1.0}, - "helix": {"af": 0.04884607344865799, "ac": 9573, "an": 195983}, "helix_heteroplasmy": {"af": 9.184470400214195e-05, "ac": 18, "an": 195983, "max_hl": 0.962689995765686}}, - "predictions": {"apogee": null, "haplogroup_defining": "Y", "hmtvar": null, "mitotip": null, "mut_taster": null, "sift": null, "mlc": 0.7514}, - "commonLowHeteroplasmy": true, "mitomapPathogenic": true, "clinvar": null, "transcripts": { - "ENSG00000198727": [{"aminoAcids": "L", "canonical": 1, "codons": "Tta/Cta", "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "transcriptId": "ENST00000361789", "isLofNagnag": null, "transcriptRank": 0, "biotype": "protein_coding", "lofFilters": null, "majorConsequence": "synonymous_variant", "consequenceTerms": ["synonymous_variant"]}]}, - "mainTranscriptId": "ENST00000361789", "selectedMainTranscriptId": null - }, + "main_transcript": {"aminoAcids": "L", "biotype": "protein_coding", "canonical": 1, "codons": "Tta/Cta", "consequenceTerms": ["synonymous_variant"], "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "loftee": [null, []], "majorConsequence": "synonymous_variant", "transcriptId": "ENST00000361789", "transcriptRank": 0}, + "saved_variant_json": {}, "family": 14 } }, { @@ -179,33 +182,12 @@ "variant_id": "suffix_19107_DEL_013746", "key": 101, "dataset_type": "SV_WGS", + "sv_type": "DEL", "genotypes": { "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], - "saved_variant_json": { - "liftedOverGenomeVersion": null, - "pos": 249045487, - "end": 249045898, - "xpos": 1249045487, - "predictions": {"strvctvre": 0.374}, - "alt": null, - "numExon": 2, - "genotypeFilters": [], - "ref": null, - "genotypes": { - "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} - }, - "liftedOverPos": null, - "liftedOverChrom": null, - "svType": "DEL", - "variantId": "suffix_19107_DEL", - "chrom": "1", - "endChrom": "1", - "genomeVersion": "37", - "populations": {"sv_callset": {}}, - "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} - }, + "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 9386315acd..3ce9618d97 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -3,22 +3,21 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone import json import logging import re from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, SavedVariant, Project, Individual +from seqr.models import Family, Sample, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS -from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil -from seqr.views.utils.variant_utils import reset_cached_search_results, update_projects_saved_variant_json, \ - get_saved_variants +from seqr.views.utils.variant_utils import reset_cached_search_results from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \ SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL @@ -215,7 +214,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_project_tuples = [] invalid_genome_version_projects = [] for project, sample_ids in samples_by_project.items(): - sample_project_tuples += [(sample_id, project.name) for sample_id in sample_ids] + sample_project_tuples += [(sample_id, project.id) for sample_id in sample_ids] project_genome_version = GENOME_VERSION_LOOKUP.get(project.genome_version, project.genome_version) if project_genome_version != genome_version: invalid_genome_version_projects.append((project.guid, project_genome_version)) @@ -228,13 +227,8 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples, updated_samples, *args = match_and_update_search_samples( - projects=samples_by_project.keys(), - sample_project_tuples=sample_project_tuples, - sample_data={'data_source': run_version, 'elasticsearch_index': ';'.join(metadata['callsets'])}, - sample_type=sample_type, - dataset_type=dataset_type, - user=None, + new_samples = cls._match_and_update_search_samples( + sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), ) new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( @@ -257,10 +251,10 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers except Exception as e: logger.error(f'Error updating individuals sample qc {run_version}: {e}') - # Reload saved variant JSON - update_projects_saved_variant_json([ - (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) + logger.info(f'Reloading saved variants in {len(families_by_project)} projects') + for project, family_guids in families_by_project.items(): + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, clickhouse_dataset_type) + logger.info(f'Updated {len(updated_saved_variants)} variants in {len(family_guids)} families for project {project.name}') @classmethod def _is_internal_project(cls, project): @@ -394,28 +388,120 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) fields=['filter_flags', 'pop_platform_filters', 'population'], ) - @classmethod - def _update_project_saved_variant_genotypes(cls, project_id, genome_version, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): + @staticmethod + def _update_project_saved_variant_genotypes(project, family_guids, dataset_type): updates = {} for family_guid in family_guids: variant_models_by_key = { - v.key: v for v in get_saved_variants(genome_version, project_id, [family_guid], clickhouse_dataset_type=clickhouse_dataset_type) + v.key: v for v in + SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid, key__isnull=False) } if not variant_models_by_key: continue variants = [] genotypes_by_key = get_clickhouse_genotypes( - project_guid, [family_guid], genome_version, clickhouse_dataset_type, variant_models_by_key.keys(), samples, + project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), ) for key, genotypes in genotypes_by_key.items(): variant = variant_models_by_key[key] - variant.genotypes = genotypes + variant.genotypes = genotypes['genotypes'] variants.append(variant) - logger.info(f'Reloading genotypes for {len(variants)} {clickhouse_dataset_type} variants in family {family_guid}') + logger.info(f'Reloading genotypes for {len(variants)} {dataset_type} variants in family {family_guid}') SavedVariant.bulk_update_models(None, variants, ['genotypes']) updates.update({v.id: v for v in variants}) return updates + @classmethod + def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): + individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples) + individual_ids = individual_ids_by_keys.values() + matched_samples = Sample.objects.filter(is_active=True, dataset_type=dataset_type, sample_type=sample_type) + cls._update_matched_families(individual_ids, matched_samples) + + sample_guids_by_individual = dict(Sample.objects.filter( + individual_id__in=individual_ids, sample_type=sample_type, dataset_type=dataset_type, **sample_data, + ).values_list('individual_id', 'guid')) + remaining_key_individuals = { + key: individual_id for key, individual_id in individual_ids_by_keys.items() + if individual_id not in sample_guids_by_individual + } + loaded_date = timezone.now() + created_sample_guids = cls._create_samples( + remaining_key_individuals, loaded_date=loaded_date, sample_type=sample_type, dataset_type=dataset_type, + **sample_data, + ) if remaining_key_individuals else [] + + sample_guids = list(sample_guids_by_individual.values()) + created_sample_guids + + activated_sample_guids = Sample.bulk_update(user=None, update_json={ + 'is_active': True, + 'loaded_date': loaded_date, + **sample_data, + }, guid__in=sample_guids, is_active=False) + + inactivate_samples = matched_samples.filter(individual_id__in=individual_ids).exclude(guid__in=sample_guids) + inactivated_sample_guids = Sample.bulk_update( + user=None, update_json={'is_active': False}, queryset=inactivate_samples, + ) + + previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) + return Sample.objects.filter(guid__in=activated_sample_guids).exclude(individual_id__in=previous_loaded_individuals) + + @staticmethod + def _get_matched_individuals(sample_project_tuples): + individual_ids_by_keys = { + (individual_id, project_id): individual_db_id + for individual_db_id, individual_id, project_id in Individual.objects.filter( + family__project_id__in={project_id for _, project_id in sample_project_tuples}, + individual_id__in={sample_id for sample_id, _ in sample_project_tuples}, + ).values_list('id', 'individual_id', 'family__project_id') + if (individual_id, project_id) in sample_project_tuples + } + + missing_keys = set(sample_project_tuples) - set(individual_ids_by_keys.keys()) + if missing_keys: + sample_ids = ', '.join(sorted([sample_id for sample_id, _ in missing_keys])) + raise ValueError(f'Matches not found for sample ids: {sample_ids}') + + return individual_ids_by_keys + + @staticmethod + def _update_matched_families(individual_ids, matched_samples): + included_families = dict( + Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status') + ) + missing_individuals = matched_samples.filter( + individual__family_id__in=included_families, + ).exclude(individual_id__in=individual_ids).values( + 'individual__family__family_id', + ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) + if missing_individuals: + missing_summary = ', '.join(sorted([ + f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals + ])) + raise ValueError( + f'The following families are included in the callset but are missing some family members: {missing_summary}' + ) + + family_ids_to_update = [ + family_id for family_id, analysis_status in included_families.items() + if analysis_status in {Family.ANALYSIS_STATUS_WAITING_FOR_DATA, Family.ANALYSIS_STATUS_LOADING_FAILED} + ] + Family.bulk_update( + user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update, + ) + + @staticmethod + def _create_samples(remaining_key_individuals, **sample_params): + new_samples = [ + Sample( + individual_id=individual_id, + sample_id=sample_key[0], + **sample_params, + ) for sample_key, individual_id in remaining_key_individuals.items()] + new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) + return [s.guid for s in new_sample_models] + update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py deleted file mode 100644 index df48683bea..0000000000 --- a/seqr/management/commands/deactivate_project_search.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - -from seqr.models import Project, Sample -from seqr.utils.search.utils import es_only - -import logging -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument('project') - - @es_only - def handle(self, *args, **options): - project = Project.objects.get(guid=options['project']) - - if input(f'Are you sure you want to deactivate search for {project.name} (y/n)? ') != 'y': - raise CommandError('Error: user did not confirm') - - updated = Sample.bulk_update(user=None, update_json={'is_active': False}, individual__family__project=project) - - logger.info(f'Deactivated {len(updated)} samples') diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py index 8904d377bc..3cd146ac40 100644 --- a/seqr/management/commands/reload_saved_variant_genotypes.py +++ b/seqr/management/commands/reload_saved_variant_genotypes.py @@ -2,7 +2,6 @@ from django.core.management.base import BaseCommand from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes from seqr.models import Project, Family, Sample -from seqr.utils.search.utils import clickhouse_only logger = logging.getLogger(__name__) @@ -14,7 +13,6 @@ def add_arguments(self, parser): parser.add_argument('project') parser.add_argument('--family-guid', help='optional family to reload variants for') - @clickhouse_only def handle(self, *args, **options): project = Project.objects.get(guid=options['project']) family_guid = options['family_guid'] @@ -24,13 +22,10 @@ def handle(self, *args, **options): if family_guid: samples = samples.filter(individual__family__guid=family_guid) dataset_types = { - f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type: dataset_type + f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct() } - for clickhouse_dataset_type, dataset_type in sorted(dataset_types.items()): - update_project_saved_variant_genotypes( - project.id, project.genome_version, family_guids, project.guid, - samples=samples.filter(dataset_type=dataset_type), clickhouse_dataset_type=clickhouse_dataset_type, - ) + for dataset_type in sorted(dataset_types): + update_project_saved_variant_genotypes(project, family_guids, dataset_type) logger.info('Done') diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py deleted file mode 100644 index 7c9f14efc7..0000000000 --- a/seqr/management/commands/reload_saved_variant_json.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging -from django.core.management.base import BaseCommand -from django.db.models.query_utils import Q -from seqr.models import Project -from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import es_only -from seqr.views.utils.variant_utils import update_projects_saved_variant_json - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = 'Transfer projects to the new seqr schema' - - def add_arguments(self, parser): - parser.add_argument('projects', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.') - parser.add_argument('--family-guid', help='optional family to reload variants for') - - @es_only - def handle(self, *args, **options): - projects_to_process = options['projects'] - family_guid = options['family_guid'] - - if projects_to_process: - projects = Project.objects.filter(Q(name__in=projects_to_process) | Q(guid__in=projects_to_process)) - logging.info("Processing %s projects" % len(projects)) - else: - projects = Project.objects.all() - logging.info("Processing all %s projects" % len(projects)) - - family_ids = [family_guid] if family_guid else None - project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, update_function=update_project_saved_variant_json) - logger.info("Done") diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index af7e6e4969..8e4e63fc88 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -5,17 +5,15 @@ from django.db.models import Q from django.db.models.functions import JSONObject -from clickhouse_search.search import get_search_queryset, add_individual_guids, \ - get_data_type_comp_het_results_queryset, get_multi_data_type_comp_het_results_queryset +from clickhouse_search.search import get_clickhouse_variants from panelapp.models import PaLocusListGene from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, Sample, LocusList from seqr.utils.communication_utils import send_project_notification from seqr.utils.gene_utils import get_genes -from seqr.utils.search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO -from seqr.utils.search.utils import clickhouse_only, get_search_samples, COMPOUND_HET +from clickhouse_search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO, COMPOUND_HET from seqr.views.utils.orm_to_json_utils import SEQR_TAG_TYPE -from seqr.views.utils.variant_utils import bulk_create_tagged_variants +from seqr.views.utils.variant_utils import bulk_create_tagged_variants, get_saved_variant_annotations from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL import logging @@ -40,7 +38,6 @@ RECESSIVE_MOI = 'R' MITO_MOI = 'M' -MAX_AFFECTED_FAMILY_FILTER = 'max_affected' CONFIRMED_FAMILY_FILTER = 'confirmed_inheritance' AFFECTED_MALE_FAMILY_FILTER = 'affected_males' @@ -114,6 +111,10 @@ **SV_QUALITY_FILTER, 'vcf_filter': 'PASS', } +PERMISSIVE_PASS_QUALITY_FILTER = { + **PASS_QUALITY_FILTER, + 'min_gq_sv': 50, +} CONFIRMED_HIGH_SPLICE_AI_SEARCH = { 'family_filter': { @@ -217,7 +218,6 @@ }, 'Compound Heterozygous - Confirmed': { 'family_filter': { - MAX_AFFECTED_FAMILY_FILTER: 1, CONFIRMED_FAMILY_FILTER: True }, 'inheritance_mode': COMPOUND_HET, @@ -321,6 +321,24 @@ **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, + 'High Splice AI - X-Linked Recessive': { + 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + **HIGH_SPLICE_AI_SEARCH, + **RECESSIVE_SEARCH_NO_IN_SILICO, + 'family_filter': { + AFFECTED_MALE_FAMILY_FILTER: True, + CONFIRMED_FAMILY_FILTER: False, + }, + }, + 'High Splice AI - X-Linked Recessive Confirmed': { + 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + **CONFIRMED_HIGH_SPLICE_AI_SEARCH, + **RECESSIVE_SEARCH_NO_IN_SILICO, + 'family_filter': { + AFFECTED_MALE_FAMILY_FILTER: True, + CONFIRMED_FAMILY_FILTER: True, + }, + }, 'Recessive': { 'inheritance_mode': HOMOZYGOUS_RECESSIVE, 'annotations': HIGH_MODERATE_ANNOTATIONS, @@ -392,7 +410,7 @@ }, 'qualityFilter': { 'min_hl': 5, - 'min_mitoCn': 250, + 'min_mitoCn': 150, }, }, }, @@ -400,25 +418,36 @@ MULTI_DATA_TYPE_SEARCHES = { 'Compound Heterozygous - One SV': { + 'annotations': HIGH_ANNOTATIONS, + 'annotations_secondary': SV_ANNOTATIONS, + 'in_silico': IN_SILICO_FILTER, + 'freqs': FREQ_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, + }, + 'Compound Heterozygous - Clinvar Pathogenic/ SV': { + 'annotations': {}, + 'annotations_secondary': SV_ANNOTATIONS, + 'pathogenicity': CLINVAR_FILTER, + 'freqs': FREQ_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, + }, + 'Compound Heterozygous - High Splice AI/ SV': { 'annotations': { - **SV_ANNOTATIONS, - **HIGH_ANNOTATIONS, + 'splice_ai': 0.8, }, - 'in_silico': IN_SILICO_FILTER, + 'annotations_secondary': SV_ANNOTATIONS, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, 'Compound Heterozygous - One SV - Confirmed': { 'family_filter': { CONFIRMED_FAMILY_FILTER: True, }, - 'annotations': { - **SV_ANNOTATIONS, - **MODERATE_ANNOTATIONS_TRANSCRIPT_EXON_VARIANT, - }, + 'annotations': MODERATE_ANNOTATIONS_TRANSCRIPT_EXON_VARIANT, + 'annotations_secondary': SV_ANNOTATIONS, 'in_silico': IN_SILICO_FILTER, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, } @@ -426,7 +455,6 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('project') - @clickhouse_only def handle(self, *args, **options): family_guid_map = {} family_name_map = {} @@ -443,7 +471,7 @@ def handle(self, *args, **options): updates = {update: set() for update in ['matched_families', 'new_tag_keys', 'update_tag_keys', 'skipped_tag_keys']} search_counts = {} samples_by_dataset_type = {} - sample_qs = get_search_samples([project]) + sample_qs = Sample.objects.filter(individual__family__project=project, is_active=True) for dataset_type, searches in SEARCHES.items(): self._run_dataset_type_searches( dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, @@ -451,7 +479,7 @@ def handle(self, *args, **options): ) self._run_multi_data_type_comp_het_search( - updates, search_counts, samples_by_dataset_type, family_guid_map, project, sample_qs, genes=gene_by_moi[RECESSIVE_MOI], + updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes=gene_by_moi[RECESSIVE_MOI], ) new_tag_keys = updates['new_tag_keys'] @@ -504,17 +532,16 @@ def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, family_variant_data = defaultdict(lambda: {'matched_searches': set(), 'matched_comp_het_searches': set(), 'support_vars': set()}) logger.info(f'Searching for prioritized {dataset_type} variants in {len(samples_by_family)} families in project {project.name}') for search_name, config_search in searches.items(): + logger.info(f'Searching for criteria: {search_name}') exclude_locations = not config_search.get('gene_list_moi') search_genes = exclude_genes if exclude_locations else gene_by_moi[config_search['gene_list_moi']] sample_data = cls._get_valid_family_sample_data( project, sample_type, samples_by_family, config_search.get('family_filter'), ) - run_search_func = cls._run_comp_het_search if config_search['inheritance_mode'] == COMPOUND_HET else cls._run_search - num_results = run_search_func( - search_name, family_variant_data, family_guid_map, dataset_type, sample_data, sample_qs, + num_results = cls._execute_search( + {dataset_type: sample_data}, search_name, family_variant_data, family_guid_map, exclude_locations=exclude_locations, genes=search_genes, **config_search, **ALL_SEARCHES_CRITERIA, - ) - logger.info(f'Found {num_results} variants for criteria: {search_name}') + ) if sample_data['num_families'] else 0 search_counts[search_name] = num_results cls._bulk_tag_variants(family_variant_data, updates, dataset_type) @@ -530,13 +557,11 @@ def _get_valid_family_sample_data(cls, project, sample_type, samples_by_family, 'project_guids': [project.guid], 'num_families': len(samples_by_family), 'num_unaffected': sum(len(s['unaffected_guids']) for s in samples_by_family.values()), - 'sample_type_families': {sample_type: samples_by_family.keys()}, + 'sample_type_families': {sample_type: set(samples_by_family.keys())}, } @staticmethod def _family_passes_filter(sample_data, family_filter): - if family_filter.get(MAX_AFFECTED_FAMILY_FILTER) and len(sample_data['affecteds']) > family_filter[MAX_AFFECTED_FAMILY_FILTER]: - return False if family_filter.get(AFFECTED_MALE_FAMILY_FILTER) and all(s['sex'] not in Individual.MALE_SEXES for s in sample_data['affecteds']): return False if CONFIRMED_FAMILY_FILTER in family_filter: @@ -553,40 +578,22 @@ def wrapped(v): return {name: today for name in v[metadata_key]} if v[metadata_key] else None return wrapped - @classmethod - def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, samples, **kwargs): - results_qs = get_search_queryset(GENOME_VERSION_GRCh38, dataset_type, sample_data, **kwargs) - - variant_fields = [] - variant_values = {} - if getattr(results_qs.model, 'GENOTYPE_OVERRIDE_FIELDS', None): - genotype_overrides_expressions = results_qs.conditional_selects(results_qs) - variant_values.update({k: genotype_overrides_expressions[k] for k in ['familyGenotypes']}) - else: - variant_fields += ['familyGenotypes'] - - results = results_qs.values(*variant_fields, 'key', 'familyGuids', **variant_values) - add_individual_guids(results, samples, encode_genotypes_json=True) - - for variant in results: - for family_guid in variant.pop('familyGuids'): - variant_data = family_variant_data[(family_guid_map[family_guid], variant['key'])] - variant_data.update(variant) - variant_data['matched_searches'].add(search_name) - - return len(results) - - @classmethod - def _run_comp_het_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, samples, **kwargs): - queryset = get_data_type_comp_het_results_queryset( - GENOME_VERSION_GRCh38, dataset_type, sample_data, **kwargs, - ) - return cls._execute_comp_het_search( - queryset, search_name, family_variant_data, family_guid_map, samples, - ) + @staticmethod + def _parse_new_saved_variants(dataset_type): + def wrapped(new_variant_keys, family_variant_data): + new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} + variants_by_id = get_saved_variant_annotations( + {k for k, v in new_variant_data.items() if not (v.get('key') and v.get('variantId'))}, + dataset_type=dataset_type, genome_version=GENOME_VERSION_GRCh38, primary_id_field='key', + ) + return { + k: {**v, 'dataset_type': dataset_type, **(variants_by_id.get(k[1], {}))} + for k, v in family_variant_data.items() if k in new_variant_keys + } + return wrapped @classmethod - def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, samples, genes): + def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes): sv_dataset_type = next(dt for dt in samples_by_dataset_type.keys() if dt.startswith('SV')) sample_type = sv_dataset_type.split('_')[-1] families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].keys()) @@ -600,33 +607,43 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by family_variant_data = defaultdict(lambda: {'matched_searches': set(), 'matched_comp_het_searches': set(), 'support_vars': set()}) logger.info(f'Searching for prioritized multi data type variants in {len(families)} families in project {project.name}') for search_name, config_search in MULTI_DATA_TYPE_SEARCHES.items(): + logger.info(f'Searching for criteria: {search_name}') sv_sample_data = cls._get_valid_family_sample_data( project, sample_type, sv_samples_by_family, config_search.get('family_filter'), ) snv_indel_sample_data = cls._get_valid_family_sample_data( project, sample_type, snv_indel_samples_by_family, config_search.get('family_filter'), ) - queryset = get_multi_data_type_comp_het_results_queryset( - GENOME_VERSION_GRCh38, sv_dataset_type, sv_sample_data, snv_indel_sample_data, num_families=len(families), - genes=genes, **config_search, **ALL_SEARCHES_CRITERIA, + sample_data_by_dataset_type = { + Sample.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data, + } + num_results = cls._execute_search( + sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, + inheritance_mode=COMPOUND_HET, **config_search, **ALL_SEARCHES_CRITERIA, genes=genes, ) - num_results = cls._execute_comp_het_search(queryset, search_name, family_variant_data, family_guid_map, samples) - logger.info(f'Found {num_results} variants for criteria: {search_name}') search_counts[search_name] = num_results cls._bulk_tag_variants(family_variant_data, updates) - @classmethod - def _execute_comp_het_search(cls, queryset, search_name, family_variant_data, family_guid_map, samples): - results = [list(v[1:]) for v in queryset] - add_individual_guids(results, samples, encode_genotypes_json=True) - for pair in results: - for family_guid in pair[0]['familyGuids']: - for variant, support_id in [(pair[0], pair[1]['key']), (pair[1], pair[0]['key'])]: - variant_data = family_variant_data[(family_guid_map[family_guid], variant['key'])] - variant_data.update(variant) - variant_data['support_vars'].add(support_id) - variant_data['matched_comp_het_searches'].add(search_name) + @staticmethod + def _execute_search(sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, **kwargs): + results = get_clickhouse_variants( + families=None, dataset_types=None, search=kwargs, user=None, genome_version=GENOME_VERSION_GRCh38, + sample_data_by_dataset_type=sample_data_by_dataset_type, encode_genotypes_json=True + ) + for result in results: + if isinstance(result, list): + for family_guid in result[0]['familyGuids']: + for variant, support_id in [(result[0], result[1]['key']), (result[1], result[0]['key'])]: + variant_data = family_variant_data[(family_guid_map[family_guid], variant['key'])] + variant_data.update(variant) + variant_data['support_vars'].add(support_id) + variant_data['matched_comp_het_searches'].add(search_name) + else: + for family_guid in result.pop('familyGuids'): + variant_data = family_variant_data[(family_guid_map[family_guid], result['key'])] + variant_data.update(result) + variant_data['matched_searches'].add(search_name) return len(results) @@ -636,7 +653,7 @@ def _bulk_tag_variants(cls, family_variant_data, updates, dataset_type=None): new_tag_keys, update_tag_keys, skipped_tag_keys = bulk_create_tagged_variants( family_variant_data, tag_name=SEQR_TAG_TYPE, get_metadata=cls._get_metadata(today, 'matched_searches'), get_comp_het_metadata=cls._get_metadata(today, 'matched_comp_het_searches'), user=None, - remove_missing_metadata=False, primary_id_field='key', dataset_type=dataset_type, genome_version=GENOME_VERSION_GRCh38, + remove_missing_metadata=False, primary_id_field='key', parse_new_saved_variants=cls._parse_new_saved_variants(dataset_type), ) updates['new_tag_keys'].update(new_tag_keys) updates['update_tag_keys'].update(update_tag_keys - updates['new_tag_keys']) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 60d4780d51..356a0a615c 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand from seqr.models import Project, Family, VariantTag, VariantTagType -from seqr.utils.search.utils import backend_specific_call from seqr.utils.search.add_data_utils import trigger_delete_families_search import logging @@ -35,7 +34,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, trigger_delete_families_search)(from_project, list(families.values_list('guid', flat=True))) + trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True))) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index bb856d7dd3..55e19a2553 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -165,6 +165,7 @@ '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2025-12-02/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', @@ -184,6 +185,9 @@ b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', @@ -325,8 +329,12 @@ 'family_samples': {'F0000123_ABC': ['NA22882', 'NA20885']}, }, { 'callsets': ['invalid_sample.vcf'], - 'sample_type': 'WGS', + 'sample_type': 'WES', 'family_samples': {'F000003_3': ['NA22882', 'NA20885']}, +}, { + 'callsets': ['invalid_sample.vcf'], + 'sample_type': 'WES', + 'family_samples': {'F000002_2': ['HG00732', 'HG00733']}, }, { 'callsets': ['gcnv.bed.gz'], 'sample_type': 'WES', @@ -349,11 +357,6 @@ def mock_opened_file(index): @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda *args: True) class CheckNewSamplesTest(object): - CREATE_SNV_INDEL_SAMPLES_LOGS = [ - ('Loading 4 WES SNV_INDEL samples in 2 projects', None), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ] UPDATE_SAMPLE_LOGS = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], @@ -366,8 +369,6 @@ class CheckNewSamplesTest(object): ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), ] def set_up(self): @@ -422,7 +423,7 @@ def set_up(self): Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): + def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): self._set_loading_files() self.reset_logs() @@ -434,7 +435,7 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): logs = self.LIST_FILE_LOGS[:1] + [(f'Loading new samples from {num_runs} run(s)', None)] runs = [ ('GRCh38/SNV_INDEL', 'auto__2023-08-09'), ('GRCh37/SNV_INDEL', 'manual__2023-11-02'), - ('GRCh38/MITO', 'auto__2024-08-12'), ('GRCh38/SV', 'auto__2024-09-14'), + ('GRCh38/MITO', 'auto__2024-08-12'), ('GRCh38/MITO', 'auto__2025-12-02'), ('GRCh38/SV', 'auto__2024-09-14'), ('GRCh38/SNV_INDEL', 'hail_search_to_clickhouse_migration_WGS_R0877_neptune'), ] if single_call: @@ -466,29 +467,32 @@ def _test_success_call(self, anvil_email_calls): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) self._test_call(run_loading_logs={ - 'GRCh38/SNV_INDEL': self.CREATE_SNV_INDEL_SAMPLES_LOGS + [ - ('update 1 Samples', {'dbUpdate': mock.ANY}), + 'GRCh38/SNV_INDEL': [ + ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), + ('create 4 Samples', {'dbUpdate': mock.ANY}), + ('update 4 Samples', {'dbUpdate': mock.ANY}), + ('update 1 Samples', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, 'GRCh38/MITO': [ - ('Loading 2 WGS MITO samples in 1 projects', None) + ('Loading 2 WES MITO samples in 1 projects', None) ], 'GRCh38/SV': [ ('Loading 4 WES SV samples in 2 projects', None), + ('update 1 Familys', {'dbUpdate': mock.ANY}), ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ('update 3 Samples', {'dbUpdate': mock.ANY}), - ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), - ('Reload Summary: ', None), ], }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', + 'auto__2025-12-02': 'The following families are included in the callset but are missing some family members: 2 (HG00731)', }) # Test notifications @@ -588,6 +592,7 @@ def test_command(self): 'auto__2023-08-09': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0003_test (GRCh37)', 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37)', + 'auto__2025-12-02': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37)', 'auto__2024-09-14': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37), R0003_test (GRCh37)', } self._test_call(error_logs=error_logs) @@ -718,16 +723,18 @@ def test_command(self): if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) self._test_call(num_runs=2, run_loading_logs={ - 'GRCh38/SNV_INDEL': self.CREATE_SNV_INDEL_SAMPLES_LOGS + airtable_logs + self.UPDATE_SAMPLE_LOGS, + 'GRCh38/SNV_INDEL': [ + ('Loading 4 WES SNV_INDEL samples in 2 projects', None), + ('create 4 Samples', {'dbUpdate': mock.ANY}), + ('update 4 Samples', {'dbUpdate': mock.ANY}), + ] + airtable_logs + self.UPDATE_SAMPLE_LOGS, }) -class LocalCheckNewSamplesTest(DifferentDbTransactionSupportMixin, AuthenticationTestCase, CheckNewSamplesTest): +class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] databases = '__all__' - ES_HOSTNAME = '' - MOCK_DATA_DIR = '/seqr/seqr-hail-search-data' PROJECT_EMAIL_TEXT = TEXT_EMAIL_TEMPLATE.format(1, 'WES', 'Non-Analyst Project') PROJECT_EMAIL_HTML = HTML_EMAIL_TEMAPLTE.format(1, 'WES', EXTERNAL_PROJECT_GUID, 'Non-Analyst Project') @@ -769,7 +776,7 @@ def _assert_has_expected_empty_list_file_calls(self): self.mock_glob.assert_called_with('/seqr/seqr-hail-search-data/GRCh37/MITO/runs/*/*', recursive=False) def _set_reloading_loading_files(self): - self.mock_glob.return_value = [LOCAL_RUN_PATHS[3], LOCAL_RUN_PATHS[6]] + self.mock_glob.return_value = [LOCAL_RUN_PATHS[3], LOCAL_RUN_PATHS[7]] def _set_loading_files(self): if not self.mock_glob.return_value: @@ -788,7 +795,7 @@ def _assert_expected_loading_file_calls(self, single_call): self.assertEqual(self.mock_mkdir.call_count, 0 if single_call else 2) self.assertEqual(list(self.mock_written_files.keys()), [ file.replace('validation_errors.json', '_ERRORS_REPORTED') - for file in [LOCAL_RUN_PATHS[2], LOCAL_RUN_PATHS[7]] + for file in [LOCAL_RUN_PATHS[2], LOCAL_RUN_PATHS[8]] ]) def _assert_expected_airtable_calls(self, *args, **kwargs): @@ -916,7 +923,7 @@ def _assert_has_expected_empty_list_file_calls(self): ) def _set_reloading_loading_files(self): - self.mock_ls_process.communicate.return_value = b'\n'.join(RUN_PATHS[6:8] + RUN_PATHS[15:17]), b'' + self.mock_ls_process.communicate.return_value = b'\n'.join(RUN_PATHS[6:8] + RUN_PATHS[18:20]), b'' self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): @@ -940,6 +947,8 @@ def _assert_expected_loading_file_calls(self, single_call): ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh37/SNV_INDEL/runs/manual__2023-11-02/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/metadata.json', -2), ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/metadata.json', -2), + ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/metadata.json', -2), + ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/metadata.json', -2), ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', -2), diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py deleted file mode 100644 index 0f83df98d6..0000000000 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -import mock -import responses - -from django.core.exceptions import ObjectDoesNotExist -from django.core.management import call_command -from django.core.management.base import CommandError - -from seqr.models import Sample, Project -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase - -PROJECT_GUID = 'R0001_1kg' -VARIANT_ID = '21-3343353-GAGA-G' - - -class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - @responses.activate - @mock.patch('seqr.management.commands.deactivate_project_search.input') - def test_command(self, mock_input): - mock_input.return_value = 'n' - - # Test invalid project - with self.assertRaises(ObjectDoesNotExist): - call_command('deactivate_project_search', 'foo') - - # Test user did not confirm. - with self.assertRaises(CommandError) as e: - call_command('deactivate_project_search', PROJECT_GUID) - self.assertEqual(str(e.exception), 'Error: user did not confirm') - - # Test success - Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') - mock_input.return_value = 'y' - self.reset_logs() - self.maxDiff = None - call_command('deactivate_project_search', PROJECT_GUID) - self.assert_json_logs(user=None, expected=[ - ('update 11 Samples', {'dbUpdate': { - 'dbEntity': 'Sample', - 'entityIds': mock.ANY, - 'updateFields': ['is_active'], - 'updateType': 'bulk_update'}, - }), - ('Deactivated 11 samples', None), - ]) - - active_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True) - self.assertEqual(active_samples.count(), 0) - - # Re-running has no effect - self.reset_logs() - call_command('deactivate_project_search', PROJECT_GUID) - self.assert_json_logs(user=None, expected=[('Deactivated 0 samples', None)]) - - -class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] - - def test_command(self): - with self.assertRaises(ValueError) as e: - call_command('deactivate_project_search', 'foo') - self.assertEqual(str(e.exception), 'handle is disabled without the elasticsearch backend') diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py deleted file mode 100644 index 51b9f3e5c5..0000000000 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ /dev/null @@ -1,100 +0,0 @@ -#-*- coding: utf-8 -*- -import mock - -from django.core.management import call_command -from django.test import TestCase - -PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' -PROJECT_GUID = 'R0001_1kg' -FAMILY_GUID = 'F000001_1' - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -class ReloadSavedVariantJsonTest(TestCase): - fixtures = ['users', '1kg_project'] - - @mock.patch('seqr.views.utils.variant_utils.logger') - @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') - def test_with_param_command(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ - [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples}) if samples else ['F000012_12']} - for variant_id in variant_ids] - - # Test with a specific project and a family id. - call_command('reload_saved_variant_json', - PROJECT_NAME, - '--family-guid={}'.format(FAMILY_GUID)) - - mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A','21-3343353-GAGA-G'], None) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) - - logger_info_calls = [ - mock.call('Updated 2 variants in 1 families for project 1kg project n\xe5me with uni\xe7\xf8de'), - mock.call('Reload Summary: '), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Updated 2 variants') - ] - mock_logger.info.assert_has_calls(logger_info_calls) - mock_get_variants.reset_mock() - mock_logger.reset_mock() - - # Test for all projects and no specific family ids - call_command('reload_saved_variant_json') - - self.assertEqual(mock_get_variants.call_count, 3) - mock_get_variants.assert_has_calls([ - mock.call( - mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], None, - ), - mock.call(mock.ANY, '37', ['1-248367227-TC-T', 'prefix_19107_DEL'], None), - mock.call(mock.ANY, '38', ['1-248367227-TC-T'], None) - ]) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), { - 129, 130, 132, 133, 134, 145, 146, 148, 149, - }) - self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), set()) - self.assertSetEqual(set(mock_get_variants.call_args_list[2].args[0].values_list('id', flat=True)), {147, 173, 174}) - - logger_info_calls = [ - mock.call('Reloading saved variants in 4 projects'), - mock.call('Updated 3 variants for project 1kg project n\xe5me with uni\xe7\xf8de'), - mock.call('Updated 2 variants for project Test Reprocessed Project'), - mock.call('Updated 1 variants for project Non-Analyst Project'), - mock.call('Reload Summary: '), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Updated 3 variants'), - mock.call(' Test Reprocessed Project: Updated 2 variants'), - mock.call(' Non-Analyst Project: Updated 1 variants'), - mock.call('Skipped the following 1 project with no saved variants: Empty Project'), - ] - mock_logger.info.assert_has_calls(logger_info_calls) - mock_get_variants.reset_mock() - mock_logger.reset_mock() - - # Test with an exception. - mock_get_variants.side_effect = Exception("Database error.") - call_command('reload_saved_variant_json', - PROJECT_GUID, - '--family-guid={}'.format(FAMILY_GUID)) - - mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A', '21-3343353-GAGA-G'], None) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) - - logger_info_calls = [ - mock.call('Reload Summary: '), - mock.call('1 failed projects'), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Database error.') - ] - mock_logger.info.assert_has_calls(logger_info_calls) - - mock_logger.error.assert_called_with('Error reloading variants in 1kg project n\xe5me with uni\xe7\xf8de: Database error.') - - -class ClickhouseReloadSavedVariantJsonTest(TestCase): - - fixtures = ['users', '1kg_project'] - - def test_command(self): - with self.assertRaises(ValueError) as ce: - call_command('reload_saved_variant_json', PROJECT_NAME) - self.assertEqual(str(ce.exception), 'handle is disabled without the elasticsearch backend') - - diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 9b7c80a949..6519a3bd25 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -10,41 +10,45 @@ PROJECT_GUID = 'R0001_1kg' SNV_INDEL_MATCHES = { - 'Clinvar Pathogenic': 0, - 'Clinvar Pathogenic - Compound Heterozygous': 0, - 'Clinvar Both Pathogenic - Compound Heterozygous': 0, - 'Clinvar Pathogenic - Recessive': 1, - 'Clinvar Pathogenic - X-Linked Recessive': 0, - 'Compound Heterozygous': 1, - 'Compound Heterozygous - Confirmed': 0, - 'Compound Heterozygous - Both High Splice AI': 0, - 'Compound Heterozygous - Both High Splice AI - Confirmed': 0, - 'Compound Heterozygous - Clinvar Pathogenic/ High Splice AI': 0, - 'Compound Heterozygous - High Splice AI': 0, - 'Compound Heterozygous - High Splice AI - Confirmed': 0, - 'De Novo/ Dominant - Confirmed': 0, - 'De Novo/ Dominant - Non-coding Transcript Exon Variant': 0, - 'De Novo/ Dominant': 0, - 'High Splice AI - De Novo/ Dominant': 0, - 'High Splice AI - De Novo/ Dominant Confirmed': 0, - 'High Splice AI - Recessive': 0, - 'High Splice AI - Recessive Confirmed': 0, - 'Recessive': 1, - 'X-Linked Recessive': 0, + 'Clinvar Pathogenic': (0, None), + 'Clinvar Pathogenic - Compound Heterozygous': (0, None), + 'Clinvar Both Pathogenic - Compound Heterozygous': (0, None), + 'Clinvar Pathogenic - Recessive': (1, None), + 'Clinvar Pathogenic - X-Linked Recessive': (0, 2), + 'Compound Heterozygous': (1, None), + 'Compound Heterozygous - Confirmed': (0, 1), + 'Compound Heterozygous - Both High Splice AI': (0, 1), + 'Compound Heterozygous - Both High Splice AI - Confirmed': (0, 1), + 'Compound Heterozygous - Clinvar Pathogenic/ High Splice AI': (0, None), + 'Compound Heterozygous - High Splice AI': (0, 1), + 'Compound Heterozygous - High Splice AI - Confirmed': (0, 1), + 'De Novo/ Dominant - Confirmed': (0, 1), + 'De Novo/ Dominant - Non-coding Transcript Exon Variant': (0, 1), + 'De Novo/ Dominant': (0, None), + 'High Splice AI - De Novo/ Dominant': (0, 1), + 'High Splice AI - De Novo/ Dominant Confirmed': (0, 1), + 'High Splice AI - Recessive': (0, 1), + 'High Splice AI - Recessive Confirmed': (0, 1), + 'High Splice AI - X-Linked Recessive': (0, 1), + 'High Splice AI - X-Linked Recessive Confirmed': (0, 0), + 'Recessive': (1, None), + 'X-Linked Recessive': (0, 2), } SV_MATCHES = { - 'SV - Compound Heterozygous': 1, - 'SV - De Novo/ Dominant': 0, - 'SV - Recessive': 1, - 'SV - X-Linked Recessive': 0, + 'SV - Compound Heterozygous': (1, None), + 'SV - De Novo/ Dominant': (0, None), + 'SV - Recessive': (1, None), + 'SV - X-Linked Recessive': (0, 0), } MITO_MATCHES = { - 'Mitochondrial - Pathogenic': 1, - 'Mitochondrial - De Novo/ Dominant': 1, + 'Mitochondrial - Pathogenic': (1, None), + 'Mitochondrial - De Novo/ Dominant': (1, None), } MULTI_TYPE_MATCHES = { - 'Compound Heterozygous - One SV': 0, - 'Compound Heterozygous - One SV - Confirmed': 1, + 'Compound Heterozygous - One SV': (0, None), + 'Compound Heterozygous - Clinvar Pathogenic/ SV': (1, None), + 'Compound Heterozygous - High Splice AI/ SV': (0, None), + 'Compound Heterozygous - One SV - Confirmed': (1, None), } class CheckNewSamplesTest(ClickhouseSearchTestCase): @@ -68,29 +72,33 @@ def test_command(self, mock_datetime, mock_slack, mock_email): }) new_saved_variants = SavedVariant.objects.filter(key__in=[2, 3, 4, 18, 19]).order_by('key').values( - 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', + 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', 'main_transcript', 'sv_type', ) + self.maxDiff = None self.assertListEqual(list(new_saved_variants), [{ 'key': 2, 'variant_id': '1-38724419-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1038724419, 'xpos_end': 1038724419, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000177000', 'ENSG00000277258'], - 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, + 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT2['transcripts']['ENSG00000177000'][0], }, {'key': 3, 'variant_id': '1-91502721-G-A', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091502721, 'xpos_end': 1091502721, 'ref': 'G', 'alt': 'A', 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, - }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, + 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT3['transcripts']['ENSG00000097046'][0], + }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, 'xpos_end': 1091511686, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000097046'], - 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, + 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT4['transcripts']['ENSG00000097046'][0], }, {'key': 18, 'variant_id': 'suffix_140593_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038717327, 'xpos_end': 17038719636, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023'], - 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, + 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DUP', }, {'key': 19, 'variant_id': 'suffix_140608_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038721781, 'xpos_end': 17038735703, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023', 'ENSG00000277258', 'ENSG00000277972'], - 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, + 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DEL', }]) expected_tags = { (2,): {"Clinvar Pathogenic - Recessive": "2025-11-15", "Recessive": "2025-11-15"}, - (2, 19): {"Compound Heterozygous - One SV - Confirmed": "2025-11-15"}, + (2, 19): {"Compound Heterozygous - One SV - Confirmed": "2025-11-15", "Compound Heterozygous - Clinvar Pathogenic/ SV": "2025-11-15"}, (3, 4): {"Compound Heterozygous": "2025-11-15"}, (6,): {'Mitochondrial - De Novo/ Dominant': '2025-11-15'}, (8,): {"Mitochondrial - Pathogenic": "2025-11-15"}, @@ -143,16 +151,17 @@ def _assert_expected_logs(self, num_new=0, num_unchanged=0, creation_stats=None) 'MITO', 1, MITO_MATCHES, **creation_stats.get('MITO', {}), ) + self._dataset_type_logs( 'multi data type', 1, MULTI_TYPE_MATCHES, **creation_stats.get('MULTI', {}), + search_dataset_types=['SNV_INDEL', 'SV_WES', 'SNV_INDEL/SV_WES'], ) + [ (f'Tagged {num_new} new and 0 previously tagged variants in 1 families, found {num_unchanged} unchanged tags:', None) - ] + [(f' {criteria}: {count} variants', None) for criteria, count in SNV_INDEL_MATCHES.items()] + [ - (f' {criteria}: {count} variants', None) for criteria, count in SV_MATCHES.items() - ] + [(f' {criteria}: {count} variants', None) for criteria, count in MITO_MATCHES.items()] + [ - (f' {criteria}: {count} variants', None) for criteria, count in MULTI_TYPE_MATCHES.items() + ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in SNV_INDEL_MATCHES.items()] + [ + (f' {criteria}: {count} variants', None) for criteria, (count, _) in SV_MATCHES.items() + ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in MITO_MATCHES.items()] + [ + (f' {criteria}: {count} variants', None) for criteria, (count, _) in MULTI_TYPE_MATCHES.items() ]) - @staticmethod - def _dataset_type_logs(dataset_type, num_families, matches, num_variants=0, tag_id_range=None): + @classmethod + def _dataset_type_logs(cls, dataset_type, num_families, matches, num_variants=0, tag_id_range=None, search_dataset_types=None): create_variants_logs = [ (f'create {num_variants} SavedVariants', { 'dbUpdate': {'dbEntity': 'SavedVariant', 'entityIds': mock.ANY, 'updateType': 'bulk_create'}, @@ -160,11 +169,22 @@ def _dataset_type_logs(dataset_type, num_families, matches, num_variants=0, tag_ ] if num_variants > 0 else [] return [ (f'Searching for prioritized {dataset_type} variants in {num_families} families in project 1kg project n\u00e5me with uni\u00e7\u00f8de', None), - ] + [ - (f'Found {count} variants for criteria: {criteria}', None) for criteria, count in matches.items() - ] + create_variants_logs + [ + ] + [(log, None) for logs in [ + cls._criteria_search_logs(search_dataset_types or [dataset_type], criteria, count, num_criteria_families, num_families) + for criteria, (count, num_criteria_families) in matches.items() + ] for log in logs] + create_variants_logs + [ (f'create VariantTag VT{db_id}_seqr_prioritized', {'dbUpdate': { 'dbEntity': 'VariantTag', 'entityId': f'VT{db_id}_seqr_prioritized', 'updateFields': ['metadata', 'variant_tag_type'], 'updateType': 'create', }}) for db_id in range(*(tag_id_range or [0])) ] + @staticmethod + def _criteria_search_logs(search_dataset_types, criteria, count, num_criteria_families, num_families): + logs = [f'Searching for criteria: {criteria}'] + if num_criteria_families == 0: + return logs + return logs + [log for logs in [ + [f'Loading {dataset_type} data for {num_criteria_families or num_families} families', + ] + ([f'Total results: {count}'] if i == len(search_dataset_types) - 1 else []) + for i, dataset_type in enumerate(search_dataset_types)] for log in logs] + diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 914ae2507f..b2feeecd53 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -2,13 +2,16 @@ import responses from seqr.models import Family, VariantTagType, VariantTag, Sample -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -class TransferFamiliesTest(object): +class TransferFamiliesClickhouseTest(AnvilAuthenticationTestCase): + fixtures = ['users', '1kg_project'] - DEACTIVATE_SEARCH = True - LOGS = [] + LOGS = [ + ('Disabled search for 7 samples in the following 1 families: 2', None), + ('Triggered Delete Families', {'detail': {'project_guid': 'R0001_1kg', 'family_guids': ['F000002_2', 'F000004_4']}}), + ] @responses.activate def test_command(self): @@ -44,24 +47,8 @@ def test_command(self): samples = Sample.objects.filter(individual__family=family) self.assertEqual(samples.count(), 7) - self.assertEqual(samples.filter(is_active=True).count(), 0 if self.DEACTIVATE_SEARCH else 7) + self.assertEqual(samples.filter(is_active=True).count(), 0) family = Family.objects.get(family_id='4') self.assertEqual(family.project.guid, 'R0003_test') self.assertEqual(family.individual_set.count(), 1) - - -class TransferFamiliesLocalTest(TransferFamiliesTest, AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - DEACTIVATE_SEARCH = False - - -class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - ES_HOSTNAME = '' - LOGS = [ - ('Disabled search for 7 samples in the following 1 families: 2', None), - ('Triggered Delete Families', {'detail': {'project_guid': 'R0001_1kg', 'family_guids': ['F000002_2', 'F000004_4']}}), - ] diff --git a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py new file mode 100644 index 0000000000..b3d71bb53d --- /dev/null +++ b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py @@ -0,0 +1,133 @@ +# Generated by Django 4.2.24 on 2026-03-19 15:57 + +from collections import defaultdict +from django.db import migrations, models + +from clickhouse_search.backend.fields import NamedTupleField +from clickhouse_search.backend.functions import ArrayIndex, ArrayObjectSort +from clickhouse_search.models.search_models import VARIANT_DETAILS_CLASS_MAP +from clickhouse_search.search import get_variant_details_queryset +import seqr.views.utils.json_utils + +BATCH_SIZE = 10000 + + +def populate_saved_variant_fields(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + no_key_variants = SavedVariant.objects.using(db_alias).filter(key__isnull=True) + sv_variants = no_key_variants.filter(saved_variant_json__svType__isnull=False, ).exclude( + saved_variant_json__svType=models.Value('null'), + ) + sv_variants.update( + sv_type=models.fields.json.KeyTextTransform('svType', 'saved_variant_json'), + dataset_type='SV', + ) + # For SVs that have dropped out of the callset, set correct variant ID for reporting + sv_variants.filter(saved_variant_json__populations__isnull=False).update( + variant_id=models.functions.Concat( + 'sv_type', models.Value(':chr'), models.fields.json.KeyTextTransform('chrom', 'saved_variant_json'), + models.Value(':'), 'saved_variant_json__pos', models.Value('-'), 'saved_variant_json__end', + output_field=models.CharField(), + )) + + no_key_variants.filter(sv_type__isnull=True).update(dataset_type='SNV_INDEL') + no_key_variants.filter(saved_variant_json__genotypes__isnull=False).update( + genotypes=models.F('saved_variant_json__genotypes'), + ) + + to_update = [] + transcript_variants = no_key_variants.filter(sv_type__isnull=True, saved_variant_json__transcripts__isnull=False).exclude(saved_variant_json__transcripts={}) + for variant in transcript_variants: + main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') + if main_transcript_id: + variant.main_transcript = next( + t for ts in variant.saved_variant_json['transcripts'].values() for t in ts + if t['transcriptId'] == main_transcript_id + ) + to_update.append(variant) + SavedVariant.objects.using(db_alias).bulk_update(to_update, ['main_transcript']) + + +def populate_clickhouse_main_transcript(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'MITO', + expression=models.F('sorted_transcript_consequences__0'), + ) + + for genome_version in ['38', '37']: + output_field = VARIANT_DETAILS_CLASS_MAP[genome_version].transcripts.field.clone() + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, + family__project__genome_version=genome_version, + expression=ArrayIndex(0, ArrayObjectSort('transcripts', sort_field='transcriptRank'), base_field=NamedTupleField(output_field.base_fields)), + ) + + output_field.group_by_key = 'transcriptId' + output_field.flatten_groups = True + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, + family__project__genome_version=genome_version, selected_main_transcript_id__isnull=False, + expression=ArrayObjectSort('transcripts', sort_field='transcriptId', output_field=output_field), + process_value=lambda transcripts, variant: transcripts.get(variant.selected_main_transcript_id, variant.main_transcript), + ) + + +def populate_clickhouse_sv_type(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + for dataset_type in ['SV_WGS', 'SV_WES']: + _update_field_from_clickhouse(SavedVariant, db_alias, 'sv_type', dataset_type) + + +def _update_field_from_clickhouse(SavedVariant, db_alias, field, dataset_type, genome_version='38', expression=None, process_value=None, **filters): + variants = SavedVariant.objects.using(db_alias).filter(key__isnull=False, dataset_type=dataset_type, **filters) + if not variants.exists(): + return + variants_by_key = defaultdict(list) + for v in variants: + variants_by_key[v.key].append(v) + all_keys = sorted(variants_by_key.keys()) + print(f'Populating {field} for {len(all_keys)} {dataset_type} ({genome_version}) clickhouse variants') + for i in range(0, len(all_keys), BATCH_SIZE): + batch_keys = all_keys[i:i + BATCH_SIZE] + clickhouse_qs = get_variant_details_queryset(genome_version, dataset_type, batch_keys) + to_update = [] + if expression: + clickhouse_qs = clickhouse_qs.annotate(**{field: expression}) + for key, value in clickhouse_qs.using('clickhouse').values_list('key', field): + for variant in variants_by_key[key]: + if process_value: + value = process_value(value, variant) + setattr(variant, field, value) + to_update.append(variant) + num_updated = SavedVariant.objects.using(db_alias).bulk_update(to_update, [field]) + print(f'Updated {num_updated} variants') + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0084_rename_restrict_hpo_sharing_project_restrict_sharing'), + ] + + operations = [ + migrations.AddField( + model_name='savedvariant', + name='main_transcript', + field=models.JSONField(default=dict, encoder=seqr.views.utils.json_utils.DjangoJSONEncoderWithSets), + ), + migrations.AddField( + model_name='savedvariant', + name='sv_type', + field=models.CharField(max_length=15, null=True), + ), + migrations.RunPython(populate_saved_variant_fields, reverse_code=migrations.RunPython.noop), + migrations.RunPython(populate_clickhouse_main_transcript, reverse_code=migrations.RunPython.noop), + migrations.RunPython(populate_clickhouse_sv_type, reverse_code=migrations.RunPython.noop), + ] diff --git a/seqr/models.py b/seqr/models.py index c407d33787..36363d728c 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -14,6 +14,7 @@ from seqr.utils.logging_utils import log_model_update, log_model_bulk_update, SeqrLogger from seqr.utils.xpos_utils import get_chrom_pos +from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.terra_api_utils import anvil_enabled from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_CHOICES from settings import MME_DEFAULT_CONTACT_NAME, MME_DEFAULT_CONTACT_HREF, MME_DEFAULT_CONTACT_INSTITUTION, \ @@ -843,7 +844,9 @@ class SavedVariant(ModelWithGUID): selected_main_transcript_id = models.CharField(max_length=20, null=True) saved_variant_json = models.JSONField(default=dict) genotypes = models.JSONField(default=dict) + main_transcript = models.JSONField(default=dict, encoder=DjangoJSONEncoderWithSets) gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True) + sv_type = models.CharField(max_length=15, null=True) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True) acmg_classification = models.JSONField(null=True) # ACMG based classification diff --git a/seqr/urls.py b/seqr/urls.py index 8b21f25637..ab0b4b461f 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -5,7 +5,6 @@ """ from seqr.views.react_app import main_app, no_login_main_app from seqr.views.status import status_view -from seqr.views.apis.dataset_api import add_variants_dataset_handler from settings import ENABLE_DJANGO_DEBUG_TOOLBAR, MEDIA_ROOT, API_LOGIN_REQUIRED_URL, LOGIN_URL, DEBUG, \ API_POLICY_REQUIRED_URL from django.conf.urls import include @@ -53,14 +52,14 @@ from seqr.views.apis.saved_variant_api import \ saved_variant_data, \ create_saved_variant_handler, \ + create_manual_saved_variant_handler, \ update_variant_tags_handler, \ update_variant_acmg_classification_handler, \ update_variant_functional_data_handler, \ create_variant_note_handler, \ update_variant_note_handler, \ delete_variant_note_handler, \ - update_variant_main_transcript, \ - update_saved_variant_json + update_variant_main_transcript from seqr.views.apis.dashboard_api import dashboard_page_data @@ -92,6 +91,7 @@ update_mme_result_status, \ update_mme_contact_note, \ update_mme_project_contact, \ + send_vlm_email, \ send_mme_contact_email from seqr.views.apis.variant_search_api import \ @@ -122,8 +122,7 @@ update_user, \ forgot_password -from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ - update_rna_seq, proxy_to_kibana, load_phenotype_prioritization_data, \ +from seqr.views.apis.data_manager_api import update_rna_seq, load_phenotype_prioritization_data, \ validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ trigger_delete_project, trigger_delete_family from seqr.views.apis.report_api import \ @@ -133,7 +132,7 @@ gregor_export, \ seqr_stats from seqr.views.apis.summary_data_api import success_story, saved_variants_page, mme_details, hpo_summary_data, \ - bulk_update_family_external_analysis, individual_metadata, send_vlm_email + bulk_update_family_external_analysis, individual_metadata from seqr.views.apis.superuser_api import get_all_users from seqr.views.apis.awesomebar_api import awesomebar_autocomplete_handler @@ -161,6 +160,7 @@ 'gene_info/.*', 'gene_lists/.*', 'variant_search/.*', + 'variant_lookup/.*', 'report/.*', 'data_management/.*', 'summary_data/.*', @@ -241,7 +241,6 @@ 'project/(?P[^/]+)/upload_individuals_table': receive_individuals_table_handler, 'project/(?P[^/]+)/save_individuals_table/(?P[^/]+)': save_individuals_table_handler, 'project/(?P[^/]+)/upload_igv_dataset': receive_igv_table_handler, - 'project/(?P[^/]+)/add_dataset/variants': add_variants_dataset_handler, 'project/(?P[^/]+)/update_rna_seq': update_project_rna_seq, 'project/(?P[^/]+)/igv_track/(?P.+)': fetch_igv_track, @@ -254,7 +253,6 @@ 'project/(?P[^/]+)/dynamic_analysis_groups/create': update_dynamic_analysis_group_handler, 'project/(?P[^/]+)/dynamic_analysis_groups/(?P[^/]+)/update': update_dynamic_analysis_group_handler, 'project/(?P[^/]+)/dynamic_analysis_groups/(?P[^/]+)/delete': delete_dynamic_analysis_group_handler, - 'project/(?P[^/]+)/update_saved_variant_json': update_saved_variant_json, 'project/(?P[^/]+)/add_workspace_data': add_workspace_data, 'search/variant/(?P[^/]+)': query_single_variant_handler, @@ -270,6 +268,7 @@ 'saved_search/(?P[^/]+)/delete': delete_saved_search_handler, 'saved_variant/create': create_saved_variant_handler, + 'saved_variant/create_manual/(?P[\w.|-]+)': create_manual_saved_variant_handler, 'saved_variant/(?P[^/]+)/update_tags': update_variant_tags_handler, 'saved_variant/(?P[^/]+)/update_acmg_classification': update_variant_acmg_classification_handler, 'saved_variant/(?P[^/]+)/update_functional_data': update_variant_functional_data_handler, @@ -333,8 +332,6 @@ 'report/gregor': gregor_export, 'report/seqr_stats': seqr_stats, - 'data_management/elasticsearch_status': elasticsearch_status, - 'data_management/delete_index': delete_index, 'data_management/get_all_users': get_all_users, 'data_management/update_rna_seq': update_rna_seq, 'data_management/load_phenotype_prioritization_data': load_phenotype_prioritization_data, @@ -352,7 +349,8 @@ 'summary_data/matchmaker': mme_details, 'summary_data/update_external_analysis': bulk_update_family_external_analysis, 'summary_data/individual_metadata/(?P[^/]+)': individual_metadata, - 'summary_data/send_vlm_email': send_vlm_email, + + 'send_vlm_email': send_vlm_email, 'create_project_from_workspace/(?P[^/]+)/(?P[^/]+)/grant_access': grant_workspace_access, 'create_project_from_workspace/(?P[^/]+)/(?P[^/]+)/validate_vcf': validate_anvil_vcf, @@ -396,17 +394,6 @@ handler401 = 'seqr.views.apis.auth_api.app_login_required_error' -kibana_urls = '^(?:{})'.format('|'.join([ - 'app', '\d+/built_assets', '\d+/bundles', 'bundles', 'elasticsearch', 'es_admin', 'node_modules/@kbn', 'internal', - 'plugins', 'translations', 'ui', 'api/apm', 'api/console', 'api/core', 'api/index_management', 'api/index_patterns', - 'api/kibana', 'api/licensing', 'api/monitoring', 'api/reporting', 'api/saved_objects', 'api/telemetry', - 'api/timelion', 'api/ui_metric', 'api/xpack', 'bootstrap', -])) - -urlpatterns += [ - re_path(kibana_urls, proxy_to_kibana, name='proxy_to_kibana'), -] - urlpatterns += [ re_path(r'^admin/login/$', RedirectView.as_view(url=LOGIN_URL, permanent=True, query_string=True)), re_path(r'^admin/', admin.site.urls), diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 33d22532a3..4f694435c9 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -11,7 +11,7 @@ import json import traceback -from seqr.utils.search.utils import ERROR_LOG_EXCEPTIONS, SEARCH_EXCEPTION_ERROR_MAP, SEARCH_EXCEPTION_MESSAGE_MAP +from seqr.utils.search.utils import InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.terra_api_utils import TerraAPIException @@ -33,12 +33,12 @@ def __init__(self, errors, warnings=None): ObjectDoesNotExist: 404, Http404: 404, ErrorsWarningsException: 400, + InvalidSearchException: 400, AuthException: 401, HTTPError: lambda e: int(e.response.status_code), TerraAPIException: lambda e: e.status_code, AnymailError: lambda e: getattr(e, 'status_code', None) or 400, } -EXCEPTION_ERROR_MAP.update(SEARCH_EXCEPTION_ERROR_MAP) EXCEPTION_JSON_MAP = { ErrorsWarningsException: lambda e: {'errors': e.errors, 'warnings': e.warnings} @@ -47,7 +47,6 @@ def __init__(self, errors, warnings=None): EXCEPTION_MESSAGE_MAP = { TerraAPIException: lambda e: LOGIN_URL if e.status_code == 401 else str(e), } -EXCEPTION_MESSAGE_MAP.update(SEARCH_EXCEPTION_MESSAGE_MAP) def _get_exception_status_code(exception): @@ -55,10 +54,8 @@ def _get_exception_status_code(exception): if isinstance(status, int): return status - try: - return status(exception) - except Exception: - return 500 + return status(exception) + def _get_core_exception_json(exception): exception_json_formatter = next((f for exc, f in EXCEPTION_JSON_MAP.items() if isinstance(exception, exc)), None) @@ -75,8 +72,6 @@ class JsonErrorMiddleware(MiddlewareMixin): def process_exception(request, exception): exception_json = _get_core_exception_json(exception) status = _get_exception_status_code(exception) - if exception.__class__ in ERROR_LOG_EXCEPTIONS: - exception_json['log_error'] = True if DEBUG or status == 500: traceback_message = traceback.format_exc() exception_json['traceback'] = traceback_message diff --git a/seqr/utils/search/elasticsearch/__init__.py b/seqr/utils/search/elasticsearch/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/seqr/utils/search/elasticsearch/constants.py b/seqr/utils/search/elasticsearch/constants.py deleted file mode 100644 index 250d2f3b40..0000000000 --- a/seqr/utils/search/elasticsearch/constants.py +++ /dev/null @@ -1,418 +0,0 @@ -from django.db.models import Min - -from reference_data.models import Omim, GeneConstraint -from seqr.models import Sample, PhenotypePrioritization -from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, \ - PATHOGENICTY_HGMD_SORT_KEY, PRIORITIZED_GENE_SORT - - -MAX_COMPOUND_HET_GENES = 1000 -MAX_INDEX_NAME_LENGTH = 4000 -MAX_SEARCH_CLAUSES = 1024 -MAX_INDEX_SEARCHES = 75 -PREFILTER_SEARCH_SIZE = 200 - -GENOTYPE_QUERY_MAP = { - REF_REF: {'not_allowed_num_alt': ['samples_no_call', 'samples_num_alt_1', 'samples_num_alt_2', 'samples']}, - REF_ALT: {'allowed_num_alt': ['samples_num_alt_1', 'samples']}, - ALT_ALT: {'allowed_num_alt': ['samples_num_alt_2', 'samples_cn_0', 'samples_cn_2', 'samples_cn_gte_4']}, - HAS_ALT: {'allowed_num_alt': ['samples_num_alt_1', 'samples_num_alt_2', 'samples']}, - HAS_REF: { - 'not_allowed_num_alt': ['samples_no_call', 'samples_num_alt_2', 'samples_cn_0', 'samples_cn_gte_4'], - }, -} - -PATH_FREQ_OVERRIDE_CUTOFF = 0.05 - -CLINVAR_PATH_SIGNIFICANCES = {'pathogenic', 'likely_pathogenic'} - -HGMD_CLASS_MAP = { - 'disease_causing': ['DM'], - 'likely_disease_causing': ['DM?'], - 'hgmd_other': ['DP', 'DFP', 'FP', 'FTV'], -} - -POPULATIONS = { - 'sv_callset': { - 'AF': 'sf', - 'filter_AF': [], - 'AC': 'sc', - 'AN': 'sn', - }, - 'callset': { - 'AF': 'AF', - 'filter_AF': [], - 'AC': 'AC', - 'AN': 'AN', - 'Hom': 'homozygote_count', - }, - 'topmed': { - 'filter_AF': [], - 'Het': None, - }, - 'exac': { - 'filter_AF': ['exac_AF_POPMAX'], - 'AC': 'exac_AC_Adj', - 'AN': 'exac_AN_Adj', - 'Hom': 'exac_AC_Hom', - 'Hemi': 'exac_AC_Hemi', - }, - 'gnomad_exomes': { - 'filter_AF': ['gnomad_exomes_AF_POPMAX_OR_GLOBAL'], - }, - 'gnomad_genomes': { - 'filter_AF': ['gnomad_genomes_AF_POPMAX_OR_GLOBAL'], - }, - 'gnomad_svs': {}, - 'callset_heteroplasmy': { - 'AN': 'AN', - 'AC': 'AC_het', - 'AF': 'AF_het', - }, - 'gnomad_mito': {'max_hl': None}, - 'gnomad_mito_heteroplasmy': { - 'AN': 'gnomad_mito_AN', - 'AC': 'gnomad_mito_AC_het', - 'AF': 'gnomad_mito_AF_het', - 'max_hl': 'gnomad_mito_max_hl' - }, - 'helix': {'max_hl': None}, - 'helix_heteroplasmy': { - 'AN': 'helix_AN', - 'AC': 'helix_AC_het', - 'AF': 'helix_AF_het', - 'max_hl': 'helix_max_hl', - } -} - -POPULATION_FIELD_CONFIGS = { - 'AF': {'format_value': float}, - 'filter_AF': {'format_value': lambda val: float(val) if val is not None else None, 'default_value': None}, - 'AC': {}, - 'AN': {}, - 'Hom': {}, - 'Hemi': {}, - 'Het': {}, - 'ID': {'format_value': str, 'default_value': None}, - 'max_hl': {'format_value': float}, -} -for population, pop_config in POPULATIONS.items(): - for freq_field in POPULATION_FIELD_CONFIGS.keys(): - if freq_field not in pop_config: - freq_suffix = freq_field - pop_config[freq_field] = '{}_{}'.format(population, freq_suffix) - -DEFAULT_POP_FIELD_CONFIG = { - 'format_value': int, - 'default_value': 0, -} -POPULATION_RESPONSE_FIELD_CONFIGS = {k: dict(DEFAULT_POP_FIELD_CONFIG, **v) for k, v in POPULATION_FIELD_CONFIGS.items()} - -CLINVAR_SORT = { - '_script': { - 'type': 'number', - 'script': { - 'source': """ - if (!doc.containsKey('clinvar_clinical_significance') || doc['clinvar_clinical_significance'].empty ) { - return 2; - } - String clinsig = doc['clinvar_clinical_significance'].value; - if (clinsig.indexOf('Pathogenic') >= 0 || clinsig.indexOf('Likely_pathogenic') >= 0) { - return 0; - } else if (clinsig.indexOf('Benign') >= 0 || clinsig.indexOf('Likely_benign') >= 0) { - return 3; - } - return 1; - """ - } - } -} - - -def _get_phenotype_priority_ranks_by_gene(samples, *args): - families = {s.individual.family for s in samples} - family_ranks = PhenotypePrioritization.objects.filter( - individual__family=list(families)[0], rank__lte=100).values('gene_id').annotate(min_rank=Min('rank')) - return {agg['gene_id']: agg['min_rank'] for agg in family_ranks} - - -SORT_FIELDS = { - PATHOGENICTY_SORT_KEY: [CLINVAR_SORT], - PATHOGENICTY_HGMD_SORT_KEY: [CLINVAR_SORT, { - '_script': { - 'type': 'number', - 'script': { - 'source': "(doc.containsKey('hgmd_class') && !doc['hgmd_class'].empty && doc['hgmd_class'].value == 'DM') ? 0 : 1" - } - } - }], - 'in_omim': [{ - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': lambda *args: [omim.gene.gene_id for omim in Omim.objects.filter( - phenotype_mim_number__isnull=False, gene__isnull=False).only('gene__gene_id')] - }, - 'source': "(doc.containsKey('mainTranscript_gene_id') && !doc['mainTranscript_gene_id'].empty && params.omim_gene_ids.contains(doc['mainTranscript_gene_id'].value)) ? 0 : 1", - } - } - }, { - '_script': { - 'type': 'number', - 'script': { - 'source': """ - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.omim_gene_ids.contains(doc['geneIds'][i])) { - return 0; - } - } - return 1 - """ - } - } - }], - PRIORITIZED_GENE_SORT: [{ - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'prioritized_ranks_by_gene': _get_phenotype_priority_ranks_by_gene, - }, - 'source': """ - int min_rank = 1000000; - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.prioritized_ranks_by_gene.getOrDefault(doc['geneIds'][i], 1000000) < min_rank) { - min_rank = params.prioritized_ranks_by_gene.get(doc['geneIds'][i]) - } - } - return min_rank; - """ - } - } - }], - 'protein_consequence': [{ - '_script': { - 'type': 'number', - 'script': { - 'source': "doc.containsKey('svType') ? 4.5 : (doc['mainTranscript_major_consequence_rank'].empty ? 1000 : doc['mainTranscript_major_consequence_rank'].value)" - } - } - }], - 'constraint': [{ - '_script': { - 'order': 'asc', - 'type': 'number', - 'script': { - 'params': { - 'constraint_ranks_by_gene': lambda *args: { - constraint.gene.gene_id: constraint.mis_z_rank + constraint.pLI_rank - for constraint in GeneConstraint.objects.all().only('gene__gene_id', 'mis_z_rank', 'pLI_rank')} - }, - 'source': """ - int min = 1000000000; - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.constraint_ranks_by_gene.getOrDefault(doc['geneIds'][i], 1000000000) < min) { - min = params.constraint_ranks_by_gene.get(doc['geneIds'][i]) - } - } - return min - """ - } - }, - }], - 'size': [{ - '_script': { - 'type': 'number', - 'script': { - 'source': "(doc.containsKey('svType') && (doc['svType'].value == 'BND' || doc['svType'].value == 'CTX')) ? -50 : doc['start'].value - doc['end'].value" - } - } - }], - XPOS_SORT_KEY: ['xpos'], -} -POPULATION_SORTS = { - sort: [{ - '_script': { - 'type': 'number', - 'script': { - 'params': {'field': POPULATIONS[pop_key]['AF']}, - 'source': "doc.containsKey(params.field) ? (doc[params.field].empty ? 0 : doc[params.field].value) : 1" - } - } - }] for sort, pop_key in {'gnomad': 'gnomad_genomes', 'gnomad_exomes': 'gnomad_exomes', 'callset_af': 'callset'}.items()} -SORT_FIELDS.update(POPULATION_SORTS) -PREDICTOR_SORT_FIELDS = { - 'cadd': 'cadd_PHRED', - 'revel': 'dbnsfp_REVEL_score', - 'eigen': 'eigen_Eigen_phred', - 'mpc': 'mpc_MPC', - 'splice_ai': 'splice_ai_delta_score', - 'primate_ai': 'primate_ai_score', -} -SORT_FIELDS.update({ - sort: [{sort_field: {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}] - for sort, sort_field in PREDICTOR_SORT_FIELDS.items() -}) - -SCREEN_KEY = 'SCREEN' -CLINVAR_KEY = 'clinvar' -CLINVAR_FIELDS = ['clinical_significance', 'variation_id', 'allele_id', 'gold_stars'] -HGMD_KEY = 'hgmd' -HGMD_FIELDS = ['accession', 'class'] -GENOTYPES_FIELD_KEY = 'genotypes' -HAS_ALT_FIELD_KEYS = ['samples_num_alt_1', 'samples_num_alt_2', 'samples'] -SORTED_TRANSCRIPTS_FIELD_KEY = 'sortedTranscriptConsequences' -CANONICAL_TRANSCRIPT_FILTER = 'non_coding_transcript_exon_variant__canonical' -NESTED_FIELDS = { - field_name: {field: {} for field in fields} for field_name, fields in { - CLINVAR_KEY: CLINVAR_FIELDS, - HGMD_KEY: HGMD_FIELDS, - }.items() -} - -GRCH38_LOCUS_FIELD = 'rg37_locus' -XSTOP_FIELD = 'xstop' -SPLICE_AI_FIELD = 'splice_ai' -CORE_FIELDS_CONFIG = { - 'alt': {}, - 'contig': {'response_key': 'chrom'}, - 'end': {'format_value': int}, - 'filters': {'response_key': 'genotypeFilters', 'format_value': ','.join, 'default_value': ''}, - 'num_exon': {'response_key': 'numExon'}, - 'originalAltAlleles': {'format_value': lambda alleles: [a.split('-')[-1] for a in alleles], 'default_value': []}, - 'ref': {}, - 'rsid': {}, - 'screen_region_type': {'response_key': 'screenRegionType', 'format_value': lambda types: types[0] if types else None}, - 'start': {'response_key': 'pos', 'format_value': int}, - 'svType': {}, - 'variantId': {}, - 'xpos': {'format_value': int}, - XSTOP_FIELD: {'format_value': int}, - 'rg37_locus_end': {'response_key': 'rg37LocusEnd', 'format_value': lambda locus: locus.to_dict()}, - 'sv_type_detail': {'response_key': 'svTypeDetail'}, - 'cpx_intervals': { - 'response_key': 'cpxIntervals', - 'format_value': lambda intervals: [interval.to_dict() for interval in (intervals or [])], - }, - 'algorithms': {'format_value': ', '.join}, - 'bothsides_support': {'response_key': 'bothsidesSupport'}, -} -MITO_CORE_FIELDS_CONFIG = { - 'common_low_heteroplasmy': {'response_key': 'commonLowHeteroplasmy'}, - 'high_constraint_region': {'response_key': 'highConstraintRegion'}, - 'mitomap_pathogenic': {'response_key': 'mitomapPathogenic'}, -} -CORE_FIELDS_CONFIG.update(MITO_CORE_FIELDS_CONFIG) -PREDICTION_FIELDS_CONFIG = { - 'cadd_PHRED': {'response_key': 'cadd'}, - 'dbnsfp_DANN_score': {}, - 'eigen_Eigen_phred': {}, - 'dbnsfp_VEST4_score': { - 'response_key': 'vest', - 'format_value': lambda x: x and next((v for v in x.split(';') if v != '.'), None), - }, - 'dbnsfp_MutPred_score': {'response_key': 'mut_pred', 'format_value': lambda x: None if x == '-' else x}, - 'mpc_MPC': {}, - 'dbnsfp_MutationTaster_pred': {'response_key': 'mut_taster'}, - 'dbnsfp_Polyphen2_HVAR_pred': {'response_key': 'polyphen'}, - 'gnomad_non_coding_constraint_z_score': {'response_key': 'gnomad_noncoding'}, - 'primate_ai_score': {'response_key': 'primate_ai'}, - 'splice_ai_delta_score': {'response_key': SPLICE_AI_FIELD}, - 'splice_ai_splice_consequence': {'response_key': 'splice_ai_consequence'}, - 'dbnsfp_REVEL_score': {}, - 'dbnsfp_SIFT_pred': {}, - 'StrVCTVRE_score': {'response_key': 'strvctvre'}, -} -MITO_PREDICTION_FIELDS_CONFIG = { - 'mitimpact_apogee': {}, - 'hap_defining_variant': {'response_key': 'haplogroup_defining', 'format_value': lambda k: 'Y' if k else None}, - 'mitotip_mitoTIP': {}, - 'hmtvar_hmtVar': {}, -} -PREDICTION_FIELDS_CONFIG.update(MITO_PREDICTION_FIELDS_CONFIG) - -def get_prediction_response_key(key): - return key.split('_')[1].lower() - -PREDICTION_FIELD_LOOKUP = { - field_config.get('response_key', get_prediction_response_key(field)): field - for field, field_config in PREDICTION_FIELDS_CONFIG.items() -} -MULTI_FIELD_PREDICTORS = { - 'fathmm': ['dbnsfp_fathmm_MKL_coding_pred', 'dbnsfp_FATHMM_pred'] -} -PREDICTION_FIELDS_RESPONSE_CONFIG = {k: {'response_key': k} for k, v in MULTI_FIELD_PREDICTORS.items()} -PREDICTION_FIELDS_RESPONSE_CONFIG.update(PREDICTION_FIELDS_CONFIG) - -QUALITY_QUERY_FIELDS = {'gq_sv': 5} -SHARED_QUALITY_FIELDS = {'gq': 5} -SNP_QUALITY_FIELDS = {'ab': 5} -SNP_QUALITY_FIELDS.update(SHARED_QUALITY_FIELDS) -SV_QUALITY_FIELDS = {'qs': 10} -SV_QUALITY_FIELDS.update(SHARED_QUALITY_FIELDS) -QUALITY_QUERY_FIELDS.update(SNP_QUALITY_FIELDS) -QUALITY_QUERY_FIELDS.update(SV_QUALITY_FIELDS) -BASE_GENOTYPE_FIELDS_CONFIG = { - 'sample_id': {}, - 'sample_type': {}, - 'num_alt': {'format_value': int, 'default_value': -1}, -} - -GENOTYPE_FIELDS_CONFIG = { - 'ad': {}, - 'dp': {}, - 'pl': {}, -} -GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SNP_QUALITY_FIELDS.keys()}) -MITO_GENOTYPE_FIELDS_CONFIG = { - 'dp': {}, - 'hl': {}, - 'mito_cn': {}, - 'contamination': {}, -} -MITO_GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -MITO_GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SHARED_QUALITY_FIELDS.keys()}) -SV_GENOTYPE_FIELDS_CONFIG = { - 'cn': {'format_value': int, 'default_value': -1}, - 'end': {}, - 'start': {}, - 'num_exon': {}, - 'geneIds': {'response_key': 'geneIds', 'format_value': list}, - 'defragged': {'format_value': bool}, - 'prev_call': {'format_value': bool}, - 'prev_overlap': {'format_value': bool}, - 'new_call': {'format_value': bool}, - 'prev_num_alt': {'format_value': lambda i: None if i is None else int(i)}, -} -SV_GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -SV_GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SV_QUALITY_FIELDS.keys()}) - -GENOTYPE_FIELDS = { - Sample.DATASET_TYPE_VARIANT_CALLS: GENOTYPE_FIELDS_CONFIG, - Sample.DATASET_TYPE_SV_CALLS: SV_GENOTYPE_FIELDS_CONFIG, - Sample.DATASET_TYPE_MITO_CALLS: MITO_GENOTYPE_FIELDS_CONFIG, -} - -QUERY_FIELD_NAMES = list(CORE_FIELDS_CONFIG.keys()) + list(PREDICTION_FIELDS_CONFIG.keys()) + \ - [field for fields in MULTI_FIELD_PREDICTORS.values() for field in fields] + \ - [SORTED_TRANSCRIPTS_FIELD_KEY, GENOTYPES_FIELD_KEY, GRCH38_LOCUS_FIELD] + HAS_ALT_FIELD_KEYS -for field_name, fields in NESTED_FIELDS.items(): - QUERY_FIELD_NAMES += ['{}_{}'.format(field_name, field) for field in fields.keys()] -for pop_config in POPULATIONS.values(): - for pop_field in pop_config.values(): - if isinstance(pop_field, list): - QUERY_FIELD_NAMES += pop_field - elif pop_field is not None: - QUERY_FIELD_NAMES.append(pop_field) - -SV_SAMPLE_OVERRIDE_FIELD_CONFIGS = { - 'pos': {'select_val': min, 'genotype_field': 'start'}, - 'end': {'select_val': max}, - 'numExon':{'select_val': max}, - 'geneIds': { - 'select_val': lambda gene_lists: set([gene_id for gene_list in gene_lists for gene_id in (gene_list or [])]), - 'equal': lambda a, b: set(a or []) == set(b or []) - }, -} diff --git a/seqr/utils/search/elasticsearch/es_gene_agg_search.py b/seqr/utils/search/elasticsearch/es_gene_agg_search.py deleted file mode 100644 index d2b04fc96a..0000000000 --- a/seqr/utils/search/elasticsearch/es_gene_agg_search.py +++ /dev/null @@ -1,106 +0,0 @@ -from collections import defaultdict - -from seqr.utils.search.elasticsearch.constants import MAX_COMPOUND_HET_GENES, HAS_ALT_FIELD_KEYS -from seqr.utils.search.elasticsearch.es_search import EsSearch - - -class EsGeneAggSearch(EsSearch): - AGGREGATION_NAME = 'gene aggregation' - CACHED_COUNTS_KEY = None - - def search(self, *args, **kwargs): - self._aggregate_by_gene() - return super(EsGeneAggSearch, self).search(*args, **kwargs) - - def _aggregate_by_gene(self): - searches = [self._search] - for index_searches in self._index_searches.values(): - searches += [index_search for index_search in index_searches] - - for search in searches: - agg = search.aggs.bucket( - 'genes', 'terms', field='mainTranscript_gene_id', size=MAX_COMPOUND_HET_GENES+1 - ) - if self._no_sample_filters or self._any_affected_sample_filters: - for key in HAS_ALT_FIELD_KEYS: - agg.bucket(key, 'terms', field=key, size=10000) - else: - agg.metric( - 'vars_by_gene', 'top_hits', size=100, _source='none' - ) - - def _should_execute_single_search(self, page=1, num_results=100): - return self._is_single_search(), {} - - def _process_single_search_response(self, parsed_response, page=1, num_results=100, deduplicate=False, **kwargs): - gene_aggs = {gene_id: {k: counts[k] for k in ['total', 'families']} for gene_id, counts in parsed_response.items()} - self._add_compound_hets(gene_aggs) - - self.previous_search_results['gene_aggs'] = gene_aggs - - return gene_aggs - - def _process_multi_search_responses(self, parsed_responses, page=1, num_results=100): - gene_aggs = parsed_responses[0] if parsed_responses else {} - for response in parsed_responses[1:]: - for gene_id, count_details in response.items(): - gene_aggs[gene_id]['sample_ids'].update(count_details['sample_ids']) - gene_aggs[gene_id]['families'].update(count_details['families']) - - gene_aggs = { - gene_id: {'total': len(counts['sample_ids']), 'families': counts['families']} - for gene_id, counts in gene_aggs.items() - } - - self._add_compound_hets(gene_aggs) - self.previous_search_results['gene_aggs'] = gene_aggs - - return gene_aggs - - def _parse_response(self, response): - if len(response.aggregations.genes.buckets) > MAX_COMPOUND_HET_GENES: - from seqr.utils.search.utils import InvalidSearchException - raise InvalidSearchException('This search returned too many genes') - - gene_counts = defaultdict(lambda: {'total': 0, 'families': defaultdict(int), 'sample_ids': set()}) - for gene_agg in response.aggregations.genes.buckets: - gene_id = gene_agg['key'] - gene_counts[gene_id]['total'] += gene_agg['doc_count'] - if 'vars_by_gene' in gene_agg: - for hit in gene_agg['vars_by_gene']: - gene_counts[gene_id]['sample_ids'].add(hit.meta.id) - for family_guid in hit.meta.matched_queries: - gene_counts[gene_id]['families'][family_guid] += 1 - else: - families_by_sample = {} - for index_samples_by_family in self.samples_by_family_index.values(): - for family_guid, samples_by_id in index_samples_by_family.items(): - for sample_id in samples_by_id.keys(): - families_by_sample[sample_id] = family_guid - - for key in HAS_ALT_FIELD_KEYS: - for sample_agg in gene_agg[key]['buckets']: - family_guid = families_by_sample.get(sample_agg['key']) - if family_guid: - gene_counts[gene_id]['families'][family_guid] += sample_agg['doc_count'] - gene_counts[gene_id]['sample_ids'].add(sample_agg['key']) - else: - # samples may be returned that are not part of the searched families if they have no - # affected individuals and were removed from the "any affected" search. - gene_counts[gene_id]['total'] -= sample_agg['doc_count'] - - return gene_counts - - def _add_compound_hets(self, gene_counts): - # Compound hets are always loaded as part of the initial search and are not part of the fetched aggregation - loaded_compound_hets = self.previous_search_results.get('grouped_results', []) + \ - self.previous_search_results.get('compound_het_results', []) - for group in loaded_compound_hets: - variants = next(iter(group.values())) - gene_id = next(iter(group)) - if gene_id and gene_id != 'null': - if gene_id not in gene_counts: - gene_counts[gene_id] = {'total': 0, 'families': defaultdict(int)} - gene_counts[gene_id]['total'] += len(variants) - for family_guid in variants[0]['familyGuids']: - gene_counts[gene_id]['families'][family_guid] += len(variants) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py deleted file mode 100644 index 47adfd192a..0000000000 --- a/seqr/utils/search/elasticsearch/es_search.py +++ /dev/null @@ -1,1581 +0,0 @@ -from collections import defaultdict -from copy import deepcopy -import elasticsearch -from elasticsearch_dsl import Search, Q, MultiSearch -import hashlib -import json -from sys import maxsize -from itertools import combinations - -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Individual -from seqr.utils.search.constants import XPOS_SORT_KEY, COMPOUND_HET, RECESSIVE, NEW_SV_FIELD, ALL_DATA_TYPES, X_LINKED_RECESSIVE, MAX_VARIANTS, \ - INHERITANCE_FILTERS, REF_REF, ANY_AFFECTED, AFFECTED, UNAFFECTED, HAS_ALT -from seqr.utils.search.elasticsearch.constants import \ - HAS_ALT_FIELD_KEYS, GENOTYPES_FIELD_KEY, POPULATION_RESPONSE_FIELD_CONFIGS, POPULATIONS, \ - SORTED_TRANSCRIPTS_FIELD_KEY, CORE_FIELDS_CONFIG, NESTED_FIELDS, PREDICTION_FIELDS_RESPONSE_CONFIG, \ - QUERY_FIELD_NAMES, GENOTYPE_QUERY_MAP, HGMD_CLASS_MAP, \ - SORT_FIELDS, MAX_COMPOUND_HET_GENES, MAX_INDEX_NAME_LENGTH, QUALITY_QUERY_FIELDS, \ - GRCH38_LOCUS_FIELD, MAX_SEARCH_CLAUSES, SV_SAMPLE_OVERRIDE_FIELD_CONFIGS, \ - PREDICTION_FIELD_LOOKUP, MULTI_FIELD_PREDICTORS, SPLICE_AI_FIELD, CLINVAR_KEY, HGMD_KEY, CLINVAR_PATH_SIGNIFICANCES, \ - PATH_FREQ_OVERRIDE_CUTOFF, CANONICAL_TRANSCRIPT_FILTER, \ - get_prediction_response_key, XSTOP_FIELD, GENOTYPE_FIELDS, SCREEN_KEY, MAX_INDEX_SEARCHES, PREFILTER_SEARCH_SIZE -from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json -from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS, get_chrom_pos -from seqr.views.utils.json_utils import _to_camel_case - -logger = SeqrLogger(__name__) - -class EsSearch(object): - - AGGREGATION_NAME = 'compound het' - CACHED_COUNTS_KEY = 'loaded_variant_counts' - - def __init__(self, samples, genome_version, previous_search_results=None, return_all_queried_families=False, user=None, sort=None, skipped_samples=None): - from seqr.utils.search.utils import InvalidSearchException - from seqr.utils.search.elasticsearch.es_utils import get_es_client, InvalidIndexException - self._client = get_es_client() - - self.samples_by_family_index = defaultdict(lambda: defaultdict(dict)) - for s in samples.select_related('individual__family'): - self.samples_by_family_index[s.elasticsearch_index][s.individual.family.guid][s.sample_id] = s - - self._set_indices(sorted(list(self.samples_by_family_index.keys()))) - self._set_index_metadata() - - if len(self.samples_by_family_index) > len(self.index_metadata): - raise InvalidIndexException('Could not find expected indices: {}'.format( - ', '.join(sorted(set(self._indices) - set(self.index_metadata.keys()), reverse = True)) - )) - elif len(self.index_metadata) > len(self.samples_by_family_index): - # Some of the indices are an alias - self._update_alias_metadata() - - invalid_genome_indices = [ - f"{index} ({meta['genomeVersion']})" for index, meta in self.index_metadata.items() - if meta['genomeVersion'] != genome_version - ] - if invalid_genome_indices: - raise InvalidSearchException( - f'The following indices do not have the expected genome version {genome_version}: {", ".join(invalid_genome_indices)}') - self._genome_version = genome_version - self._skipped_samples = skipped_samples - - self.indices_by_dataset_type = defaultdict(list) - for index in self._indices: - dataset_type = self._get_index_dataset_type(index) - self.indices_by_dataset_type[dataset_type].append(index) - - self.previous_search_results = {} if previous_search_results is None else previous_search_results - self._return_all_queried_families = return_all_queried_families - self._user = user - - self._search = Search() - self._index_searches = defaultdict(list) - self._family_individual_affected_status = {} - self._allowed_consequences = None - self._allowed_consequences_secondary = None - self._consequence_overrides = {} - self._filtered_gene_ids = None - self._paired_index_comp_het = False - self._no_sample_filters = False - self._any_affected_sample_filters = False - - self._sort = deepcopy(SORT_FIELDS.get(sort, [])) if sort else None - if self._sort: - self._sort_variants(samples) - - @staticmethod - def _parse_xstop(result): - xstop = result.pop(XSTOP_FIELD, None) - if xstop: - end_chrom, end = get_chrom_pos(xstop) - if end_chrom != result['chrom'] or end != result['end']: - if result['svType'] == 'INS': - result['svSourceDetail'] = {'chrom': end_chrom} - else: - result.update({ - 'endChrom': end_chrom, - 'end': end, - }) - - def _get_index_dataset_type(self, index): - return self.get_index_metadata_dataset_type(self.index_metadata[index]) - - @staticmethod - def get_index_metadata_dataset_type(index_metadata): - data_type = index_metadata.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) - if data_type == 'VARIANTS': - data_type = Sample.DATASET_TYPE_VARIANT_CALLS - return data_type - - def _set_indices(self, indices): - self._indices = indices - self._set_index_name() - - def _set_index_name(self): - self.index_name = ','.join(sorted(self._indices)) - if len(self.index_name) > MAX_INDEX_NAME_LENGTH: - alias = hashlib.md5(self.index_name.encode('utf-8')).hexdigest() # nosec - cache_key = 'index_alias__{}'.format(alias) - if safe_redis_get_json(cache_key) != self.index_name: - self._client.indices.update_aliases(body={'actions': [ - {'add': {'indices': self._indices, 'alias': alias}} - ]}) - safe_redis_set_json(cache_key, self.index_name) - self.index_name = alias - - def _set_index_metadata(self): - from seqr.utils.search.elasticsearch.es_utils import get_index_metadata - self.index_metadata = get_index_metadata(self.index_name, self._client, include_fields=True) - - def _update_alias_metadata(self): - additional_meta_indices = set(self.index_metadata.keys()) - set(self._indices) - aliases = [ind for ind in self._indices if ind not in self.index_metadata] - alias_map = defaultdict(list) - if len(aliases) == 1: - alias_map[aliases[0]] = additional_meta_indices - else: - for index, index_aliases in self._client.indices.get_alias(index=','.join(aliases)).items(): - for alias in index_aliases['aliases']: - alias_map[alias].append(index) - self._indices = list(self.index_metadata.keys()) - for alias, alias_indices in alias_map.items(): - alias_samples = self.samples_by_family_index.pop(alias, {}) - for alias_index in alias_indices: - if not self.samples_by_family_index[alias_index]: - self.samples_by_family_index[alias_index] = {} - self.samples_by_family_index[alias_index].update(alias_samples) - - def _set_family_affected_status(self, inheritance_filter): - for family_samples in list(self.samples_by_family_index.values()): - for family_guid, samples_by_id in family_samples.items(): - individual_affected_status = _get_family_affected_status(samples_by_id, inheritance_filter) - if family_guid not in self._family_individual_affected_status: - self._family_individual_affected_status[family_guid] = {} - self._family_individual_affected_status[family_guid].update(individual_affected_status) - - def _dataset_type_indices(self, dataset_type): - new_indices = self.indices_by_dataset_type[dataset_type] - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - new_indices += self.indices_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS] - return new_indices - - def _sort_variants(self, sample_data): - main_sort_dict = self._sort[0] if len(self._sort) and isinstance(self._sort[0], dict) else None - - # Add parameters to scripts - if main_sort_dict and main_sort_dict.get('_script', {}).get('script', {}).get('params'): - called_params = None - for key, val_func in self._sort[0]['_script']['script']['params'].items(): - if callable(val_func): - self._sort[0]['_script']['script']['params'][key] = val_func(sample_data) - called_params = self._sort[0]['_script']['script']['params'] - if called_params: - for sort_dict in self._sort[1:]: - sort_dict['_script']['script']['params'] = called_params - - - # Add unmapped_type - if main_sort_dict and 'unmapped_type' in list(main_sort_dict.values())[0]: - sort_field = list(main_sort_dict.keys())[0] - field_type = next(( - metadata['fields'][sort_field] for metadata in self.index_metadata.values() - if metadata['fields'].get(sort_field) - ), 'double') - if field_type == 'keyword': - self._sort[0][sort_field]['unmapped_type'] = field_type - self._sort[0][sort_field].pop('numeric_type') - - if XPOS_SORT_KEY not in self._sort: - self._sort.append(XPOS_SORT_KEY) - - # always final sort on variant ID to keep different variants at the same position grouped properly - if 'variantId' not in self._sort: - self._sort.append('variantId') - - self._search = self._search.sort(*self._sort) - - def _filter(self, new_filter): - self._search = self._search.filter(new_filter) - return self - - def filter_variants(self, inheritance_mode=None, inheritance_filter=None, genes=None, intervals=None, rs_ids=None, variant_ids=None, exclude_locations=False, - frequencies=None, pathogenicity=None, in_silico=None, annotations=None, annotations_secondary=None, - quality_filter=None, custom_query=None, skip_genotype_filter=False, dataset_type=None, secondary_dataset_type=None, **kwargs): - - self._filter_custom(custom_query) - - self._filter_by_location(genes, intervals, variant_ids, rs_ids, exclude_locations) - - self._parse_annotation_overrides(annotations, pathogenicity) - - self._filter_by_frequency(frequencies) - - self._filter_by_in_silico(in_silico) - - if quality_filter and quality_filter.get('vcf_filter') is not None: - self._filter(~Q('exists', field='filters')) - - inheritance_filter = inheritance_filter or {} - - quality_affected_only = quality_filter and quality_filter.get('affected_only') - if inheritance_mode or inheritance_filter or quality_affected_only: - self._set_family_affected_status(inheritance_filter) - - quality_filters_by_family = self._get_quality_filters_by_family(quality_filter, quality_affected_only) - - comp_het_dataset_type = dataset_type - if dataset_type != secondary_dataset_type: - comp_het_dataset_type = None - if dataset_type == ALL_DATA_TYPES: - dataset_type = None - comp_het_dataset_type = None - - has_comp_het_search = inheritance_mode in {RECESSIVE, COMPOUND_HET} and not self.previous_search_results.get('grouped_results') - if has_comp_het_search: - self._filter_compound_hets(quality_filters_by_family, annotations_secondary, comp_het_dataset_type) - if inheritance_mode == COMPOUND_HET: - return - - self._filter_by_annotations() - if skip_genotype_filter and not inheritance_mode: - return - - self._filter_by_genotype(inheritance_mode, inheritance_filter, quality_filters_by_family, dataset_type) - - def _parse_annotation_overrides(self, annotations, pathogenicity): - clinvar_terms, hgmd_classes = _parse_pathogenicity_filter(pathogenicity or {}) - - annotations = {k: v for k, v in (annotations or {}).items() if v} - - # Temporary code, remove it after all the SV indices are reloaded - if annotations.get('structural_consequence'): - if 'MSV_EXON_OVERLAP' in annotations['structural_consequence']: - annotations['structural_consequence'].append('MSV_EXON_OVR') - if 'INTRAGENIC_EXON_DUP' in annotations['structural_consequence']: - annotations['structural_consequence'].append('DUP_LOF') - - new_svs = bool(annotations.pop(NEW_SV_FIELD, False)) - splice_ai = annotations.pop(SPLICE_AI_FIELD, None) - screen = annotations.pop(SCREEN_KEY, None) - self._allowed_consequences = sorted({ann for anns in annotations.values() for ann in anns}) - if clinvar_terms: - self._consequence_overrides[CLINVAR_KEY] = clinvar_terms - if hgmd_classes: - self._consequence_overrides[HGMD_KEY] = hgmd_classes - if splice_ai: - self._consequence_overrides[SPLICE_AI_FIELD] = float(splice_ai) - if screen: - self._consequence_overrides[SCREEN_KEY] = screen - if new_svs: - self._consequence_overrides[NEW_SV_FIELD] = new_svs - - def _filter_by_location(self, genes, intervals, variant_ids, rs_ids, exclude_locations): - if genes or intervals: - self._filter(_location_filter(genes, intervals, exclude_locations)) - if genes and not exclude_locations: - self._filtered_gene_ids = set(genes.keys()) - elif variant_ids: - self.filter_by_variant_ids(variant_ids) - elif rs_ids: - self._filter(Q('terms', rsid=rs_ids)) - - def _filter_custom(self, custom_query): - if custom_query: - if not isinstance(custom_query, list): - custom_query = [custom_query] - for q_dict in custom_query: - self._filter(Q(q_dict)) - - def _filter_by_in_silico(self, in_silico_filters): - in_silico_filters = deepcopy(in_silico_filters or {}) - require_score = in_silico_filters.pop('requireScore', False) - in_silico_filters = {k: v for k, v in in_silico_filters.items() if v is not None and len(v) != 0} - if in_silico_filters: - self._filter(self._in_silico_filter(in_silico_filters, require_score=require_score)) - - def _in_silico_filter(self, in_silico_filters, require_score): - prediction_key_lookup = { - in_silico_filter: PREDICTION_FIELD_LOOKUP.get(in_silico_filter.lower(), in_silico_filter) - for in_silico_filter in in_silico_filters.keys() - } - - multi_predictor_fields = { - in_silico_filter: MULTI_FIELD_PREDICTORS[in_silico_filter.lower()] - for in_silico_filter in in_silico_filters.keys() if in_silico_filter.lower() in MULTI_FIELD_PREDICTORS - } - if multi_predictor_fields: - all_fields = set() - for metadata in self.index_metadata.values(): - all_fields.update(metadata['fields']) - index_fields = { - k: next(field for field in fields if field in all_fields) - for k, fields in multi_predictor_fields.items() - } - prediction_key_lookup.update(index_fields) - - return _in_silico_filter({prediction_key_lookup[k]: v for k, v in in_silico_filters.items()}, require_score) - - def _filter_by_frequency(self, frequencies): - frequencies = {pop: v for pop, v in (frequencies or {}).items() if pop in POPULATIONS} - if not frequencies: - return - - path_filter = self._get_clinvar_pathogenic_override_filter() - path_override = path_filter is not None and any( - freqs.get('af') or 1 < PATH_FREQ_OVERRIDE_CUTOFF for freqs in frequencies.values()) - - q = Q() - path_q = Q() - for pop, freqs in sorted(frequencies.items()): - if freqs.get('af') is not None: - filter_field = next( - (field_key for field_key in POPULATIONS[pop]['filter_AF'] - if any(field_key in index_metadata['fields'] for index_metadata in self.index_metadata.values())), - POPULATIONS[pop]['AF']) - q &= _pop_freq_filter(filter_field, freqs['af']) - if path_override: - path_q &= _pop_freq_filter(filter_field, max(freqs['af'], PATH_FREQ_OVERRIDE_CUTOFF)) - elif freqs.get('ac') is not None: - q &= _pop_freq_filter(POPULATIONS[pop]['AC'], freqs['ac']) - - if freqs.get('hh') is not None: - q &= _pop_freq_filter(POPULATIONS[pop]['Hom'], freqs['hh']) - q &= _pop_freq_filter(POPULATIONS[pop]['Hemi'], freqs['hh']) - - if path_override: - q |= (path_filter & path_q) - - self._filter(q) - - def _get_clinvar_pathogenic_override_filter(self): - clinvar_path_terms = [ - f for f in self._consequence_overrides.get(CLINVAR_KEY, []) if f in CLINVAR_PATH_SIGNIFICANCES - ] - if clinvar_path_terms: - return _pathogenicity_filter(clinvar_path_terms) - return None - - def _get_annotation_override_filter(self): - filters = [] - pathogenicity_filter = _pathogenicity_filter( - self._consequence_overrides.get(CLINVAR_KEY), self._consequence_overrides.get(HGMD_KEY), - ) - if pathogenicity_filter: - filters.append(pathogenicity_filter) - splice_ai = self._consequence_overrides.get(SPLICE_AI_FIELD) - if splice_ai: - filters.append(self._in_silico_filter({SPLICE_AI_FIELD: splice_ai}, require_score=True)) - screen = self._consequence_overrides.get(SCREEN_KEY) - if screen: - filters.append(Q('terms', screen_region_type=screen)) - - if not filters: - return None - return _or_filters(filters) - - def _filter_by_annotations(self): - annotation_override_filter = self._get_annotation_override_filter() - new_svs = self._consequence_overrides.get(NEW_SV_FIELD) - - if self._allowed_consequences: - consequences_filter = _annotations_filter(self._allowed_consequences) - - if annotation_override_filter: - # Pathogencicity and transcript consequences act as "OR" filters instead of the usual "AND" - consequences_filter |= annotation_override_filter - self._filter(consequences_filter) - elif annotation_override_filter and not new_svs: - self._filter(annotation_override_filter) - - def filter_by_variant_ids(self, variant_ids): - self._filter(Q('terms', variantId=variant_ids)) - return self - - def _filter_by_genotype(self, inheritance_mode, inheritance_filter, quality_filters_by_family, dataset_type): - has_inheritance_filter = inheritance_filter or inheritance_mode - all_sample_search = (not quality_filters_by_family) and (inheritance_mode == ANY_AFFECTED or not has_inheritance_filter) - no_filter_indices = set() - - indices = self._dataset_type_indices(dataset_type) if dataset_type else self._indices - for index in indices: - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - - if all_sample_search: - index_skipped_sample_count = sum([s.elasticsearch_index == index for s in self._skipped_samples or []]) - search_sample_count = sum(len(samples) for samples in family_samples_by_id.values()) + index_skipped_sample_count - index_sample_count = Sample.objects.filter(elasticsearch_index=index, is_active=True).count() - if search_sample_count == index_sample_count: - if inheritance_mode == ANY_AFFECTED: - sample_ids = [] - for family_guid, samples_by_id in family_samples_by_id.items(): - sample_ids += [ - sample_id for sample_id, sample in samples_by_id.items() - if self._family_individual_affected_status[family_guid][sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED] - self._index_searches[index].append(self._search.filter(_any_affected_sample_filter(sample_ids))) - self._any_affected_sample_filters = True - else: - # If searching across all families in an index with no inheritance mode we do not need to explicitly - # filter on inheritance, as all variants have some inheritance for at least one family - self._no_sample_filters = True - no_filter_indices.add(index) - continue - - genotypes_qs = [ - self._get_family_sample_query( - family_guid, family_samples_by_id, quality_filters_by_family, - index_fields, inheritance_mode, inheritance_filter - ) for family_guid in sorted(family_samples_by_id.keys()) - ] - if len(genotypes_qs) > MAX_SEARCH_CLAUSES: - self._index_searches[index].append(self._search.filter(_or_filters(genotypes_qs[:MAX_SEARCH_CLAUSES]))) - genotypes_qs = genotypes_qs[MAX_SEARCH_CLAUSES:] - - self._index_searches[index].append(self._search.filter(_or_filters(genotypes_qs))) - - if no_filter_indices and self._index_searches: - for index in no_filter_indices: - self._index_searches[index].append(self._search) - - def _get_family_sample_query(self, family_guid, family_samples_by_id, quality_filters_by_family, index_fields, inheritance_mode, inheritance_filter): - samples_by_id = family_samples_by_id[family_guid] - affected_status = self._family_individual_affected_status.get(family_guid) - - # Filter samples by inheritance - if inheritance_mode == ANY_AFFECTED: - # Only return variants where at least one of the affected samples has an alt allele - sample_ids = [sample_id for sample_id, sample in samples_by_id.items() - if affected_status[sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED] - family_samples_q = _any_affected_sample_filter(sample_ids) - elif inheritance_filter or inheritance_mode: - if inheritance_mode: - inheritance_filter.update(INHERITANCE_FILTERS[inheritance_mode]) - - family_samples_q = _family_genotype_inheritance_filter( - inheritance_mode, inheritance_filter, samples_by_id, affected_status, index_fields, - ) - else: - # If no inheritance specified only return variants where at least one of the requested samples has an alt allele - family_samples_q = _any_affected_sample_filter(list(samples_by_id.keys())) - - return _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - - def _filter_compound_hets(self, quality_filters_by_family, annotations_secondary, dataset_type): - comp_het_consequences = set(self._allowed_consequences) - annotation_override_filter = self._get_annotation_override_filter() - if annotations_secondary: - self._allowed_consequences_secondary = sorted({ann for anns in annotations_secondary.values() for ann in anns}) - comp_het_consequences.update(self._allowed_consequences_secondary) - - annotation_filter = _annotations_filter(comp_het_consequences) - if annotation_override_filter: - annotation_filter |= annotation_override_filter - comp_het_search = self._search.filter(annotation_filter) - - comp_het_qs_by_index = defaultdict(list) - if dataset_type or len(self._indices) <= 1: - indices = self.indices_by_dataset_type[dataset_type] if dataset_type else self._indices - for index in sorted(indices, reverse=True): - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - - for family_guid, samples_by_id in sorted(family_samples_by_id.items()): - affected_status = self._family_individual_affected_status[family_guid] - family_samples_q = _family_genotype_inheritance_filter( - COMPOUND_HET, INHERITANCE_FILTERS[COMPOUND_HET], samples_by_id, affected_status, index_fields, - ) - samples_q = _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - comp_het_qs_by_index[index].append(samples_q) - else: - self._get_paired_indices_comp_het_queries(comp_het_qs_by_index, quality_filters_by_family) - - for index, compound_het_qs in comp_het_qs_by_index.items(): - comp_het_qs_list = [ - compound_het_qs[:MAX_SEARCH_CLAUSES], compound_het_qs[MAX_SEARCH_CLAUSES:] - ] if len(compound_het_qs) > MAX_SEARCH_CLAUSES else [compound_het_qs] - for compound_het_q in comp_het_qs_list: - compound_het_search = comp_het_search.filter(_or_filters(compound_het_q)) - compound_het_search.aggs.bucket( - 'genes', 'terms', field='geneIds', min_doc_count=2, size=MAX_COMPOUND_HET_GENES + 1 - ).metric( - 'vars_by_gene', 'top_hits', size=100, sort=self._sort, _source=QUERY_FIELD_NAMES - ) - self._index_searches[index].append(compound_het_search) - - def _get_paired_indices_comp_het_queries(self, comp_het_qs_by_index, quality_filters_by_family): - paired_index_families = defaultdict(dict) - sv_indices = self.indices_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS] - variant_indices = self.indices_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] - if sv_indices and variant_indices: - for sv_index in sv_indices: - sv_families = set(self.samples_by_family_index[sv_index].keys()) - for var_index in variant_indices: - overlapping_families = sv_families & set(self.samples_by_family_index[var_index].keys()) - if overlapping_families: - paired_index_families[sv_index].update({var_index: overlapping_families}) - paired_index_families[var_index].update({sv_index: overlapping_families}) - - seen_paired_indices = set() - for index in sorted(self._indices, reverse=True): - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - seen_paired_indices.add(index) - - paired_families = {} - for pair_index, families in paired_index_families[index].items(): - paired_families.update({family: pair_index for family in families}) - - for family_guid in sorted(family_samples_by_id.keys()): - paired_index = paired_families.get(family_guid) - if paired_index and paired_index in seen_paired_indices: - continue - - samples_by_id = family_samples_by_id[family_guid] - - affected_status = self._family_individual_affected_status[family_guid] - inheritance_filters = self._comp_het_inheritance_filter(index, paired_index) - family_samples_q = _family_genotype_inheritance_filter( - COMPOUND_HET, inheritance_filters, samples_by_id, affected_status, index_fields, - ) - - family_index = index - if paired_index: - self._paired_index_comp_het = True - pair_index_fields = self.index_metadata[paired_index]['fields'] - pair_samples_by_id = self.samples_by_family_index[paired_index][family_guid] - pair_inheritance_filters = self._comp_het_inheritance_filter(paired_index, True) - family_samples_q |= _family_genotype_inheritance_filter( - COMPOUND_HET, pair_inheritance_filters, pair_samples_by_id, affected_status, pair_index_fields, - ) - family_index = ','.join(sorted([index, paired_index])) - - samples_q = _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - comp_het_qs_by_index[family_index].append(samples_q) - - def _comp_het_inheritance_filter(self, index, has_paired_index): - if has_paired_index and self._get_index_dataset_type(index) == Sample.DATASET_TYPE_VARIANT_CALLS: - # SNPs in trans with deletions may be called as hom alt instead of ref alt - return { - AFFECTED: HAS_ALT, - UNAFFECTED: INHERITANCE_FILTERS[COMPOUND_HET][UNAFFECTED], - } - return INHERITANCE_FILTERS[COMPOUND_HET] - - def search(self, page=1, num_results=100): - indices = self._indices - - logger.info('Searching in elasticsearch indices: {}'.format(', '.join(indices)), self._user) - - is_single_search, search_kwargs = self._should_execute_single_search(page=page, num_results=num_results) - - if is_single_search: - return self._execute_single_search(**search_kwargs) - elif not self._index_searches: - return self._execute_single_search(**search_kwargs) - else: - return self._execute_multi_search(**search_kwargs) - - def _is_single_search(self): - return len(self._indices) == 1 and len(self._index_searches) < 2 and \ - len(self._index_searches.get(self._indices[0], [])) <= 1 - - def _should_execute_single_search(self, page=1, num_results=100): - is_single_search = self._is_single_search() - num_loaded = len(self.previous_search_results.get('all_results', [])) - - if is_single_search and not self.previous_search_results.get('grouped_results'): - start_index = None - if (page - 1) * num_results < num_loaded: - start_index = num_loaded - return True, {'page': page, 'num_results': num_results, 'start_index': start_index} - elif not self._index_searches: - # If doing all project-families all inheritance search, do it as a single query - # Load all variants, do not skip pages - num_loaded += self.previous_search_results.get('duplicate_doc_count', 0) - if num_loaded >= (page - 1) * num_results: - start_index = num_loaded - else: - start_index = 0 - return True, {'page': page, 'num_results': num_results, 'start_index': start_index, 'deduplicate': True} - else: - return False, {'page': page, 'num_results': num_results} - - def _execute_single_search(self, page=1, num_results=100, start_index=None, deduplicate=False, **kwargs): - num_results_for_search = num_results * len(self._indices) if deduplicate else num_results - if num_results_for_search > MAX_VARIANTS and deduplicate: - num_results_for_search = MAX_VARIANTS - searches, log_messages = self._get_paginated_searches( - self.index_name, page=page, num_results=num_results_for_search, start_index=start_index - ) - logger.info(log_messages[0], self._user) - search = searches[0] - response = self._execute_search(search) - parsed_response = self._parse_response(response) - return self._process_single_search_response( - parsed_response, page=page, num_results=num_results, deduplicate=deduplicate, **kwargs) - - def _process_single_search_response(self, parsed_response, page=1, num_results=100, deduplicate=False, **kwargs): - variant_results, total_results, is_compound_het, _ = parsed_response - self.previous_search_results['total_results'] = total_results - - results_start_index = (page - 1) * num_results - if is_compound_het: - variant_results = _sort_compound_hets(variant_results) - self.previous_search_results['grouped_results'] = variant_results - end_index = min(results_start_index + num_results, total_results) - return get_compound_het_page(variant_results, results_start_index, end_index) - - if deduplicate: - variant_results = self._deduplicate_results(variant_results) - - # Only save contiguous pages of results: - previous_all_results = self.previous_search_results.get('all_results', []) - if len(previous_all_results) >= results_start_index: - self.previous_search_results['all_results'] = self.previous_search_results.get('all_results', []) + variant_results - variant_results = self.previous_search_results['all_results'][results_start_index:] - - return variant_results[:num_results] - - def _execute_multi_search(self, **kwargs): - indices = sorted(self._index_searches.keys(), reverse = True) or self._indices - - if self.CACHED_COUNTS_KEY and not self.previous_search_results.get(self.CACHED_COUNTS_KEY): - self.previous_search_results[self.CACHED_COUNTS_KEY] = {} - - paginated_index_searches = {} - index_logs = {} - for index_name in indices: - start_index = 0 - if self.CACHED_COUNTS_KEY: - if self.previous_search_results[self.CACHED_COUNTS_KEY].get(index_name): - index_total = self.previous_search_results[self.CACHED_COUNTS_KEY][index_name]['total'] - start_index = self.previous_search_results[self.CACHED_COUNTS_KEY][index_name]['loaded'] - if start_index >= index_total: - continue - else: - self.previous_search_results[self.CACHED_COUNTS_KEY][index_name] = {'loaded': 0, 'total': 0} - - searches, log_messages = self._get_paginated_searches(index_name, start_index=start_index, **kwargs) - if searches: - paginated_index_searches[index_name] = searches - index_logs[index_name] = log_messages - - if len(paginated_index_searches) > MAX_INDEX_SEARCHES: - index_possible_variants, all_inheritance_response = self._get_possible_hit_indices() - if index_possible_variants: - paginated_index_searches = { - index_name: [ - search.query('ids', values=possible_ids) for search in paginated_index_searches[index_name] - ] for index_name, possible_ids in index_possible_variants.items() - if paginated_index_searches.get(index_name) - } - elif all_inheritance_response is not None: - # all inheritance search succeeded but has no results, return an empty response - return self._process_single_search_response(self._parse_response(all_inheritance_response), **kwargs) - - ms = MultiSearch() - for index_name, searches in paginated_index_searches.items(): - for search in searches: - ms = ms.add(search) - for message in index_logs[index_name]: - logger.info(message, self._user) - - responses = self._execute_search(ms) if ms._searches else [] - parsed_responses = [self._parse_response(response) for response in responses] - return self._process_multi_search_responses(parsed_responses, **kwargs) - - def _get_possible_hit_indices(self): - no_inheritance_search = self._search.index(self.index_name).source('')[:PREFILTER_SEARCH_SIZE] - response = no_inheritance_search.using(self._client).execute() - if response.hits.total['value'] > len(response.hits): - return None, None - - index_possible_variants = defaultdict(list) - for hit in response.hits: - index_possible_variants[hit.meta.index].append(hit.meta.id) - - logger.info(f'Filtering search to {len(index_possible_variants)} indices with possible hits', self._user) - return index_possible_variants, response - - def _process_multi_search_responses(self, parsed_responses, page=1, num_results=100): - new_results = [] - compound_het_results = self.previous_search_results.get('compound_het_results', []) - loaded_counts = defaultdict(lambda: defaultdict(int)) - for response_hits, response_total, is_compound_het, index_name in parsed_responses: - if not response_hits: - continue - - if is_compound_het: - compound_het_results += response_hits - loaded_count = response_total - index_name = '{}_compound_het'.format(index_name) - if not self.previous_search_results['loaded_variant_counts'].get(index_name): - self.previous_search_results['loaded_variant_counts'][index_name] = {'total': 0, 'loaded': 0} - else: - new_results += response_hits - loaded_count = len(response_hits) - - loaded_counts[index_name]['total'] += response_total - loaded_counts[index_name]['loaded'] += loaded_count - - for index_name, counts in loaded_counts.items(): - self.previous_search_results['loaded_variant_counts'][index_name]['total'] = counts['total'] - self.previous_search_results['loaded_variant_counts'][index_name]['loaded'] += counts['loaded'] - - total_results = sum( - counts['total'] for counts in self.previous_search_results['loaded_variant_counts'].values()) - self.previous_search_results['total_results'] = total_results - - # combine new results with unsorted previously loaded results to correctly sort/paginate - all_loaded_results = self.previous_search_results.get('all_results', []) - new_results += self.previous_search_results.get('variant_results', []) - - new_results = sorted(new_results, key=lambda variant: variant['_sort']) - variant_results = self._deduplicate_results(new_results) - - if compound_het_results or self.previous_search_results.get('grouped_results'): - if compound_het_results: - compound_het_results = self._deduplicate_compound_het_results(compound_het_results) - compound_het_results = _sort_compound_hets(compound_het_results) - loaded_results = sum( - counts['loaded'] for counts in self.previous_search_results['loaded_variant_counts'].values()) - return self._process_compound_hets( - compound_het_results, variant_results, num_results, all_loaded=loaded_results == total_results) - else: - end_index = num_results * page - num_loaded = num_results * page - len(all_loaded_results) - self.previous_search_results['all_results'] = all_loaded_results + variant_results[:num_loaded] - self.previous_search_results['variant_results'] = variant_results[num_loaded:] - return self.previous_search_results['all_results'][end_index-num_results:end_index] - - def _parse_response(self, response): - index_name = response.hits[0].meta.index if response.hits else None - if hasattr(response.aggregations, 'genes') and response.hits: - response_hits, response_total = self._parse_compound_het_response(response) - return response_hits, response_total, True, index_name - - response_total = response.hits.total['value'] - logger.info('Total hits: {} ({} seconds)'.format(response_total, response.took / 1000.0), self._user) - return [self._parse_hit(hit) for hit in response], response_total, False, index_name - - - def _parse_hit(self, raw_hit): - hit = {k: raw_hit[k] for k in QUERY_FIELD_NAMES if k in raw_hit} - index_name = raw_hit.meta.index - index_family_samples = self.samples_by_family_index[index_name] - data_type = self._get_index_dataset_type(index_name) - - family_guids, genotypes = self._parse_genotypes(raw_hit, hit, index_family_samples, data_type) - - result = _get_field_values(hit, CORE_FIELDS_CONFIG, format_response_key=str) - result.update({ - field_name: _get_field_values(hit, fields, lookup_field_prefix=field_name) - for field_name, fields in NESTED_FIELDS.items() - }) - if hasattr(raw_hit.meta, 'sort'): - result['_sort'] = [_parse_es_sort(sort, self._sort[i]) for i, sort in enumerate(raw_hit.meta.sort)] - - result['genomeVersion'] = self._genome_version - if self._genome_version == GENOME_VERSION_GRCh38: - self._add_liftover(result, hit) - self._parse_xstop(result) - result[CLINVAR_KEY]['version'] = self.index_metadata[index_name].get('clinvar_version') - - # If an SV has genotype-specific coordinates that differ from the main coordinates, use those - if data_type == Sample.DATASET_TYPE_SV_CALLS and genotypes: - self._set_sv_genotype_coords(genotypes, result) - - populations = { - population: _get_field_values( - hit, POPULATION_RESPONSE_FIELD_CONFIGS, format_response_key=lambda key: key.lower(), - lookup_field_prefix=population, - existing_fields=self.index_metadata[index_name]['fields'], - get_addl_fields=lambda field: pop_config[field] if isinstance(pop_config[field], list) else [pop_config[field]], - skip_fields=[field for field, val in pop_config.items() if val is None], - ) - for population, pop_config in POPULATIONS.items() - } - - sorted_transcripts = [ - {_to_camel_case(k): v for k, v in transcript.to_dict().items()} - for transcript in hit[SORTED_TRANSCRIPTS_FIELD_KEY] or [] - ] - - # Temporary code, remove it after all the SV indices are reloaded - for trans in sorted_transcripts: - if trans.get('majorConsequence'): - if trans['majorConsequence'] == 'MSV_EXON_OVR': - trans['majorConsequence'] = 'MSV_EXON_OVERLAP' - elif trans['majorConsequence'] == 'DUP_LOF': - trans['majorConsequence'] = 'INTRAGENIC_EXON_DUP' - - transcripts = defaultdict(list) - for transcript in sorted_transcripts: - if transcript['geneId']: - transcripts[transcript['geneId']].append(transcript) - gene_ids = result.pop('geneIds', None) - if gene_ids: - transcripts = {gene_id: ts for gene_id, ts in transcripts.items() if gene_id in gene_ids} - main_transcript_id, selected_main_transcript_id = self._get_main_transcript(sorted_transcripts) - - result.update({ - 'familyGuids': sorted(family_guids), - 'genotypes': genotypes, - 'mainTranscriptId': main_transcript_id, - 'selectedMainTranscriptId': selected_main_transcript_id, - 'populations': populations, - 'predictions': _get_field_values( - hit, PREDICTION_FIELDS_RESPONSE_CONFIG, format_response_key=get_prediction_response_key, - get_addl_fields=lambda field: MULTI_FIELD_PREDICTORS.get(field, []) - ), - 'transcripts': dict(transcripts), - }) - return result - - def _parse_genotypes(self, raw_hit, hit, index_family_samples, data_type): - if hasattr(raw_hit.meta, 'matched_queries'): - family_guids = list(raw_hit.meta.matched_queries) - elif self._return_all_queried_families: - family_guids = list(index_family_samples.keys()) - else: - # Searches for all inheritance and all families do not filter on inheritance so there are no matched_queries - alt_allele_samples = set() - for alt_samples_field in HAS_ALT_FIELD_KEYS: - if alt_samples_field in hit: - alt_allele_samples.update(hit[alt_samples_field]) - - if self._any_affected_sample_filters: - # If using the any inheritance filter only include matched families - def _is_matched_sample(family_guid, sample): - return self._family_individual_affected_status[family_guid][sample.individual.guid] == \ - Individual.AFFECTED_STATUS_AFFECTED - else: - _is_matched_sample = lambda *args: True - - family_guids = [family_guid for family_guid, samples_by_id in index_family_samples.items() - if any(sample_id in alt_allele_samples and _is_matched_sample(family_guid, sample) - for sample_id, sample in samples_by_id.items())] - - genotypes = {} - genotype_fields_config = GENOTYPE_FIELDS[data_type] - for family_guid in family_guids: - samples_by_id = index_family_samples[family_guid] - for genotype_hit in hit[GENOTYPES_FIELD_KEY]: - sample = samples_by_id.get(genotype_hit['sample_id']) - if sample: - genotype_hit['sample_type'] = sample.sample_type - genotypes[sample.individual.guid] = _get_field_values(genotype_hit, genotype_fields_config) - - if len(samples_by_id) != len(genotypes) and data_type == Sample.DATASET_TYPE_SV_CALLS: - # Family members with no variants are not included in the SV index - for sample_id, sample in samples_by_id.items(): - if sample.individual.guid not in genotypes: - genotypes[sample.individual.guid] = _get_field_values( - {'sample_id': sample_id}, genotype_fields_config) - genotypes[sample.individual.guid]['isRef'] = True - genotypes[sample.individual.guid]['cn'] = \ - 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 - - return family_guids, genotypes - - @classmethod - def _set_sv_genotype_coords(cls, genotypes, result): - # If an SV has genotype-specific coordinates that differ from the main coordinates, use those - if any(not gen.get('isRef') for gen in genotypes.values()) and all( - (gen.get('isRef') or gen.get('start') or gen.get('end')) for gen in genotypes.values()): - for field, conf in SV_SAMPLE_OVERRIDE_FIELD_CONFIGS.items(): - gen_field = conf.get('genotype_field', field) - val = conf['select_val']([ - gen.get(gen_field) or result.get(field) for gen in genotypes.values() if not gen.get('isRef') - ]) - if val != result.get(field): - result[field] = val - if field == 'pos': - result['xpos'] = get_xpos(result['chrom'], val) - - for gen in genotypes.values(): - for field, conf in SV_SAMPLE_OVERRIDE_FIELD_CONFIGS.items(): - gen_field = conf.get('genotype_field', field) - compare_func = conf.get('equal') or (lambda a, b: a == b) - if compare_func(gen.get(gen_field), result.get(field)): - gen[gen_field] = None - - def _get_main_transcript(self, sorted_transcripts): - main_transcript_id = sorted_transcripts[0]['transcriptId'] \ - if len(sorted_transcripts) and 'transcriptRank' in sorted_transcripts[0] else None - - selected_main_transcript_id = None - if main_transcript_id and (self._filtered_gene_ids or self._allowed_consequences): - gene_transcripts = [ - t for t in sorted_transcripts if t.get('geneId') in self._filtered_gene_ids - ] if self._filtered_gene_ids else sorted_transcripts - - selected_main_transcript_id = gene_transcripts[0].get('transcriptId') - if self._allowed_consequences: - consequence_transcript_id = next(( - t.get('transcriptId') for t in gene_transcripts if - self._is_matched_transcript(t, self._allowed_consequences)), None) - if not consequence_transcript_id and self._allowed_consequences_secondary: - consequence_transcript_id = next(( - t.get('transcriptId') for t in gene_transcripts if self._is_matched_transcript(t, self._allowed_consequences_secondary) - ), None) - selected_main_transcript_id = consequence_transcript_id or selected_main_transcript_id - if selected_main_transcript_id == main_transcript_id: - selected_main_transcript_id = None - - return main_transcript_id, selected_main_transcript_id - - @staticmethod - def _is_matched_transcript(t, allowed_consequences): - is_match = t.get('majorConsequence') in allowed_consequences - if CANONICAL_TRANSCRIPT_FILTER in allowed_consequences and t.get('canonical') and \ - t.get('majorConsequence') == 'non_coding_transcript_exon_variant': - is_match = True - return is_match - - @staticmethod - def _add_liftover(result, hit): - lifted_over_genome_version = None - lifted_over_chrom = None - lifted_over_pos = None - grch37_locus = hit.get(GRCH38_LOCUS_FIELD, None) - if grch37_locus: - lifted_over_genome_version = GENOME_VERSION_GRCh37 - lifted_over_chrom = grch37_locus['contig'] - lifted_over_pos = grch37_locus['position'] - else: - from seqr.utils.search.utils import run_liftover - grch37_coords = run_liftover(GENOME_VERSION_GRCh37, hit['contig'], hit['start']) - if grch37_coords: - lifted_over_genome_version = GENOME_VERSION_GRCh37 - lifted_over_chrom, lifted_over_pos = grch37_coords - result.update({ - 'liftedOverGenomeVersion': lifted_over_genome_version, - 'liftedOverChrom': lifted_over_chrom, - 'liftedOverPos': lifted_over_pos, - }) - - def _parse_compound_het_response(self, response): - if len(response.aggregations.genes.buckets) > MAX_COMPOUND_HET_GENES: - from seqr.utils.search.utils import InvalidSearchException - raise InvalidSearchException('This search returned too many compound heterozygous variants. Please add stricter filters') - - family_unaffected_individual_guids = { - family_guid: {individual_guid for individual_guid, affected_status in individual_affected_status.items() if - affected_status == Individual.AFFECTED_STATUS_UNAFFECTED} - for family_guid, individual_affected_status in self._family_individual_affected_status.items() - } - - self._allowed_consequences += self._consequence_overrides.keys() - if self._allowed_consequences_secondary: - self._allowed_consequences_secondary += self._consequence_overrides.keys() - - compound_het_pairs_by_gene = {} - for gene_agg in response.aggregations.genes.buckets: - self._parse_compound_het_gene(gene_agg, compound_het_pairs_by_gene, family_unaffected_individual_guids) - - total_compound_het_results = sum(len(compound_het_pairs) for compound_het_pairs in compound_het_pairs_by_gene.values()) - logger.info('Total compound het hits: {}'.format(total_compound_het_results), self._user) - - compound_het_results = [] - for k, compound_het_pairs in compound_het_pairs_by_gene.items(): - compound_het_results.extend([{k: compound_het_pair} for compound_het_pair in compound_het_pairs]) - return compound_het_results, total_compound_het_results - - def _parse_compound_het_gene(self, gene_agg, compound_het_pairs_by_gene, family_unaffected_individual_guids): - gene_variants = [self._parse_hit(hit) for hit in gene_agg['vars_by_gene']] - gene_id = gene_agg['key'] - - if gene_id in compound_het_pairs_by_gene: - return - - # Variants are returned if any transcripts have the filtered consequence, but to be compound het - # the filtered consequence needs to be present in at least one transcript in the gene of interest - if self._allowed_consequences: - gene_variants = self._filter_invalid_annotation_compound_hets(gene_id, gene_variants) - - if len(gene_variants) < 2: - return - - # Do not include groups multiple times if identical variants are in the same multiple genes - if any((not variant['mainTranscriptId']) or all(t['transcriptId'] != variant['mainTranscriptId'] - for t in variant['transcripts'][gene_id]) for variant in - gene_variants): - if not self._is_primary_compound_het_gene(gene_id, gene_variants, compound_het_pairs_by_gene): - return - - family_compound_het_pairs = defaultdict(list) - for variant in gene_variants: - for family_guid in variant['familyGuids']: - family_compound_het_pairs[family_guid].append(variant) - - self._filter_invalid_family_compound_hets(gene_id, family_compound_het_pairs, - family_unaffected_individual_guids) - - gene_compound_het_pairs = [ch_pair for ch_pairs in family_compound_het_pairs.values() for ch_pair in ch_pairs] - for compound_het_pair in gene_compound_het_pairs: - for variant in compound_het_pair: - variant['familyGuids'] = [family_guid for family_guid in variant['familyGuids'] - if len(family_compound_het_pairs[family_guid]) > 0] - variant.pop('gene_consequences', None) - gene_compound_het_pairs = [compound_het_pair for compound_het_pair in gene_compound_het_pairs - if compound_het_pair[0]['familyGuids'] and compound_het_pair[1]['familyGuids']] - if gene_compound_het_pairs: - compound_het_pairs_by_gene[gene_id] = gene_compound_het_pairs - - def _is_primary_compound_het_gene(self, gene_id, gene_variants, compound_het_pairs_by_gene): - primary_genes = set() - for variant in gene_variants: - if variant['mainTranscriptId']: - for gene, transcripts in variant['transcripts'].items(): - if any(t['transcriptId'] == variant['mainTranscriptId'] for t in transcripts): - primary_genes.add(gene) - break - if len(primary_genes) == 1: - is_valid_gene = True - primary_gene = primary_genes.pop() - if self._allowed_consequences: - is_valid_gene = all(any( - consequence in self._allowed_consequences for consequence in - variant['gene_consequences'].get(primary_gene, []) - ) for variant in gene_variants) - if is_valid_gene: - if primary_gene != gene_id: - return False - - else: - variant_ids = [variant['variantId'] for variant in gene_variants] - for gene in primary_genes: - if variant_ids == [compound_het_pair[0]['variantId'] for compound_het_pair in - compound_het_pairs_by_gene.get(gene, [])] and \ - variant_ids == [compound_het_pair[1]['variantId'] for compound_het_pair in - compound_het_pairs_by_gene.get(gene, [])]: - return False - return True - - def _filter_invalid_annotation_compound_hets(self, gene_id, gene_variants): - allowed_consequences = self._allowed_consequences + (self._allowed_consequences_secondary or []) - has_canonical_transcript_filter = CANONICAL_TRANSCRIPT_FILTER in allowed_consequences - for variant in gene_variants: - all_gene_consequences = [] - if variant.get('svType'): - all_gene_consequences.append(variant['svType']) - if _is_matched_clinvar_significance(variant.get(CLINVAR_KEY, {}).get('clinicalSignificance'), self._consequence_overrides.get(CLINVAR_KEY)): - all_gene_consequences.append(CLINVAR_KEY) - if variant.get(HGMD_KEY, {}).get('class') in self._consequence_overrides.get(HGMD_KEY, []): - all_gene_consequences.append(HGMD_KEY) - splice_ai = variant.get('predictions', {}).get(SPLICE_AI_FIELD) - if splice_ai and splice_ai >= self._consequence_overrides.get(SPLICE_AI_FIELD, 100): - all_gene_consequences.append(SPLICE_AI_FIELD) - - variant['gene_consequences'] = {} - for k, transcripts in variant['transcripts'].items(): - transcript_consequences = [ - (transcript['majorConsequence'], transcript.get('canonical')) - for transcript in transcripts if transcript.get('majorConsequence') - ] - variant['gene_consequences'][k] = all_gene_consequences + [ - consequence for consequence, _ in transcript_consequences - ] - if has_canonical_transcript_filter and any( - canonical for consequence, canonical in transcript_consequences - if consequence == 'non_coding_transcript_exon_variant'): - variant['gene_consequences'][k].append(CANONICAL_TRANSCRIPT_FILTER) - - return [variant for variant in gene_variants if any( - consequence in allowed_consequences - for consequence in variant['gene_consequences'].get(gene_id, []) - )] - - def _filter_invalid_family_compound_hets(self, gene_id, family_compound_het_pairs, family_unaffected_individual_guids): - for family_guid, variants in family_compound_het_pairs.items(): - unaffected_individuals = family_unaffected_individual_guids.get(family_guid, []) - - hom_alt_variant_ids = set() - if self._paired_index_comp_het: - hom_alt_variant_ids = { - var['variantId'] for var in variants if any(gen.get('numAlt') == 2 for gen in var['genotypes'].values()) - } - - valid_combinations = [] - for ch_1_index, ch_2_index in combinations(range(len(variants)), 2): - variant_1 = variants[ch_1_index] - variant_2 = variants[ch_2_index] - - if hom_alt_variant_ids: - # SNPs overlapped by trans deletions may be incorrectly called as hom alt, and should be - # considered comp hets with said deletions. Any other hom alt variants are not valid comp hets - hom_alt_var = next( - (var for var in [variant_1, variant_2] if var['variantId'] in hom_alt_variant_ids), None) - if hom_alt_var: - pair_var = variant_1 if hom_alt_var == variant_2 else variant_2 - is_valid = pair_var.get('svType') == 'DEL' and pair_var['pos'] <= hom_alt_var['pos'] <= pair_var['end'] - if not is_valid: - continue - - is_valid_for_individual = True - for individual_guid in unaffected_individuals: - genotype_1 = variant_1['genotypes'].get(individual_guid) - genotype_2 = variant_2['genotypes'].get(individual_guid) - if genotype_1 and genotype_2 and genotype_1.get('numAlt') != 0 and not genotype_1.get('isRef') and \ - genotype_2.get('numAlt') != 0 and not genotype_2.get('isRef'): - is_valid_for_individual = False - break - if not is_valid_for_individual: - continue - - if self._allowed_consequences and self._allowed_consequences_secondary: - # Make a copy of lists to prevent blowing up memory usage - consequences = [] + variant_1['gene_consequences'].get(gene_id, []) - consequences += variant_2['gene_consequences'].get(gene_id, []) - if all(consequence not in self._allowed_consequences for consequence in consequences) or all( - consequence not in self._allowed_consequences_secondary for consequence in consequences): - continue - - valid_combinations.append([ch_1_index, ch_2_index]) - - family_compound_het_pairs[family_guid] = [ - [variants[valid_ch_1_index], variants[valid_ch_2_index]] for - valid_ch_1_index, valid_ch_2_index in valid_combinations] - - def _deduplicate_results(self, sorted_new_results): - original_result_count = len(sorted_new_results) - variant_results = [] - for variant in sorted_new_results: - if variant_results and variant_results[-1]['variantId'] == variant['variantId']: - self._merge_duplicate_variants(variant_results[-1], variant) - else: - variant_results.append(variant) - - previous_duplicates = self.previous_search_results.get('duplicate_doc_count', 0) - new_duplicates = original_result_count - len(variant_results) - self.previous_search_results['duplicate_doc_count'] = previous_duplicates + new_duplicates - - self.previous_search_results['total_results'] -= self.previous_search_results['duplicate_doc_count'] - - return variant_results - - @classmethod - def _merge_duplicate_variants(cls, variant, duplicate_variant): - for guid, genotype in duplicate_variant['genotypes'].items(): - if guid in variant['genotypes']: - variant['genotypes'][guid]['otherSample'] = {k: v for k, v in genotype.items() if k != 'otherSample'} - else: - variant['genotypes'][guid] = genotype - variant['familyGuids'] = sorted(set(variant['familyGuids'] + duplicate_variant['familyGuids'])) - - # Always show the most up-to-date clinvar - clinvar_version = variant[CLINVAR_KEY]['version'] or '1900-01-01' - dup_clinvar_version = duplicate_variant[CLINVAR_KEY]['version'] or '1900-01-01' - if dup_clinvar_version > clinvar_version: - variant[CLINVAR_KEY] = duplicate_variant[CLINVAR_KEY] - - def _deduplicate_compound_het_results(self, compound_het_results): - duplicates = 0 - results = {} - for gene_compound_het_pair in compound_het_results: - gene = next(iter(gene_compound_het_pair)) - compound_het_pair = gene_compound_het_pair[gene] - if gene in results: - variant_ids = {variant['variantId'] for variant in compound_het_pair} - existing_index = next( - (i for i, existing in enumerate(results[gene]) if - {variant['variantId'] for variant in existing} == variant_ids), None, - ) - if existing_index is not None: - existing_compound_het_pair = results[gene][existing_index] - self._merge_duplicate_variants(existing_compound_het_pair[0], compound_het_pair[0]) - self._merge_duplicate_variants(existing_compound_het_pair[1], compound_het_pair[1]) - duplicates += 1 - else: - results[gene].append(compound_het_pair) - else: - results[gene] = [compound_het_pair] - - deduplicated_results = [] - for gene, compound_het_pairs in results.items(): - deduplicated_results += [{gene: ch_pair} for ch_pair in compound_het_pairs] - - self.previous_search_results['duplicate_doc_count'] = duplicates + self.previous_search_results.get('duplicate_doc_count', 0) - self.previous_search_results['total_results'] -= duplicates - - return deduplicated_results - - def _process_compound_hets(self, compound_het_results, variant_results, num_results, all_loaded=False): - if not self.previous_search_results.get('grouped_results'): - self.previous_search_results['grouped_results'] = [] - - # Sort merged result sets - grouped_variants = [{None: [var]} for var in variant_results] - grouped_variants = compound_het_results + grouped_variants - grouped_variants = _sort_compound_hets(grouped_variants) - - # Get requested page of variants - merged_variant_results = [] - num_compound_hets = 0 - num_single_variants = 0 - for variants_group in grouped_variants: - variants = next(iter(variants_group.values())) - - self.previous_search_results['grouped_results'].append(variants_group) - if len(variants) > 1: - merged_variant_results.append(variants) - num_compound_hets += 1 - else: - merged_variant_results += variants - num_single_variants += 1 - if not all_loaded and len(merged_variant_results) >= num_results: - break - - self.previous_search_results['compound_het_results'] = compound_het_results[num_compound_hets:] - self.previous_search_results['variant_results'] = variant_results[num_single_variants:] - return merged_variant_results - - def _get_paginated_searches(self, index_name, page=1, num_results=100, start_index=None): - searches = [] - log_messages = [] - for search in self._index_searches.get(index_name, [self._search]): - search = search.index(index_name.split(',')) - - if search.aggs.to_dict(): - # For compound het search get results from aggregation instead of top level hits - search = search[:1] - log_messages.append('Loading {}s for {}'.format(self.AGGREGATION_NAME, index_name)) - else: - end_index = page * num_results - if start_index is None: - start_index = end_index - num_results - - search = search[start_index:end_index] - search = search.source(QUERY_FIELD_NAMES) - log_messages.append('Loading {} records {}-{}'.format(index_name, start_index, end_index)) - - searches.append(search) - return searches, log_messages - - def _execute_search(self, search): - logger.debug(json.dumps(search.to_dict(), indent=2), self._user) - try: - return search.using(self._client).execute() - except elasticsearch.exceptions.ConnectionTimeout as e: - tasks = self._get_long_running_tasks() - if tasks: - logger.error('ES Query Timeout: Found {} long running searches'.format(len(tasks)), self._user, detail=tasks) - else: - logger.warning('ES Query Timeout. No long running searches found', self._user) - raise e - - def _get_long_running_tasks(self): - search_tasks = self._client.tasks.list(actions='*search', group_by='parents') - long_running = [] - for parent_id, task in search_tasks['tasks'].items(): - if task['running_time_in_nanos'] > 10 ** 11: - long_running.append({'task': task, 'parent_task_id': parent_id}) - return long_running - - def _get_quality_filters_by_family(self, quality_filter, affected_only): - quality_field_configs = { - 'min_{}'.format(field): {'field': field, 'step': step} for field, step in QUALITY_QUERY_FIELDS.items() - } - quality_filter = dict({field: 0 for field in quality_field_configs.keys()}, **(quality_filter or {})) - for field, config in quality_field_configs.items(): - if quality_filter[field] % config['step'] != 0: - raise Exception('Invalid {} filter {}'.format(config['field'], quality_filter[field])) - - quality_filters_by_family = {} - new_svs = self._consequence_overrides.get(NEW_SV_FIELD) - if new_svs or any(quality_filter[field] for field in quality_field_configs.keys()): - family_sample_ids = defaultdict(set) - for index in self._indices: - family_samples_by_id = self.samples_by_family_index[index] - for family_guid, samples_by_id in family_samples_by_id.items(): - if affected_only: - family_affected_status = self._family_individual_affected_status[family_guid] - sample_ids = { - sample_id for sample_id, sample in samples_by_id.items() - if family_affected_status[sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED - } - else: - sample_ids = samples_by_id.keys() - family_sample_ids[family_guid].update(sample_ids) - - path_filter = self._get_clinvar_pathogenic_override_filter() - for family_guid, sample_ids in sorted(family_sample_ids.items()): - quality_q = Q('terms', samples_new_call=sorted(sample_ids)) if new_svs else Q() - for sample_id in sorted(sample_ids): - for field, config in sorted(quality_field_configs.items()): - if quality_filter[field]: - q = _build_or_filter('term', [ - {'samples_{}_{}_to_{}'.format(config['field'], i, i + config['step']): sample_id} - for i in range(0, quality_filter[field], config['step']) - ]) - if field == 'min_ab': - # AB only relevant for hets - quality_q &= ~Q(q) | ~Q('term', samples_num_alt_1=sample_id) - else: - quality_q &= ~Q(q) - if path_filter: - quality_q |= path_filter - quality_filters_by_family[family_guid] = quality_q - return quality_filters_by_family - - -def _get_family_affected_status(samples_by_id, inheritance_filter): - individual_affected_status = inheritance_filter.get('affected') or {} - affected_status = {} - for sample in samples_by_id.values(): - indiv = sample.individual - affected_status[indiv.guid] = individual_affected_status.get(indiv.guid) or indiv.affected - - return affected_status - - -def _any_affected_sample_filter(sample_ids): - sample_ids = sorted(sample_ids) - return Q('terms', samples_num_alt_1=sample_ids) | Q('terms', samples_num_alt_2=sample_ids) | Q('terms', samples=sample_ids) - - -def _family_genotype_inheritance_filter(inheritance_mode, inheritance_filter, samples_by_id, individual_affected_status, index_fields): - samples_q = None - - individuals = [sample.individual for sample in samples_by_id.values()] - - individual_genotype_filter = inheritance_filter.get('genotype') or {} - - if inheritance_mode == X_LINKED_RECESSIVE: - samples_q = Q('range', xpos={'gte': get_xpos('X', 1), 'lte': get_xpos('Y', 1)}) - for individual in individuals: - if individual_affected_status[individual.guid] == Individual.AFFECTED_STATUS_UNAFFECTED \ - and individual.sex == Individual.SEX_MALE: - individual_genotype_filter[individual.guid] = REF_REF - - is_sv_comp_het = inheritance_mode == COMPOUND_HET and 'samples' in index_fields - for sample_id, sample in sorted(samples_by_id.items()): - - individual_guid = sample.individual.guid - affected = individual_affected_status[individual_guid] - - genotype = individual_genotype_filter.get(individual_guid) or inheritance_filter.get(affected) - if genotype: - if is_sv_comp_het and affected == Individual.AFFECTED_STATUS_UNAFFECTED: - # Unaffected individuals for SV compound het search can have any genotype so are not included - continue - - not_allowed_num_alt = [ - num_alt for num_alt in GENOTYPE_QUERY_MAP[genotype].get('not_allowed_num_alt', []) - if num_alt in index_fields - ] - allowed_num_alt = [ - num_alt for num_alt in GENOTYPE_QUERY_MAP[genotype].get('allowed_num_alt', []) - if num_alt in index_fields - ] - num_alt_to_filter = not_allowed_num_alt or allowed_num_alt - sample_filters = [{num_alt_key: sample_id} for num_alt_key in num_alt_to_filter] - - sample_q = _build_or_filter('term', sample_filters) - if not_allowed_num_alt: - sample_q = ~Q(sample_q) - - if not samples_q: - samples_q = sample_q - else: - samples_q &= sample_q - - return samples_q - - -def _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family): - sample_queries = [family_samples_q] - quality_q = quality_filters_by_family.get(family_guid) - if quality_q: - sample_queries.append(quality_q) - - return Q('bool', must=sample_queries, _name=family_guid) - - -def _location_filter(genes, intervals, exclude_locations): - q = None - - if genes: - q = Q('terms', geneIds=list(genes.keys())) - - if intervals: - for interval in intervals: - if interval.get('offset'): - offset_pos = int((interval['end'] - interval['start']) * interval['offset']) - interval_q = Q( - 'range', xpos=_pos_offset_range_filter(interval['chrom'], interval['start'], offset_pos)) & Q( - 'range', xstop=_pos_offset_range_filter(interval['chrom'], interval['end'], offset_pos)) - else: - xstart = get_xpos(interval['chrom'], interval['start']) - xstop = get_xpos(interval['chrom'], interval['end']) - range_filters = [{ - key: { - 'gte': xstart, - 'lte': xstop, - } - } for key in ['xpos', 'xstop']] - interval_q = _build_or_filter('range', range_filters) - interval_q |= Q('range', xpos={'lte': xstart}) & Q('range', xstop={'gte': xstop}) & \ - Q('range', xpos={'gte': get_xpos(interval['chrom'], MIN_POS)}) & \ - Q('range', xstop={'lte': get_xpos(interval['chrom'], MAX_POS)}) - - if q: - q |= interval_q - else: - q = interval_q - - if exclude_locations: - return ~q - else: - return q - - -def _pos_offset_range_filter(chrom, pos, offset): - return { - 'lte': get_xpos(chrom, min(pos + offset, MAX_POS)), - 'gte': get_xpos(chrom, max(pos - offset, MIN_POS)), - } - -def _parse_pathogenicity_filter(pathogenicity): - clinvar_filters = pathogenicity.get(CLINVAR_KEY, []) - hgmd_filters = pathogenicity.get(HGMD_KEY, []) - - hgmd_class = set() - if hgmd_filters: - for hgmd_filter in hgmd_filters: - hgmd_class.update(HGMD_CLASS_MAP.get(hgmd_filter, [])) - - return sorted(clinvar_filters), sorted(hgmd_class) - - -VUS_FILTER = 'vus_or_conflicting' -VUS_REGEX = 'Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))' - - -def _pathogenicity_filter(clinvar_filters, hgmd_classes=None): - pathogenicity_filter = None - - path_regex = '|'.join([ - VUS_REGEX if clinvar_filter == VUS_FILTER else f'.*{clinvar_filter.capitalize()}.*' - for clinvar_filter in clinvar_filters or [] - ]) - if path_regex: - pathogenicity_filter = Q('regexp', clinvar_clinical_significance=path_regex) - - if hgmd_classes: - hgmd_q = Q('terms', hgmd_class=hgmd_classes) - pathogenicity_filter = pathogenicity_filter | hgmd_q if pathogenicity_filter else hgmd_q - - return pathogenicity_filter - - -def _is_matched_clinvar_significance(clinical_significance, clinvar_filters): - if not (clinical_significance and clinvar_filters): - return False - - if VUS_FILTER in clinvar_filters: - exclude = [ - path.capitalize() for path in ['pathogenic', 'likely_pathogenic', 'likely_benign', 'benign'] - if path not in clinvar_filters - ] - return all(substring not in clinical_significance for substring in exclude) - - return any(clinvar_filter.capitalize() in clinical_significance for clinvar_filter in clinvar_filters) - - -def _annotations_filter(vep_consequences): - consequences_filter = Q('terms', transcriptConsequenceTerms=sorted(vep_consequences)) - - if 'intergenic_variant' in vep_consequences: - # VEP doesn't add annotations for many intergenic variants so match variants where no transcriptConsequenceTerms - consequences_filter |= ~Q('exists', field='transcriptConsequenceTerms') - - return consequences_filter - - -def _in_silico_filter(in_silico_filters, require_score): - in_silico_qs = [] - for prediction_key, value in in_silico_filters.items(): - try: - score_q = Q('range', **{prediction_key: {'gte': float(value)}}) - except ValueError: - score_q = Q('prefix', **{prediction_key: value}) - - if not require_score: - score_q |= ~Q('exists', field=prediction_key) - - in_silico_qs.append(score_q) - - return _or_filters(in_silico_qs) - - -def _pop_freq_filter(filter_key, value): - return Q('range', **{filter_key: {'lte': value}}) | ~Q('exists', field=filter_key) - - -def _build_or_filter(op, filters): - return _or_filters([Q(op, **filter_kwargs) for filter_kwargs in filters]) - - -def _or_filters(filter_qs): - q = filter_qs[0] - for filter_q in filter_qs[1:]: - q |= filter_q - return q - - -def _sort_compound_hets(grouped_variants): - return sorted(grouped_variants, key=lambda variants: next(iter(variants.values()))[0]['_sort']) - - -def get_compound_het_page(grouped_variants, start_index, end_index): - skipped = 0 - variant_results = [] - variant_count = 0 - for variants in grouped_variants: - curr_variant = next(iter(variants.values())) - if skipped < start_index: - skipped += 1 - else: - if len(curr_variant) == 1: - variant_results += curr_variant - else: - variant_results.append(curr_variant) - variant_count += 1 - if variant_count + skipped >= end_index: - return variant_results - return None - - -def _parse_es_sort(sort, sort_config): - if sort in {'Infinity', '-Infinity', None}: - # ES returns these values for sort when a sort field is missing, using the correct value for the given direction - sort = maxsize - elif hasattr(sort_config, 'values') and any(cfg.get('order') == 'desc' for cfg in sort_config.values()): - sort = float(sort) * -1 - - return sort - - -def _get_field_values(hit, field_configs, format_response_key=_to_camel_case, get_addl_fields=None, lookup_field_prefix='', existing_fields=None, skip_fields=None): - return { - field_config.get('response_key') or format_response_key(field): _value_if_has_key( - hit, - (get_addl_fields(field) if get_addl_fields else []) + - ['{}_{}'.format(lookup_field_prefix, field) if lookup_field_prefix else field], - existing_fields=existing_fields, - **field_config - ) if field not in (skip_fields or []) else None - for field, field_config in field_configs.items() - } - - -def _value_if_has_key(hit, keys, format_value=None, default_value=None, existing_fields=None, **kwargs): - for key in keys: - if key in hit: - return format_value(default_value if hit[key] is None else hit[key]) if format_value else hit[key] - return default_value if not existing_fields or any(key in existing_fields for key in keys) else None diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py deleted file mode 100644 index 5fe4a68106..0000000000 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ /dev/null @@ -1,368 +0,0 @@ -from collections import defaultdict -from elasticsearch import Elasticsearch -from elasticsearch.exceptions import ConnectionError as EsConnectionError, TransportError -import elasticsearch_dsl -from urllib3.connectionpool import connection_from_url - -from seqr.models import Sample, SavedVariant -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json -from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY -from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch -from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page -from seqr.views.utils.json_utils import _to_camel_case -from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ - ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER - - -class InvalidIndexException(Exception): - pass - - -ES_EXCEPTION_ERROR_MAP = { - InvalidIndexException: 400, - EsConnectionError: 504, - TransportError: lambda e: int(e.status_code) if e.status_code != 'N/A' else 400, -} -ES_EXCEPTION_MESSAGE_MAP = { - EsConnectionError: str, - TransportError: lambda e: '{}: {} - {} - {}'.format(e.__class__.__name__, e.status_code, repr(e.error), _get_transport_error_type(e.info)), -} -ES_ERROR_LOG_EXCEPTIONS = {InvalidIndexException} - -MAX_VARIANTS_FETCH = 1000 - - -def _get_transport_error_type(error): - error_type = 'no detail' - if isinstance(error, dict): - root_cause = error.get('root_cause') - error_info = error.get('error') - if (not root_cause) and isinstance(error_info, dict): - root_cause = error_info.get('root_cause') - - if root_cause: - error_type = root_cause[0].get('type') or root_cause[0].get('reason') - elif error_info and not isinstance(error_info, dict): - error_type = repr(error_info) - return error_type - - -def es_backend_enabled(): - return bool(ELASTICSEARCH_SERVICE_HOSTNAME) - - -def get_es_client(timeout=60, **kwargs): - client_kwargs = { - 'hosts': [{'host': ELASTICSEARCH_SERVICE_HOSTNAME, 'port': ELASTICSEARCH_SERVICE_PORT}], - 'timeout': timeout, - } - if ELASTICSEARCH_CREDENTIALS: - client_kwargs['http_auth'] = ELASTICSEARCH_CREDENTIALS - if ELASTICSEARCH_PROTOCOL: - client_kwargs['scheme'] = ELASTICSEARCH_PROTOCOL - if ES_SSL_CONTEXT: - client_kwargs['ssl_context'] = ES_SSL_CONTEXT - return Elasticsearch(**client_kwargs, **kwargs) - - -def ping_elasticsearch(): - if not get_es_client(timeout=3, max_retries=0).ping(): - raise ValueError('No response from elasticsearch ping') - - -def ping_kibana(): - resp = connection_from_url('http://{}'.format(KIBANA_SERVER)).urlopen('HEAD', '/status', timeout=3, retries=3) - if resp.status >= 400: - raise ValueError('Kibana Error {}: {}'.format(resp.status, resp.reason)) - - -SAMPLE_FIELDS_LIST = ['samples', 'samples_num_alt_1'] -# support .bgz instead of requiring .vcf.bgz due to issues with DSP delivery of large callsets -DATASET_FILE_EXTENSIONS = VCF_FILE_EXTENSIONS[:-1] + ('.bgz', '.bed', '.mt') - - -def get_index_metadata(index_name, client, include_fields=False, use_cache=True): - if use_cache: - cache_key = 'index_metadata__{}'.format(index_name) - cached_metadata = safe_redis_get_json(cache_key) - if cached_metadata: - return cached_metadata - - try: - mappings = client.indices.get_mapping(index=index_name) - except Exception as e: - raise InvalidIndexException('{} - Error accessing index: {}'.format( - index_name, e.error if hasattr(e, 'error') else str(e))) - index_metadata = {} - for index_name, mapping in mappings.items(): - variant_mapping = mapping['mappings'] - index_metadata[index_name] = variant_mapping.get('_meta', {}) - if include_fields: - index_metadata[index_name]['fields'] = { - field: field_props.get('type') for field, field_props in variant_mapping['properties'].items() - } - if use_cache and include_fields: - # Only cache metadata with fields - safe_redis_set_json(cache_key, index_metadata) - return index_metadata - - -def validate_es_index_metadata_and_get_samples(request_json, project): - if 'elasticsearchIndex' not in request_json: - raise ValueError('request must contain field: "elasticsearchIndex"') - - elasticsearch_index = request_json['elasticsearchIndex'].strip() - kwargs = { - 'project': project, - 'dataset_type': request_json['datasetType'], - 'genome_version': request_json.get('genomeVersion'), - } - - es_client = get_es_client() - - all_index_metadata = get_index_metadata(elasticsearch_index, es_client, include_fields=True) - if elasticsearch_index in all_index_metadata: - index_metadata = all_index_metadata.get(elasticsearch_index) - _validate_index_metadata(index_metadata, elasticsearch_index, **kwargs) - sample_field = _get_samples_field(index_metadata) - sample_type = index_metadata['sampleType'] - else: - # Aliases return the mapping for all indices in the alias - metadatas = list(all_index_metadata.values()) - sample_field = _get_samples_field(metadatas[0]) - sample_type = metadatas[0]['sampleType'] - for metadata in metadatas[1:]: - _validate_index_metadata(metadata, elasticsearch_index, **kwargs) - if sample_field != _get_samples_field(metadata): - raise ValueError('Found mismatched sample fields for indices in alias') - if sample_type != metadata['sampleType']: - raise ValueError('Found mismatched sample types for indices in alias') - - sample_ids = _get_es_sample_ids(elasticsearch_index, sample_field, es_client) - sample_data = {'elasticsearch_index': elasticsearch_index} - - return sample_ids, sample_type, sample_data - - -def _get_es_sample_ids(elasticsearch_index, sample_field, es_client): - s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index) - s = s.params(size=0) - s.aggs.bucket('sample_ids', elasticsearch_dsl.A('terms', field=sample_field, size=10000)) - response = s.execute() - return [agg['key'] for agg in response.aggregations.sample_ids.buckets] - - -def _get_samples_field(index_metadata): - return next((field for field in SAMPLE_FIELDS_LIST if field in index_metadata['fields'].keys())) - - -def _validate_index_metadata(index_metadata, elasticsearch_index, project=None, genome_version=None, - dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS): - metadata_fields = ['genomeVersion', 'sampleType', 'sourceFilePath'] - if any(field not in (index_metadata or {}) for field in metadata_fields): - raise ValueError("Index metadata must contain fields: {}".format(', '.join(metadata_fields))) - - sample_type = index_metadata['sampleType'] - if sample_type not in {choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES}: - raise ValueError("Sample type not supported: {}".format(sample_type)) - - if index_metadata['genomeVersion'] != (genome_version or project.genome_version): - raise ValueError('Index "{0}" has genome version {1} but this project uses version {2}'.format( - elasticsearch_index, index_metadata['genomeVersion'], project.genome_version - )) - - dataset_path = index_metadata['sourceFilePath'] - if not dataset_path.endswith(DATASET_FILE_EXTENSIONS): - raise ValueError("Variant call dataset path must end with {}".format(' or '.join(DATASET_FILE_EXTENSIONS))) - - index_dataset_type = EsSearch.get_index_metadata_dataset_type(index_metadata) - if index_dataset_type != dataset_type: - raise ValueError('Index "{0}" has dataset type {1} but expects {2}'.format( - elasticsearch_index, index_dataset_type, dataset_type - )) - - -def delete_es_index(index): - client = get_es_client() - client.indices.delete(index) - updated_indices, _ = _get_es_indices(client) - return updated_indices - - -def get_elasticsearch_status(): - client = get_es_client() - - disk_status = { - disk['node']: disk for disk in - _get_es_meta(client, 'allocation', ['node', 'shards', 'disk.avail', 'disk.used', 'disk.percent']) - } - - node_stats = {} - for node in _get_es_meta(client, 'nodes', ['name', 'heap.percent']): - if node['name'] in disk_status: - disk_status[node.pop('name')].update(node) - else: - node_stats[node['name']] = node - - indices, seqr_index_projects = _get_es_indices(client) - - errors = ['{} does not exist and is used by project(s) {}'.format( - index, ', '.join(['{} ({} samples)'.format(p.name, len(indivs)) for p, indivs in project_individuals.items()]) - ) for index, project_individuals in sorted(seqr_index_projects.items()) if project_individuals] - - return { - 'indices': indices, - 'diskStats': list(disk_status.values()), - 'nodeStats': list(node_stats.values()), - 'errors': errors, - } - - -def _get_es_meta(client, meta_type, fields, filter_rows=None): - return [{ - _to_camel_case(field.replace('.', '_')): o[field] for field in fields - } for o in getattr(client.cat, meta_type)(format="json", h=','.join(fields)) - if filter_rows is None or filter_rows(o)] - - -def _get_es_indices(client): - indices = _get_es_meta( - client, 'indices', ['index', 'docs.count', 'store.size', 'creation.date.string'], - filter_rows=lambda index: all( - not index['index'].startswith(omit_prefix) for omit_prefix in ['.', 'index_operations_log'])) - - aliases = defaultdict(list) - for alias in _get_es_meta(client, 'aliases', ['alias', 'index']): - aliases[alias['alias']].append(alias['index']) - - index_metadata = get_index_metadata('_all', client, use_cache=False) - - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index__isnull=False).select_related('individual__family__project') - - seqr_index_projects = defaultdict(lambda: defaultdict(set)) - es_projects = set() - for sample in active_samples: - for index_name in sample.elasticsearch_index.split(','): - project = sample.individual.family.project - es_projects.add(project) - if index_name in aliases: - for aliased_index_name in aliases[index_name]: - seqr_index_projects[aliased_index_name][project].add(sample.individual.guid) - else: - seqr_index_projects[index_name.rstrip('*')][project].add(sample.individual.guid) - - for index in indices: - index_name = index['index'] - index.update(index_metadata[index_name]) - - projects_for_index = [] - for index_prefix in list(seqr_index_projects.keys()): - if index_name.startswith(index_prefix): - projects_for_index += list(seqr_index_projects.pop(index_prefix).keys()) - index['projects'] = [ - {'projectGuid': project.guid, 'projectName': project.name} for project in projects_for_index] - - return indices, seqr_index_projects - - -def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): - variant_ids = sorted(set(variant_ids)) - variants = EsSearch( - samples, genome_version, user=user, sort=XPOS_SORT_KEY, - ).filter_by_variant_ids(variant_ids) - return variants.search(num_results=len(variant_ids)) - - -def get_es_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=None, num_results=None, - gene_agg=False, skip_genotype_filter=False): - es_search_cls = EsGeneAggSearch if gene_agg else EsSearch - - es_search = es_search_cls( - samples, - genome_version, - previous_search_results=previous_search_results, - user=user, - sort=sort, - skipped_samples=search.get('skipped_samples'), - ) - - es_search.filter_variants( - inheritance_mode=search.get('inheritance_mode'), inheritance_filter=search.get('inheritance_filter'), - frequencies=search.get('freqs'), pathogenicity=search.get('pathogenicity'), - annotations=search.get('annotations'), annotations_secondary=search.get('annotations_secondary'), - in_silico=search.get('in_silico'), quality_filter=search.get('qualityFilter'), - custom_query=search.get('customQuery'), locus=search.get('locus'), skip_genotype_filter=skip_genotype_filter, - dataset_type=search.get('dataset_type'), secondary_dataset_type=search.get('secondary_dataset_type'), - genes=search.get('genes'), intervals=search.get('intervals'), rs_ids=search.get('rs_ids'), - variant_ids=search.get('variant_ids'), exclude_locations=search.get('exclude_locations'), - ) - - return es_search.search(page=page, num_results=num_results) - - -def process_es_previously_loaded_results(previous_search_results, start_index, end_index): - grouped_results = previous_search_results.get('grouped_results') - results = None - if grouped_results: - results = get_compound_het_page(grouped_results, start_index, end_index) - - return results - - -def process_es_previously_loaded_gene_aggs(previous_search_results): - total_results = previous_search_results.get('total_results') - if total_results is None or 'all_results' in previous_search_results or 'grouped_results' not in previous_search_results: - return None - - loaded = sum(counts.get('loaded', 0) for counts in previous_search_results.get('loaded_variant_counts', {}).values()) - if loaded != total_results: - return None - - gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - - for group in previous_search_results['grouped_results']: - variants = next(iter(group.values())) - gene_id = next(iter(group)) - if not gene_id or gene_id == 'null': - gene_id = next(( - gene_id for gene_id, transcripts in variants[0]['transcripts'].items() - if any(t['transcriptId'] == variants[0]['mainTranscriptId'] for t in transcripts) - ), None) if variants[0]['mainTranscriptId'] else None - if gene_id: - gene_aggs[gene_id]['total'] += len(variants) - for family_guid in variants[0]['familyGuids']: - gene_aggs[gene_id]['families'][family_guid] += len(variants) - return gene_aggs - - -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): - from seqr.views.utils.variant_utils import get_saved_variants - saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') - - if not saved_variants: - return None - - family_ids = set() - variant_ids = set() - saved_variants_map = {} - for v in saved_variants: - family_ids.add(v.family_id) - variant_ids.add(v.variant_id) - saved_variants_map[(v.variant_id, v.family.guid)] = v - - variant_ids = sorted(variant_ids) - samples = Sample.objects.filter(individual__family_id__in=family_ids, is_active=True) - variants_json = [] - for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: - variants_json += get_es_variants_for_variant_ids(samples, genome_version, sub_var_ids, user) - - updated_saved_variants = {} - for var in variants_json: - for family_guid in var['familyGuids']: - saved_variant = saved_variants_map.get((var['variantId'], family_guid)) - if saved_variant: - saved_variant.saved_variant_json = var - updated_saved_variants[saved_variant.guid] = saved_variant - SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) - - return updated_saved_variants diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py deleted file mode 100644 index a4bfc081ef..0000000000 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ /dev/null @@ -1,3795 +0,0 @@ -from copy import deepcopy -import mock -import jmespath -import json -import re -from collections import defaultdict -from datetime import timedelta -from django.test import TestCase -from elasticsearch.exceptions import ConnectionTimeout, TransportError -from sys import maxsize -from urllib3.exceptions import ReadTimeoutError -from urllib3_mock import Responses - -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, get_es_variants_for_variant_ids, InvalidSearchException -from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status -from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException -from seqr.utils.search.utils import _get_liftover -from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_SV_WGS_VARIANT,\ - PARSED_MITO_VARIANT, TRANSCRIPT_2, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT - - -# The responses library for mocking requests does not work with urllib3 (which is used by elasticsearch) -# The urllib3_mock library works for those requests, but it has limited functionality, so this extension adds helper -# methods for easier usage -class Urllib3Responses(Responses): - def add_json(self, url, json_response, method=None, match_querystring=True, **kwargs): - if not method: - method = self.GET - body = json.dumps(json_response) - self.add(method, url, match_querystring=match_querystring, content_type='application/json', body=body, **kwargs) - - def replace_json(self, url, *args, **kwargs): - existing_index = next(i for i, match in enumerate(self._urls) if match['url'] == url) - self.add_json(url, *args, **kwargs) - self._urls[existing_index] = self._urls.pop() - - def call_request_json(self, index=-1): - return json.loads(self.calls[index].request.body) - - def activate(self, func): - def wrapper(*args, **kwargs): - with self: - return func(*args, **kwargs) - return wrapper - -urllib3_responses = Urllib3Responses() - - -INDEX_NAME = 'test_index' -SECOND_INDEX_NAME = 'test_index_second' -HG38_INDEX_NAME = 'test_index_hg38' -SV_INDEX_NAME = 'test_index_sv' -SV_WGS_INDEX_NAME = 'test_index_sv_wgs' -MITO_WGS_INDEX_NAME = 'test_index_mito_wgs' -INDEX_ALIAS = '377e97bd791cf92a78296bc184e0976a' -ALIAS_MAP = {INDEX_ALIAS: ','.join([INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME, MITO_WGS_INDEX_NAME])} -SUB_INDICES = ['sub_index_1', 'sub_index_2'] -SECOND_SUB_INDICES = ['sub_index_a', 'sub_index_b'] - -ES_VARIANTS = [ - { - '_source': { - 'homozygote_count': 3, - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '1-248367227-TC-T' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': 0, - 'cadd_PHRED': '25.9', - 'splice_ai_delta_score': 0.75, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': 125568, - 'g1k_AN': None, - 'topmed_AF': 0.00016724, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'TC', - 'exac_AC_Hom': 0, - 'topmed_AC': 21, - 'dbnsfp_REVEL_score': None, - 'dbnsfp_VEST4_score': '0.335;0.341;0.38', - 'primate_ai_score': None, - 'variantId': '1-248367227-TC-T', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'nonsense_mediated_decay', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Gene3D:1', - ], - 'canonical': None, - 'transcript_rank': 1, - 'cdna_end': 143, - 'lof_filter': None, - 'hgvs': 'ENSP00000413641.1:p.Leu48del', - 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', - 'cdna_start': 141, - 'transcript_id': 'ENST00000428085', - 'protein_id': 'ENSP00000413641', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000413641.1:p.Leu48del', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant', - 'inframe_deletion', - 'NMD_transcript_variant' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'screen_region_type' : [ - 'dELS', - 'CTCF-bound' - ], - 'gnomad_non_coding_constraint_z_score': 1.01272, - 'hgmd_class': 'DM', - 'AC': 2, - 'exac_AN_Adj': 121308, - 'mpc_MPC': None, - 'AF': 0.063, - 'alt': 'T', - 'clinvar_clinical_significance': 'Pathogenic/Likely_pathogenic', - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': 0.0004590314436538903, - 'gnomad_genomes_FAF_AF': 0.000437, - 'exac_AF': 0.00006589, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 8, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': 0, - 'gnomad_genomes_AN': 30946, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.0009151523074911753, - 'gnomad_genomes_Hemi': None, - 'xpos': 1248367227, - 'start': 248367227, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245930, - 'contig': '1', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.0006726888333653661, - 'gnomad_exomes_AC': 16, - 'dbnsfp_FATHMM_pred': 'T', - 'dbnsfp_fathmm_MKL_coding_pred': 'D', - 'gnomad_exomes_AF': 0.00006505916317651364, - 'gnomad_genomes_AF': 0.00012925741614425127, - 'gnomad_genomes_AC': 4, - 'genotypes': [ - { - 'num_alt': 2, - 'ab': 1, - 'dp': 74, - 'gq': 99, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 88, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - ], - 'samples_num_alt_1': ['NA20885'], - 'samples_num_alt_2': ['NA20870'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3'], SECOND_INDEX_NAME: ['F000011_11']}, - }, - { - '_source': { - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '2-103343353-GAGA-G' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': None, - 'cadd_PHRED': None, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': None, - 'g1k_AN': None, - 'topmed_AF': None, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'GAGA', - 'exac_AC_Hom': 0, - 'topmed_AC': None, - 'dbnsfp_REVEL_score': None, - 'primate_ai_score': 1, - 'variantId': '2-103343353-GAGA-G', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'PROSITE_profiles:PS50850', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 421, - 'lof_filter': None, - 'hgvs': 'ENSP00000258436.5:p.Leu126del', - 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', - 'cdna_start': 419, - 'transcript_id': 'ENST00000258436', - 'protein_id': 'ENSP00000258436', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000258436.5:p.Leu126del', - 'major_consequence': 'inframe_deletion', - 'consequence_terms': [ - 'inframe_deletion' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'hgmd_class': None, - 'AC': 1, - 'exac_AN_Adj': 121336, - 'mpc_MPC': None, - 'AF': 0.031, - 'alt': 'G', - 'clinvar_clinical_significance': None, - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': None, - 'gnomad_genomes_FAF_AF': None, - 'exac_AF': 0.00004942, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 6, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': None, - 'gnomad_genomes_AN': None, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'dbnsfp_VEST4_score': '.;.;.;.', - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.00016269686320447742, - 'gnomad_genomes_Hemi': None, - 'xpos': 2103343353, - 'start': 103343353, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245714, - 'contig': '2', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.000242306760358614, - 'gnomad_exomes_AC': 6, - 'dbnsfp_FATHMM_pred': None, - 'gnomad_exomes_AF': 0.000024418633044922146, - 'gnomad_genomes_AF': None, - 'gnomad_genomes_AC': None, - 'genotypes': [ - { - 'num_alt': 1, - 'ab': 0.70212764, - 'dp': 50, - 'gq': 46, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - { - 'num_alt': 2, - 'ab': 0, - 'dp': 67, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00732', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00733', - } - ], - 'samples_num_alt_1': ['NA20870', 'HG00733', 'NA20885'], - 'samples_num_alt_2': ['HG00732'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3', 'F000002_2'], SECOND_INDEX_NAME: ['F000011_11']}, - }, -] -BUILD_38_NO_LIFTOVER_ES_VARIANT = deepcopy(ES_VARIANTS[1]) -BUILD_38_NO_LIFTOVER_ES_VARIANT['_source'].update({ - 'start': 103343363, - 'xpos': 2103343363, - 'variantId': '2-103343363-GAGA-G' -}) -BUILD_38_ES_VARIANT = deepcopy(BUILD_38_NO_LIFTOVER_ES_VARIANT) -BUILD_38_ES_VARIANT['_source']['rg37_locus'] = {'contig': '2', 'position': 103343353} - -ES_SV_VARIANT = { - '_source': { - 'genotypes': [ - { - 'qs': 33, - 'cn': 1, - 'defragged': False, - 'sample_id': 'HG00731', - 'num_exon': 2, - 'start': 49045487, - 'end': 49045899, - 'geneIds': ['ENSG00000228198'], - 'prev_call': False, - 'prev_overlap': False, - 'new_call': True, - }, - { - 'qs': 80, - 'cn': 2, - 'defragged': False, - 'sample_id': 'HG00733', - 'num_exon': 1, - 'start': 49045987, - 'end': 49045890, - 'geneIds': ['ENSG00000228198', 'ENSG00000135953'], - 'prev_call': False, - 'prev_overlap': True, - 'new_call': False, - } - ], - 'xpos': 1049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 1049045587, - 'num_exon': 1, - 'pos': 49045487, - 'StrVCTVRE_score': 0.374, - 'svType': 'INS', - 'xstop': 9049045898, - 'variantId': 'prefix_19107_DEL', - 'samples': ['HG00731'], - 'sc': 7, - 'contig': '1', - 'bothsides_support': True, - 'sortedTranscriptConsequences': [ - { - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_id': 'ENSG00000135953' - }, - { - 'gene_id': 'ENSG00000186092' - }, - ], - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_INDEX_NAME: ['F000002_2']}, - } - -ES_SV_WGS_VARIANT = { - '_source': { - 'genotypes': [ - { - 'gq': 33, - 'sample_id': 'NA21234', - 'num_alt': 1, - 'prev_num_alt': 2, - } - ], - 'xpos': 2049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 2049045387, - 'pos': 49045387, - 'svType': 'CPX', - 'xstop': 20012345678, - 'variantId': 'prefix_19107_CPX', - 'algorithms': ['wham', 'manta'], - 'sc': 7, - 'contig': '2', - 'rg37_locus': {'contig': '2', 'position': 49272526}, - 'rg37_locus_end': {'contig': '20', 'position': 12326326}, - 'sortedTranscriptConsequences': [ - { - 'gene_symbol': 'OR4F5', - 'major_consequence': 'DUP_PARTIAL', - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_symbol': 'FBXO28', - 'major_consequence': 'MSV_EXON_OVR', - 'gene_id': 'ENSG00000228199' - }, - { - 'gene_symbol': 'FAM131C', - 'major_consequence': 'DUP_LOF', - 'gene_id': 'ENSG00000228201' - }, - { - "gene_symbol": "H3-2", - "gene_id": None, - "major_consequence": "NEAREST_TSS" - }, - ], - 'cpx_intervals': [{'type': 'DUP', 'chrom': '2', 'start': 1000, 'end': 3000}, - {'type': 'INV', 'chrom': '20', 'start': 11000, 'end': 13000}], - 'sv_type_detail': 'dupINV', - 'gnomad_svs_ID': 'gnomAD-SV_v2.1_BND_1_1', - 'gnomad_svs_AF': 0.00679, - 'gnomad_svs_AC': 22, - 'gnomad_svs_AN': 3240, - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_WGS_INDEX_NAME: ['F000014_14']}, - } - -ES_MITO_WGS_VARIANT = { - "_source" : { - "genotypes" : [ - { - "num_alt": 2, - "gq": 60.0, - "hl": 1.0, - "mito_cn": 319.03225806451616, - "contamination": 0.0, - "dp": 5139.0, - "sample_id": "HG00731" - }, - ], - "samples_gq_60_to_65" : [ - "HG00731" - ], - "samples_num_alt_2" : [ "HG00731" ], - "AC" : 0, - "AC_het" : 1, - "AF" : 0.0, - "AF_het" : 3.968253968253968E-4, - "alt" : "A", - "AN" : 2520, - "clinvar_allele_id" : None, - "clinvar_clinical_significance" : "Likely_pathogenic", - "clinvar_gold_stars" : None, - "codingGeneIds" : [ - "ENSG00000198840" - ], - "common_low_heteroplasmy" : False, - "contig" : "M", - "dbnsfp_SIFT_pred" : "D", - "dbnsfp_MutationTaster_pred" : "N", - "dbnsfp_FATHMM_pred" : "T", - "dbnsfp_GERP_RS" : "5.07", - "dbnsfp_phastCons100way_vertebrate" : "0.958000", - "docId" : "M-10195-C-A", - "domains" : [ - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Gene3D:1", - "PANTHER:PTHR11058", - "Pfam:PF00507" - ], - "end" : 10195, - "filters" : [ ], - "geneIds" : [ - "ENSG00000198840" - ], - "gnomad_mito_AN": 56421, - "gnomad_mito_AC": 1368, - "gnomad_mito_AC_het": 3, - "gnomad_mito_AF": 0.024246292, - "gnomad_mito_AF_het": 5.317169E-5, - "gnomad_mito_max_hl": 1.0, - "hap_defining_variant" : False, - "helix_AC": 1312, - "helix_AC_het": 5, - "helix_AF": 0.0033268193, - "helix_AF_het": 4.081987E-5, - "helix_max_hl": 0.90441, - "high_constraint_region" : True, - "hmtvar_hmtVar" : 0.71, - "mainTranscript_biotype" : "protein_coding", - "mainTranscript_canonical" : 1, - "mainTranscript_category" : "missense", - "mainTranscript_cdna_start" : 137, - "mainTranscript_cdna_end" : 137, - "mainTranscript_codons" : "cCc/cAc", - "mainTranscript_gene_id" : "ENSG00000198840", - "mainTranscript_gene_symbol" : "MT-ND3", - "mainTranscript_hgvs" : "p.Pro46His", - "mainTranscript_hgvsc" : "ENST00000361227.2:c.137C>A", - "mainTranscript_major_consequence" : "missense_variant", - "mainTranscript_major_consequence_rank" : 11, - "mainTranscript_transcript_id" : "ENST00000361227", - "mainTranscript_amino_acids" : "P/H", - "mainTranscript_domains" : "Gene3D:1,ENSP_mappings:5xtc,PANTHER:PTHR11058,ENSP_mappings:5xtd,Pfam:PF00507", - "mainTranscript_hgvsp" : "ENSP00000355206.2:p.Pro46His", - "mainTranscript_polyphen_prediction" : "probably_damaging", - "mainTranscript_protein_id" : "ENSP00000355206", - "mainTranscript_sift_prediction" : "deleterious_low_confidence", - "mitimpact_apogee" : 0.42, - "mitomap_pathogenic" : True, - "pos" : 10195, - "ref" : "C", - "rg37_locus" : { - "contig" : "MT", - "position" : 10195 - }, - "rsid" : None, - "sortedTranscriptConsequences" : [ - { - "biotype" : "protein_coding", - "canonical" : 1, - "cdna_start" : 137, - "cdna_end" : 137, - "codons" : "cCc/cAc", - "gene_id" : "ENSG00000198840", - "gene_symbol" : "MT-ND3", - "hgvsc" : "ENST00000361227.2:c.137C>A", - "hgvsp" : "ENSP00000355206.2:p.Pro46His", - "transcript_id" : "ENST00000361227", - "amino_acids" : "P/H", - "polyphen_prediction" : "probably_damaging", - "protein_id" : "ENSP00000355206", - "protein_start" : 46, - "sift_prediction" : "deleterious_low_confidence", - "consequence_terms" : [ - "missense_variant" - ], - "domains" : [ - "Gene3D:1", - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Pfam:PF00507", - "PANTHER:PTHR11058", - "PANTHER:PTHR11058" - ], - "major_consequence" : "missense_variant", - "category" : "missense", - "hgvs" : "p.Pro46His", - "major_consequence_rank" : 11, - "transcript_rank" : 0 - } - ], - "start" : 10195, - "transcriptConsequenceTerms" : [ - "missense_variant" - ], - "transcriptIds" : [ - "ENST00000361227" - ], - "variantId" : "M-10195-C-A", - "xpos" : 25000010195, - "xstart" : 25000010195, - "xstop" : 25000010195 - }, - 'matched_queries': {MITO_WGS_INDEX_NAME: ['F000002_2']}, -} - -OR2M3_COMPOUND_HET_ES_VARIANTS = deepcopy(ES_VARIANTS) -transcripts = OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] -transcripts[0]['major_consequence'] = 'frameshift_variant' -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] = [transcripts[1], transcripts[0]] -OR2M3_COMPOUND_HET_ES_VARIANTS[0]['_source']['rg37_locus'] = {'contig': '1', 'position': 248367217} -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['rg37_locus'] = {'contig': '2', 'position': 103343343} -MFSD9_COMPOUND_HET_ES_VARIANTS = deepcopy(OR2M3_COMPOUND_HET_ES_VARIANTS) -for var in MFSD9_COMPOUND_HET_ES_VARIANTS: - var['_source']['variantId'] = '{}-het'.format(var['_source']['variantId']) -EXTRA_FAMILY_ES_VARIANTS = deepcopy(ES_VARIANTS) + [deepcopy(ES_VARIANTS[0])] -EXTRA_FAMILY_ES_VARIANTS[2]['matched_queries'][INDEX_NAME] = ['F000005_5'] -MISSING_SAMPLE_ES_VARIANTS = deepcopy(ES_VARIANTS) -MISSING_SAMPLE_ES_VARIANTS[1]['_source']['samples_num_alt_1'] = [] - -ES_SV_COMP_HET_VARIANT = deepcopy(ES_SV_VARIANT) -ES_SV_COMP_HET_VARIANT['_source']['xpos'] = 2101343374 -ES_SV_COMP_HET_VARIANT['_source']['start'] = 101343374 -ES_SV_COMP_HET_VARIANT['_source']['xstop'] = 1104943628 -ES_SV_COMP_HET_VARIANT['_source']['end'] = 104943628 -ES_SV_COMP_HET_VARIANT['_source']['num_exon'] = 2 -ES_SV_COMP_HET_VARIANT['_source']['variantId'] = 'prefix_191011_DEL' -ES_SV_COMP_HET_VARIANT['_source']['svType'] = 'DEL' -for gen in ES_SV_COMP_HET_VARIANT['_source']['genotypes']: - gen.update({'start': None, 'end': None, 'num_exon': None}) - gen.pop('geneIds') - -COMPOUND_HET_INDEX_VARIANTS = { - INDEX_NAME: { - 'ENSG00000135953': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228198': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228199': EXTRA_FAMILY_ES_VARIANTS, - }, - SECOND_INDEX_NAME: { - 'ENSG00000135953': MFSD9_COMPOUND_HET_ES_VARIANTS, 'ENSG00000228198': OR2M3_COMPOUND_HET_ES_VARIANTS, - }, - '{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME): {'ENSG00000135953': MISSING_SAMPLE_ES_VARIANTS}, - SV_INDEX_NAME: {'ENSG00000228198': [ES_SV_VARIANT], 'ENSG00000135953': []}, - '{},{}'.format(INDEX_NAME, SV_INDEX_NAME): {'ENSG00000228198': [ES_SV_VARIANT, ES_SV_COMP_HET_VARIANT, ES_VARIANTS[1]], 'ENSG00000135953': []}, -} - -INDEX_ES_VARIANTS = { - INDEX_NAME: ES_VARIANTS, - SECOND_INDEX_NAME: [ES_VARIANTS[1]], - SV_INDEX_NAME: [ES_SV_VARIANT], - SV_WGS_INDEX_NAME: [ES_SV_WGS_VARIANT], - HG38_INDEX_NAME: [BUILD_38_ES_VARIANT, BUILD_38_NO_LIFTOVER_ES_VARIANT], - MITO_WGS_INDEX_NAME: [ES_MITO_WGS_VARIANT], -} -INDEX_ES_VARIANTS.update({k: ES_VARIANTS for k in SUB_INDICES + SECOND_SUB_INDICES}) - -PARSED_ANY_AFFECTED_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_ANY_AFFECTED_VARIANTS[1]['familyGuids'] = ['F000003_3'] -PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes'] = {'I000007_na20870': PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes']['I000007_na20870']} - -PARSED_COMPOUND_HET_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_COMPOUND_HET_VARIANTS[0]['_sort'] = [1248367327] -PARSED_COMPOUND_HET_VARIANTS[1]['_sort'] = [2103343453] -PARSED_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000003_3'] - -PARSED_SV_COMPOUND_HET_VARIANTS = [deepcopy(PARSED_SV_VARIANT), deepcopy(PARSED_COMPOUND_HET_VARIANTS[1])] -PARSED_SV_COMPOUND_HET_VARIANTS[0].update({ - '_sort': [2101343474], - 'xpos': 2101343374, - 'pos': 101343374, - 'end': 104943628, - 'variantId': 'prefix_191011_DEL', - 'svType': 'DEL', -}) -del PARSED_SV_COMPOUND_HET_VARIANTS[0]['svSourceDetail'] -PARSED_SV_COMPOUND_HET_VARIANTS[0]['transcripts']['ENSG00000186092'] = [{'geneId': 'ENSG00000186092'}] -for gen in PARSED_SV_COMPOUND_HET_VARIANTS[0]['genotypes'].values(): - gen.update({'start': None, 'end': None, 'numExon': None, 'geneIds': None}) -PARSED_SV_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000002_2'] - -PARSED_COMPOUND_HET_VARIANTS_PROJECT_2 = deepcopy(PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT) -for variant in PARSED_COMPOUND_HET_VARIANTS_PROJECT_2: - variant.update({ - 'variantId': '{}-het'.format(variant['variantId']), - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': variant['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '37', - 'selectedMainTranscriptId': None, - }) - variant['clinvar']['version'] = None - -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[1]['selectedMainTranscriptId'] = None - -PARSED_CADD_VARIANTS = deepcopy(PARSED_NO_CONSEQUENCE_FILTER_VARIANTS) -PARSED_CADD_VARIANTS[0]['_sort'][0] = -25.9 -PARSED_CADD_VARIANTS[1]['_sort'][0] = maxsize - - -PARSED_MULTI_INDEX_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_MULTI_INDEX_VARIANT['familyGuids'].append('F000011_11') -PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'] = { - 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, 'sampleType': 'WES', -} - -PARSED_HG38_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_HG38_VARIANT.update({ - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverPos': PARSED_MULTI_INDEX_VARIANT['pos'], - 'liftedOverChrom': PARSED_MULTI_INDEX_VARIANT['chrom'], - 'pos': PARSED_MULTI_INDEX_VARIANT['pos'] + 10, - 'xpos': PARSED_MULTI_INDEX_VARIANT['xpos'] + 10, - 'variantId': PARSED_MULTI_INDEX_VARIANT['variantId'].replace( - str(PARSED_MULTI_INDEX_VARIANT['pos']), str(PARSED_MULTI_INDEX_VARIANT['pos'] + 10) - ), - '_sort': [PARSED_MULTI_INDEX_VARIANT['_sort'][0] + 10], -}) -PARSED_HG38_VARIANT['clinvar']['version'] = None - -PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -for guid, genotype in PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT = deepcopy(PARSED_VARIANTS[1]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT_0 = deepcopy(PARSED_VARIANTS[0]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS = deepcopy(PARSED_COMPOUND_HET_VARIANTS) -for variant in PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS: - for guid, genotype in variant['genotypes'].items(): - variant['genotypes'][guid] = dict(otherSample=genotype, **genotype) - - -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT.update({ - 'familyGuids': ['F000003_3', 'F000011_11'], - 'genotypes': { - ind_guid: PARSED_MULTI_INDEX_VARIANT['genotypes'][ind_guid] - for ind_guid in ['I000007_na20870', 'I000015_na20885'] - }, -}) -MAPPING_FIELDS = [ - 'start', - 'end', - 'rsid', - 'originalAltAlleles', - 'filters', - 'xpos', - 'alt', - 'ref', - 'contig', - 'variantId', - 'dbnsfp_MutationTaster_pred', - 'mpc_MPC', - 'dbnsfp_DANN_score', - 'eigen_Eigen_phred', - 'dbnsfp_REVEL_score', - 'splice_ai_delta_score', - 'splice_ai_splice_consequence', - 'dbnsfp_FATHMM_pred', - 'dbnsfp_fathmm_MKL_coding_pred', - 'dbnsfp_MutPred_score', - 'dbnsfp_VEST4_score', - 'primate_ai_score', - 'dbnsfp_SIFT_pred', - 'dbnsfp_Polyphen2_HVAR_pred', - 'cadd_PHRED', - 'gnomad_non_coding_constraint_z_score', - 'sortedTranscriptConsequences', - 'screen_region_type', - 'genotypes', - 'samples_no_call', - 'samples_num_alt_1', - 'samples_num_alt_2', - 'clinvar_clinical_significance', - 'clinvar_allele_id', - 'clinvar_variation_id', - 'clinvar_gold_stars', - 'hgmd_accession', - 'hgmd_class', - 'AC', - 'AF', - 'AN', - 'gnomad_genomes_AC', - 'gnomad_genomes_Hom', - 'gnomad_genomes_Hemi', - 'gnomad_genomes_AF', - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_genomes_AN', - 'gnomad_genomes_Het', - 'gnomad_genomes_ID', - 'gnomad_exomes_AC', - 'gnomad_exomes_Hom', - 'gnomad_exomes_Hemi', - 'gnomad_exomes_AF', - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_exomes_AN', - 'gnomad_exomes_Het', - 'gnomad_exomes_ID', - 'exac_AC_Adj', - 'exac_AC_Hom', - 'exac_AC_Hemi', - 'exac_AF_POPMAX', - 'exac_AF', - 'exac_AN_Adj', - 'exac_Het', - 'exac_ID', - 'topmed_AC', - 'topmed_Hom', - 'topmed_Hemi', - 'topmed_Het', - 'topmed_AF', - 'topmed_AN', - 'topmed_ID', - 'gnomad_genomes_FAF_AF', - 'rg37_locus', - 'rg37_locus_end', - 'xstop', - 'bothsides_support', -] - -SV_MAPPING_FIELDS = [ - 'start', - 'end', - 'xpos', - 'contig', - 'variantId', - 'sortedTranscriptConsequences', - 'genotypes', - 'samples', - 'samples_cn_0', - 'samples_cn_1', - 'samples_cn_2', - 'samples_cn_3', - 'samples_cn_gte_4', - 'sf', - 'sc', - 'sn', - 'num_exon', - 'svType', - 'StrVCTVRE_score', - 'sv_type_detail', - 'cpx_intervals', - 'gnomad_svs_AC', - 'gnomad_svs_Hom', - 'gnomad_svs_Hemi', - 'gnomad_svs_AF', - 'gnomad_svs_AN', - 'gnomad_svs_filter_AF', - 'gnomad_svs_Het', - 'gnomad_svs_ID', - 'bothsides_support', -] - -MITO_MAPPING_FIELDS = [ - "AC", - "AC_het", - "AF", - "AF_het", - "AN", - "alt", - "clinvar_allele_id", - "clinvar_clinical_significance", - "clinvar_gold_stars", - "common_low_heteroplasmy", - "contig", - "dbnsfp_FATHMM_pred", - "dbnsfp_MutationTaster_pred", - "dbnsfp_Polyphen2_HVAR_pred", - "dbnsfp_REVEL_score", - "dbnsfp_SIFT_pred", - "end", - "filters", - "genotypes", - "gnomad_mito_AC", - "gnomad_mito_AC_het", - "gnomad_mito_AF", - "gnomad_mito_AF_het", - "gnomad_mito_AN", - "gnomad_mito_max_hl", - "hap_defining_variant", - "helix_AC", - "helix_AC_het", - "helix_AF", - "helix_AF_het", - "helix_max_hl", - "high_constraint_region", - "hmtvar_hmtVar", - "mitimpact_apogee", - "mitomap_pathogenic", - "mitotip_mitoTIP", - "ref", - "rg37_locus", - "rsid", - "samples_num_alt_1", - "samples_num_alt_2", - "sortedTranscriptConsequences", - "start", - "variantId", - "xpos", - "xstop", -] - -MITO_SOURCE_ONLY_FIELDS = [ - 'callset_max_hl', - 'exac_max_hl', - 'gnomad_exomes_max_hl', - 'gnomad_genomes_max_hl', - 'gnomad_svs_max_hl', - 'sv_callset_max_hl', - 'topmed_max_hl', - 'helix_ID', - 'helix_Hemi', - 'helix_AN', - 'helix_filter_AF', - 'helix_Het', - 'helix_Hom', - 'gnomad_mito_filter_AF', - 'gnomad_mito_Hom', - 'gnomad_mito_ID', - 'gnomad_mito_Het', - 'gnomad_mito_Hemi', - 'callset_heteroplasmy_Het', - 'callset_heteroplasmy_ID', - 'callset_heteroplasmy_Hemi', - 'callset_heteroplasmy_filter_AF', - 'callset_heteroplasmy_Hom', - 'callset_heteroplasmy_max_hl', - 'gnomad_mito_heteroplasmy_filter_AF', - 'gnomad_mito_heteroplasmy_Hemi', - 'gnomad_mito_heteroplasmy_Hom', - 'gnomad_mito_heteroplasmy_ID', - 'gnomad_mito_heteroplasmy_Het', - 'helix_heteroplasmy_Het', - 'helix_heteroplasmy_ID', - 'helix_heteroplasmy_Hemi', - 'helix_heteroplasmy_filter_AF', - 'helix_heteroplasmy_Hom', -] - -SOURCE_FIELDS = { - 'homozygote_count', 'callset_Hemi', 'callset_Het', 'callset_ID', 'sv_callset_Hemi', - 'sv_callset_Hom', 'sv_callset_Het', 'sv_callset_ID', 'algorithms', -} -SOURCE_FIELDS.update(MAPPING_FIELDS) -SOURCE_FIELDS.update(SV_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_SOURCE_ONLY_FIELDS) -SOURCE_FIELDS -= { - 'samples_no_call', 'samples_cn_0', 'samples_cn_1', 'samples_cn_2', 'samples_cn_3', 'samples_cn_gte_4', 'topmed_Het', - 'gnomad_genomes_FAF_AF', -} - -FIELD_TYPE_MAP = { - 'cadd_PHRED': {'type': 'keyword'}, - 'primate_ai_score': {'type': 'float'}, - 'rg37_locus': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}}, - 'rg37_locus_end': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}} -} -MAPPING_PROPERTIES = {field: FIELD_TYPE_MAP.get(field, {'type': 'keyword'}) for field in MAPPING_FIELDS} - -CORE_INDEX_METADATA = { - INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'clinvar_version': '2023-03-05'}, - 'properties': MAPPING_PROPERTIES, - }, - SECOND_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'VARIANTS'}, - 'properties': MAPPING_PROPERTIES, - }, - SV_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'SV'}, - 'properties': {field: {'type': 'keyword'} for field in SV_MAPPING_FIELDS}, - }, - MITO_WGS_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'MITO'}, - 'properties': {field: {'type': 'keyword'} for field in MITO_MAPPING_FIELDS}, - }, -} -INDEX_METADATA = { - HG38_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SECOND_INDEX_NAME]), - SV_WGS_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SV_INDEX_NAME]), -} -for meta in INDEX_METADATA.values(): - meta['_meta']['genomeVersion'] = '38' -INDEX_METADATA.update(CORE_INDEX_METADATA) - -ALL_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples': ['HG00731', 'HG00732', 'HG00733']}}, - ]}} - ], - '_name': 'F000002_2' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20870']}}, - {'terms': {'samples_num_alt_2': ['NA20870']}}, - {'terms': {'samples': ['NA20870']}}, - ]}} - ], - '_name': 'F000003_3' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20874']}}, - {'terms': {'samples_num_alt_2': ['NA20874']}}, - {'terms': {'samples': ['NA20874']}}, - ]}} - ], - '_name': 'F000005_5' - }} - ] - } -} - -COMPOUND_HET_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -COMPOUND_HET_PATH_INHERITANCE_QUERY = deepcopy(COMPOUND_HET_INHERITANCE_QUERY) -for fam_q in COMPOUND_HET_PATH_INHERITANCE_QUERY['bool']['should']: - fam_quality_q = fam_q['bool']['must'][1] - fam_quality_q['bool'] = {'should': [ - deepcopy(fam_quality_q), - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}} - ]} - -RECESSIVE_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -ANNOTATION_QUERY = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}} - -REDIS_CACHE = {} -def _set_cache(k, v): - REDIS_CACHE[k] = v -MOCK_REDIS = mock.MagicMock() -MOCK_REDIS.get.side_effect = REDIS_CACHE.get -MOCK_REDIS.set.side_effect =_set_cache - -def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=None, index=INDEX_NAME): - parsed_hits = deepcopy(hits) - for hit in parsed_hits: - hit.update({ - '_index': index, - '_id': hit['_source']['variantId'], - '_type': 'structural_variant' if SV_INDEX_NAME in index or SV_WGS_INDEX_NAME in index else 'variant', - }) - matched_queries = hit.pop('matched_queries') - if include_matched_queries: - hit['matched_queries'] = [] - for subindex in index.split(','): - if subindex in matched_queries: - hit['_index'] = subindex - hit['matched_queries'] += matched_queries[subindex] - - if sort or increment_sort: - sort_key = sort[0] if sort else 'xpos' - if isinstance(sort_key, dict): - if '_script' in sort_key: - sort_key = sort_key['_script']['script']['params'].get('field', 'xpos') # pylint: disable=invalid-sequence-index - else: - sort_key = next(iter(sort_key.keys())) - sort_value = jmespath.search(sort_key, hit['_source']) - if sort_value is None: - sort_value = 'Infinity' - if increment_sort: - sort_value += 100 - hit['_sort'] = [sort_value] - return parsed_hits - - -def create_mock_response(search, index=INDEX_NAME): - index = ALIAS_MAP.get(index, index) - indices = index.split(',') - include_matched_queries = False - variant_id_filters = None - gene_ids_filters = set() - if 'query' in search: - for search_filter in search['query']['bool']['filter']: - if not variant_id_filters: - variant_id_filters = search_filter.get('terms', {}).get('variantId') - if not gene_ids_filters and not search.get('aggs'): - gene_ids_filters.update(search_filter.get('terms', {}).get('geneIds') or []) - possible_inheritance_filters = search_filter.get('bool', {}).get('should', []) + [search_filter] - if any('_name' in possible_filter.get('bool', {}) for possible_filter in possible_inheritance_filters): - include_matched_queries = True - break - - response_dict = { - 'took': 1, - 'hits': {'total': {'value': 5}, 'hits': []} - } - for index_name in sorted(indices): - index_hits = mock_hits( - INDEX_ES_VARIANTS[index_name], include_matched_queries=include_matched_queries, sort=search.get('sort'), - index=index_name) - if variant_id_filters: - index_hits = [hit for hit in index_hits if hit['_id'] in variant_id_filters] - elif gene_ids_filters: - index_hits = [hit for hit in index_hits if any( - gene_ids_filters.intersection({t['gene_id'] for t in hit['_source']['sortedTranscriptConsequences']}) - )] - response_dict['hits']['hits'] += index_hits - - try: - response_dict['hits']['hits'] = sorted(response_dict['hits']['hits'], key=lambda v: v['_sort']) - except (KeyError, TypeError): - pass - - if search.get('aggs'): - index_vars = COMPOUND_HET_INDEX_VARIANTS.get(index, {}) - buckets = [{'key': gene_id, 'doc_count': 3} for gene_id in ['ENSG00000135953', 'ENSG00000228198']] - if search['aggs']['genes']['aggs'].get('vars_by_gene'): - for bucket in buckets: - bucket['vars_by_gene'] = { - 'hits': { - 'hits': mock_hits(index_vars.get(bucket['key'], ES_VARIANTS), increment_sort=True, index=index) - }} - else: - for bucket in buckets: - doc_count = 0 - for sample_field in ['samples', 'samples_num_alt_1', 'samples_num_alt_2']: - gene_samples = defaultdict(int) - for var in index_vars.get(bucket['key'], ES_VARIANTS): - for sample in var['_source'].get(sample_field, []): - gene_samples[sample] += 1 - bucket[sample_field] = {'buckets': [{'key': k, 'doc_count': v} for k, v in gene_samples.items()]} - doc_count += sum(gene_samples.values()) - bucket['doc_count'] = doc_count - - response_dict['aggregations'] = {'genes': {'buckets': buckets}} - - if len(response_dict['hits']['hits']) == 0: - response_dict['hits']['total']['value'] = 0 - elif gene_ids_filters == {'ENSG00000186092'}: - response_dict['hits']['total']['value'] = 1 - - - return response_dict - -def get_indices_from_url(url): - return url.split('/')[1] - -def get_metadata_callback(request): - indices = get_indices_from_url(request.url).split(',') - response = {index: {'mappings': INDEX_METADATA[index]} for index in indices} - return 200, {}, json.dumps(response) - -def get_search_callback(request): - body = json.loads(request.body) - response = create_mock_response(body, get_indices_from_url(request.url)) - return 200, {}, json.dumps(response) - -def parse_msearch_body(body): - return [json.loads(row) for row in body.decode().split('\n') if row] - -def get_msearch_callback(request): - body = parse_msearch_body(request.body) - response = { - 'responses': [ - create_mock_response(exec_search, index=','.join(body[i-1]['index'])) - for i, exec_search in enumerate(body) if not exec_search.get('index')] - } - return 200, {}, json.dumps(response) - -def setup_search_responses(): - urllib3_responses.add_callback( - urllib3_responses.POST, '/_msearch', callback=get_msearch_callback, - content_type='application/json', match_querystring=True) - urllib3_responses.add_callback( - urllib3_responses.POST, re.compile('^/[,\w]+/_search$'), callback=get_search_callback, - content_type='application/json', match_querystring=True) - -def setup_responses(): - urllib3_responses.add_callback( - urllib3_responses.GET, re.compile('^/[,\w]+/_mapping$'), callback=get_metadata_callback, - content_type='application/json', match_querystring=True) - setup_search_responses() - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) -class EsUtilsTest(TestCase): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - Sample.objects.filter(sample_id='NA19678').update(is_active=False) - Sample.objects.filter(sample_id='NA20885').update(sample_type='WES') - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - - def assertExecutedSearch(self, filters=None, start_index=0, size=2, index=INDEX_NAME, expected_source_fields=SOURCE_FIELDS, call_index=-1, **kwargs): - executed_search = urllib3_responses.call_request_json(index=call_index) - searched_indices = get_indices_from_url(urllib3_responses.calls[call_index].request.url) - self.assertListEqual(sorted(searched_indices.split(',')), sorted(index.split(','))) - self.assertSameSearch( - executed_search, - dict(filters=filters, start_index=start_index, size=size, **kwargs), - expected_source_fields=expected_source_fields, - ) - - def assertExecutedSearches(self, searches): - executed_search = parse_msearch_body(urllib3_responses.calls[-1].request.body) - self.assertEqual(len(executed_search), len(searches) * 2) - for i, expected_search in enumerate(searches): - self.assertDictEqual(executed_search[i * 2], {'index': expected_search.get('index', INDEX_NAME).split(',')}) - self.assertSameSearch(executed_search[(i * 2) + 1], expected_search) - - def assertSameSearch(self, executed_search, expected_search_params, expected_source_fields=SOURCE_FIELDS): - expected_search = { - 'from': expected_search_params['start_index'], - 'size': expected_search_params['size'] - } - - if expected_search_params['filters']: - expected_search['query'] = { - 'bool': { - 'filter': expected_search_params['filters'] - } - } - - if expected_search_params.get('query'): - expected_search['query']['bool']['must'] = expected_search_params['query'] - - if not expected_search_params.get('unsorted'): - expected_search['sort'] = expected_search_params.get('sort') or ['xpos', 'variantId'] - - if expected_search_params.get('gene_aggs'): - expected_search['aggs'] = { - 'genes': {'terms': {'field': 'geneIds', 'min_doc_count': 2, 'size': 1001}, 'aggs': { - 'vars_by_gene': { - 'top_hits': {'sort': expected_search['sort'], '_source': mock.ANY, 'size': 100} - } - }}} - elif expected_search_params.get('gene_count_aggs'): - expected_search['aggs'] = {'genes': { - 'terms': {'field': 'mainTranscript_gene_id', 'size': 1001}, - 'aggs': expected_search_params['gene_count_aggs'] - }} - del expected_search['sort'] - else: - expected_search['_source'] = mock.ANY - self.assertDictEqual(executed_search, expected_search) - - if not expected_search_params.get('gene_count_aggs'): - source = executed_search['aggs']['genes']['aggs']['vars_by_gene']['top_hits']['_source'] \ - if expected_search_params.get('gene_aggs') else executed_search['_source'] - self.assertSetEqual(expected_source_fields, set(source)) - - def assertCachedResults(self, results_model, expected_results, sort='xpos'): - cache_key = 'search_results__{}__{}'.format(results_model.guid, sort) - self.assertIn(cache_key, REDIS_CACHE.keys()) - self.assertDictEqual(json.loads(REDIS_CACHE[cache_key]), expected_results) - MOCK_REDIS.expire.assert_called_with(cache_key, timedelta(weeks=2)) - - @urllib3_responses.activate - def test_get_es_variants_for_variant_ids(self): - setup_responses() - samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') - get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], - size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ) - - @urllib3_responses.activate - def test_get_single_es_variant(self): - setup_responses() - family = self.families.filter(guid='F000003_3').first() - variant = get_single_variant(family, '2-103343353-GAGA-G') - self.assertDictEqual(variant, {**PARSED_ANY_AFFECTED_VARIANTS[1], 'selectedMainTranscriptId': None}) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['2-103343353-GAGA-G']}}], - size=1, index=INDEX_NAME - ) - - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, 'prefix_19107_DEL') - self.assertDictEqual(variant, PARSED_SV_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['prefix_19107_DEL']}}], size=1, index=SV_INDEX_NAME, - ) - - variant = get_single_variant(family, 'M-10195-C-A') - self.assertDictEqual(variant, PARSED_MITO_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['M-10195-C-A']}}], size=1, index=MITO_WGS_INDEX_NAME, - ) - - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G') - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_gene_agg_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_search.logger') - @urllib3_responses.activate - def test_invalid_get_es_variants(self, mock_logger): - setup_responses() - mito_sample = Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME) - mito_sample.individual_id = 6 - mito_sample.save() - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - - results_model.families.set(self.families) - Sample.objects.filter(elasticsearch_index=INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), 'The following indices do not have the expected genome version 37: test_index_hg38 (38)', - ) - Sample.objects.filter(elasticsearch_index=HG38_INDEX_NAME).update(elasticsearch_index=INDEX_NAME) - - results_model.families.set(self.families) - search_model.search = { - 'inheritance': {'mode': 'compound_het'}, - 'locus': {'rawItems': 'WASH7P'}, - 'annotations': {'frameshift': ['frameshift_variant']}, - } - search_model.save() - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), - 'This search returned too many compound heterozygous variants. Please add stricter filters') - - with self.assertRaises(InvalidSearchException) as cm: - get_variant_query_gene_counts(results_model, None) - self.assertEqual(str(cm.exception), 'This search returned too many genes') - - search_model.search = {'qualityFilter': {'min_gq': 7}} - search_model.save() - with self.assertRaises(Exception) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Invalid gq filter 7') - - search_model.search = {} - search_model.save() - urllib3_responses.reset() - urllib3_responses.add( - urllib3_responses.POST, '/_msearch', body=ReadTimeoutError('', '', 'timeout')) - urllib3_responses.add_json('/_tasks?actions=*search&group_by=parents', {'tasks': { - 123: {'running_time_in_nanos': 10}, - 456: {'running_time_in_nanos': 10 ** 12}, - }}) - with self.assertRaises(ConnectionTimeout): - query_variants(results_model) - self.assertListEqual( - [call.request.url for call in urllib3_responses.calls], - ['/_msearch', '/_tasks?actions=%2Asearch&group_by=parents']) - mock_logger.error.assert_called_with('ES Query Timeout: Found 1 long running searches', None, detail=[ - {'task': {'running_time_in_nanos': 10 ** 12}, 'parent_task_id': '456'}, - ]) - - urllib3_responses.reset() - urllib3_responses.add_json('/_msearch', {'responses': [ - {'error': {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}} - ]}, method=urllib3_responses.POST) - - with self.assertRaises(TransportError) as cm: - query_variants(results_model) - self.assertDictEqual( - cm.exception.info, - {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}) - - _set_cache(f'index_metadata__{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}', None) - urllib3_responses.add( - urllib3_responses.GET, f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', body=Exception('Connection error')) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'test_index,test_index_mito_wgs,test_index_sv - Error accessing index: Connection error') - - urllib3_responses.replace_json(f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', {}) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') - - @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) - @urllib3_responses.activate - def test_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={'annotations': {'frameshift': ['frameshift_variant']}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_VARIANTS[1]) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY]) - - # does not save non-consecutive pages - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=2) - - # test pagination - variants, total_results = query_variants(results_model, page=2, num_results=2) - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': PARSED_VARIANTS + PARSED_VARIANTS, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=2, size=2) - - # test does not re-fetch page - urllib3_responses.reset() - variants, total_results = query_variants(results_model, page=1, num_results=3) - self.assertEqual(len(variants), 3) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS[:1]) - self.assertEqual(total_results, 5) - - # test load_all - setup_responses() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - # test does not re-fetch once all loaded - urllib3_responses.reset() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - @urllib3_responses.activate - def test_filtered_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'pathogenicity': { - 'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting'], - 'hgmd': ['disease_causing', 'likely_disease_causing'], - }, - 'annotations': { - 'in_frame': ['inframe_insertion', 'inframe_deletion'], - 'other': ['5_prime_UTR_variant', 'intergenic_variant'], - 'SCREEN': ['dELS', 'DNase-only'], - 'splice_ai': '0.8', - }, - 'freqs': { - 'callset': {'af': 0.1}, - 'exac': {'ac': 2, 'af': None}, - 'g1k': {'ac': None, 'af': 0.001}, - 'gnomad_exomes': {'af': 0.01, 'ac': 3, 'hh': 3}, - 'gnomad_genomes': {'af': 0.01, 'hh': 3}, - 'topmed': {'ac': 2, 'af': None}, - }, - 'qualityFilter': {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass', 'affected_only': True}, - 'in_silico': {'cadd': '11.5', 'sift': 'D', 'fathmm': 'D'}, - 'inheritance': {'mode': 'de_novo'}, - 'customQuery': {'term': {'customFlag': 'flagVal'}}, - 'exclude': {'rawItems': 'WASH7P, chr2:1234-5678, chr7:100-10100%10'}, - }) - - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - variants, total_results = query_variants(results_model, sort='cadd', num_results=2) - - self.assertListEqual(variants, PARSED_CADD_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(filters=[ - {'term': {'customFlag': 'flagVal'}}, - { - 'bool': { - 'must_not': [ - {'range': {'xpos': {'gte': 2000001234, 'lte': 2000005678}}}, - {'range': {'xstop': {'gte': 2000001234, 'lte': 2000005678}}}, - {'bool': {'must': [ - {'range': {'xpos': {'lte': 2000001234}}}, - {'range': {'xstop': {'gte': 2000005678}}}, - {'range': {'xpos': {'gte': 2000000001}}}, - {'range': {'xstop': {'lte': 2300000000}}}, - ]}}, - {'terms': {'geneIds': ['ENSG00000227232']}}, - {'bool': {'must': [ - {'range': {'xpos': {'gte': 7000000001, 'lte': 7000001100}}}, - {'range': {'xstop': {'gte': 7000009100, 'lte': 7000011100}}}]}}, - ] - } - }, - {'bool': {'should': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'exac_AC_Adj'}}]}}, - {'range': {'exac_AC_Adj': {'lte': 2}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hom'}}]}}, - {'range': {'gnomad_exomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hemi'}}]}}, - {'range': {'gnomad_exomes_Hemi': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hom'}}]}}, - {'range': {'gnomad_genomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hemi'}}]}}, - {'range': {'gnomad_genomes_Hemi': {'lte': 3}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'topmed_AC'}}]}}, - {'range': {'topmed_AC': {'lte': 2}}} - ]} - } - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }} - ] - }}, - - ]}}, - {'bool': {'should': [ - {'bool': {'must_not': [{'exists': {'field': 'cadd_PHRED'}}]}}, - {'range': {'cadd_PHRED': {'gte': 11.5}}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_SIFT_pred'}}]}}, - {'prefix': {'dbnsfp_SIFT_pred': 'D'}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_fathmm_MKL_coding_pred'}}]}}, - {'prefix': {'dbnsfp_fathmm_MKL_coding_pred': 'D'}}, - ]}}, - {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}}, - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'transcriptConsequenceTerms'}}]}}, - {'terms': { - 'transcriptConsequenceTerms': [ - '5_prime_UTR_variant', - 'inframe_deletion', - 'inframe_insertion', - 'intergenic_variant', - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*|Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))', - }}, - {'terms': {'hgmd_class': ['DM', 'DM?']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.8}}}, - {'terms': {'screen_region_type': ['dELS', 'DNase-only']}}, - ] - } - }, - {'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - }}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'HG00731'}}, - {'term': {'samples_ab_5_to_10': 'HG00731'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'HG00731'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_10_to_15': 'HG00731'}}, - ], - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*' - }}]}} - ], - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}} - ]}}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'NA20870'}}, - {'term': {'samples_ab_5_to_10': 'NA20870'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'NA20870'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - {'term': {'samples_gq_10_to_15': 'NA20870'}}, - ] - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }}]}} - ], - '_name': 'F000003_3' - }}, - ] - }} - ], sort=[{'cadd_PHRED': {'order': 'desc', 'unmapped_type': 'keyword'}}, 'xpos', 'variantId']) - - # Test sort does not error on pagination - query_variants(results_model, sort='cadd', num_results=2, page=2) - - @urllib3_responses.activate - def test_sv_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'new_structural_variants': ['NEW']}, - 'freqs': {'sv_callset': {'af': 0.1}}, - 'in_silico': {'strvctvre': '3.1', 'requireScore': True}, - 'qualityFilter': {'min_qs': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - - self.assertListEqual(variants, [PARSED_SV_VARIANT]) - self.assertExecutedSearch(filters=[ - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'sf'}}]}}, - {'range': {'sf': {'lte': 0.1}}} - ] - }}, - {'range': {'StrVCTVRE_score': {'gte': 3.1}}}, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_qs_0_to_10': 'HG00731'}}, - {'term': {'samples_qs_10_to_20': 'HG00731'}}, - {'term': {'samples_qs_0_to_10': 'HG00732'}}, - {'term': {'samples_qs_10_to_20': 'HG00732'}}, - {'term': {'samples_qs_0_to_10': 'HG00733'}}, - {'term': {'samples_qs_10_to_20': 'HG00733'}}, - ], - 'must': [{'terms': {'samples_new_call': ['HG00731', 'HG00732', 'HG00733']}}], - }}, - ], - '_name': 'F000002_2' - }} - ], index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_sv_wgs_get_es_variants(self): - self.families = Family.objects.filter(guid='F000014_14') - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DUP', 'CPX']}, - 'qualityFilter': {'min_gq_sv': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - self.assertListEqual(variants, [PARSED_SV_WGS_VARIANT]) - - self.assertExecutedSearch(filters=[ - {'terms': {'transcriptConsequenceTerms': ['CPX', 'DUP']}}, - {'bool': { - 'must': [{'bool': {'must': [{'term': {'samples': 'NA21234'}}, {'term': {'samples': 'NA21987'}}], - 'must_not': [{'term': {'samples': 'NA21654'}}]}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_sv_0_to_5': 'NA21234'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21234'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21234'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21234'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21654'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21654'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21654'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21654'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21987'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21987'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21987'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21987'}}, - ], - }} - ], - '_name': 'F000014_14' - }} - ], index=SV_WGS_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_get_es_variants(self): - setup_responses() - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, [PARSED_SV_VARIANT] + PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - self.assertExecutedSearches([ - dict(filters=[], start_index=0, size=5, index=SV_INDEX_NAME), - dict(filters=[], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['pathogenicity'] = {'clinvar': ['pathogenic']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - path_filter = {'regexp': { - 'clinvar_clinical_significance': '.*Pathogenic.*' - }} - self.assertExecutedSearches([ - dict(filters=[path_filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[path_filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - # test with dataset filtering applied - search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - filter = {'bool': {'should': [{'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}}, path_filter]}} - self.assertExecutedSearches([ - dict(filters=[filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['annotations'] = { - 'structural': ['DEL'], 'structural_consequence': ['MSV_EXON_OVERLAP', 'INTRAGENIC_EXON_DUP']} - search_model.search['pathogenicity'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - self.assertExecutedSearch( - filters=[{'terms': { - 'transcriptConsequenceTerms': ['DEL', 'DUP_LOF', 'INTRAGENIC_EXON_DUP', 'MSV_EXON_OVERLAP', 'MSV_EXON_OVR'] - }}], - start_index=0, size=5, index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_no_affected_inheritance_get_es_variants(self): - setup_responses() - # The family has multiple data types loaded but only one loaded in an affected individual - Sample.objects.get(individual_id=4, elasticsearch_index=SV_INDEX_NAME).delete() - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - - search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - - query_variants(results_model, num_results=2) - self.assertExecutedSearch(filters=[{'bool': { - 'must': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], - }}], '_name': 'F000002_2', - }}]) - - @urllib3_responses.activate - def test_compound_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_compound_het_get_es_variants_secondary_annotation(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant'], 'splice_ai': '0.5'}, - 'inheritance': {'mode': 'compound_het'}, - 'annotations_secondary': {'other': ['intron']}, - 'pathogenicity': {'clinvar': ['pathogenic'], 'hgmd': ['disease_causing']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - annotation_query = {'bool': {'should': [ - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}}, - {'terms': {'hgmd_class': ['DM']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.5}}}, - {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}}, - ]}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_PATH_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - # variants require both primary and secondary annotations - setup_responses() - del search_model.search['pathogenicity'] - del search_model.search['annotations']['splice_ai'] - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertIsNone(variants) - self.assertEqual(total_results, 0) - - annotation_query = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - @urllib3_responses.activate - def test_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10, 'vcf_filter': 'pass'}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS[1]) - self.assertEqual(total_results, 6) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 6, - }) - - pass_filter_query = {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}} - - self.assertExecutedSearches([ - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=0, size=2, - ), - ]) - - # test pagination - - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_MULTI_SAMPLE_VARIANT) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}, - {'null': [PARSED_VARIANTS[0]]}, {'null': [PARSED_MULTI_SAMPLE_VARIANT]}], - 'duplicate_doc_count': 1, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 4, 'total': 5}}, - 'total_results': 5, - }) - - self.assertExecutedSearches([dict(filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=2, size=4)]) - - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_multi_datatype_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'recessive'}, - 'annotations': {'frameshift': ['frameshift_variant'], 'structural': ['DEL']} - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 4) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[2][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[2][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - self.assertDictEqual(variants[3], PARSED_VARIANTS[1]) - - annotations_q = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - self.assertExecutedSearches([ - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotations_q, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - annotations_q, - { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - }}, - ] - } - } - ], start_index=0, size=10, - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL']}, - 'annotations_secondary': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - - annotation_secondary_query = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - - self.assertExecutedSearches([ - dict( - filters=[ - {'terms': {'transcriptConsequenceTerms': ['DEL']}}, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_comp_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL'], 'SCREEN': ['dELS']}, - 'annotations_secondary': {'structural_consequence': ['LOF']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - query_variants(results_model, num_results=10) - - annotation_secondary_query = {'bool': {'should': [ - {'terms': {'transcriptConsequenceTerms': ['DEL', 'LOF']}}, - {'terms': {'screen_region_type': ['dELS']}}]}} - - self.assertExecutedSearches([ - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(index=INDEX_NAME, size=2, filters=[ - {'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G']}}, ANNOTATION_QUERY]) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_ANY_AFFECTED_VARIANTS) - self.assertEqual(total_results, 10) - - self.assertExecutedSearches([dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], index=MITO_WGS_INDEX_NAME, start_index=0, size=2), dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], index=INDEX_NAME, start_index=0, size=2)]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_SEARCH_CLAUSES', 1) - @urllib3_responses.activate - def test_many_family_inheitance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 14) - self.assertDictEqual(variants[0], PARSED_MULTI_SAMPLE_VARIANT_0) - self.assertListEqual(variants[1], PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_VARIANT, PARSED_MITO_VARIANT], - 'grouped_results': [{'null': [PARSED_MULTI_SAMPLE_VARIANT_0]}, {'ENSG00000228198': PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, - MITO_WGS_INDEX_NAME: {'loaded': 1, 'total': 5}, - INDEX_NAME: {'loaded': 4, 'total': 10}}, - 'total_results': 14, - }) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - ] - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}], - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - } - }, - ], - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - } - }, - ], - start_index=0, - size=2, - ), - ]) - - @urllib3_responses.activate - def test_multi_project_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2'])) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[1]) - self.assertEqual(total_results, 11) - - self.assertCachedResults(results_model, { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 2, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 11, - }) - self.assertTrue('index_metadata__{}'.format(INDEX_NAME) in REDIS_CACHE) - self.assertTrue('index_metadata__{}'.format(SECOND_INDEX_NAME) in REDIS_CACHE) - - project_2_search = dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME) - project_1_search = dict( - filters=[ - ANNOTATION_QUERY, - RECESSIVE_INHERITANCE_QUERY, - ], start_index=0, size=2, index=INDEX_NAME) - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - '_name': 'F000011_11', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}} - ]}} - ] - }} - ], - gene_aggs=True, start_index=0, size=1, index=SECOND_INDEX_NAME, - ), - project_2_search, - dict( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, start_index=0, size=1, index=INDEX_NAME, - ), - project_1_search, - ]) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_VARIANTS[0], PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) - self.assertEqual(total_results, 9) - - cache_results = { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}, - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT} - ], - 'duplicate_doc_count': 4, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 4, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 9, - } - self.assertCachedResults(results_model, cache_results) - - project_2_search['start_index'] = 1 - project_2_search['size'] = 3 - project_1_search['start_index'] = 2 - self.assertExecutedSearches([project_2_search, project_1_search]) - - # If one project is fully loaded, only query the second project - cache_results['loaded_variant_counts'][INDEX_NAME]['total'] = 4 - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(cache_results)) - query_variants(results_model, num_results=2, page=3) - project_2_search['start_index'] = 2 - project_2_search['size'] = 4 - self.assertExecutedSearches([project_2_search]) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_MULTI_INDEX_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 4) - - cached_variants = expected_variants + [PARSED_MITO_VARIANT] - self.assertCachedResults(results_model, { - 'all_results': cached_variants, - 'duplicate_doc_count': 1, - 'total_results': 4, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME, MITO_WGS_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=6, - ) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - expected_variants = [PARSED_MITO_VARIANT, PARSED_VARIANTS[0]] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 3) - - self.assertCachedResults(results_model, { - 'all_results': cached_variants + cached_variants, - 'duplicate_doc_count': 2, - 'total_results': 3, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=8, - start_index=4, - ) - - # test skipping page fetches all consecutively - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, page=2) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=12, - ) - - @urllib3_responses.activate - def test_multi_project_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }, - ) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 14) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_SEARCHES', 1) - @urllib3_responses.activate - def test_multi_project_prefilter_indices_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'de_novo'}, - 'qualityFilter': {'min_gq': 10}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 14) - self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) - - gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} - prefilter_search = dict( - filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', - size=200, expected_source_fields=set(), - ) - sv_search = dict( - filters=[ - gene_filter, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }} - ], start_index=0, size=2, index=SV_INDEX_NAME) - num_calls = 3 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - # Search total is greater than returned hits, so proceed with regular multi-search - self.assertExecutedSearches([ - sv_search, - dict(filters=[ - gene_filter, - { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict(filters=[ - gene_filter, - {'bool': {'should': [ - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA19675'}}, - {'term': {'samples_num_alt_2': 'NA19675'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA19675'}}, - {'term': {'samples_gq_5_to_10': 'NA19675'}}, - ]}}], '_name': 'F000001_1'}}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], 'minimum_should_match': 1}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - ]}}], '_name': 'F000003_3'}}, - ]}}], start_index=0, size=2, index=INDEX_NAME), - ]) - - # Test successful prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000186092'] - search_model.search['locus']['rawItems'] = 'ENSG00000186092' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 1) - num_calls += 2 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - sv_search['query'] = [{'ids': {'values': ['prefix_19107_DEL']}}] - self.assertExecutedSearches([sv_search]) - - # Test no results in prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000269732'] - search_model.search['locus']['rawItems'] = 'ENSG00000269732' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 0) - # Only the prefliter search is run, no multi-search - self.assertEqual(len(urllib3_responses.calls), num_calls + 1) - self.assertExecutedSearch(**prefilter_search) - - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_VARIANTS', 3) - @urllib3_responses.activate - def test_skip_genotype_filter(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, _ = query_variants(results_model, num_results=2, skip_genotype_filter=True) - expected_transcript_variant = deepcopy(PARSED_VARIANTS[0]) - expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] - self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), - filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], - size=3, - ) - - # test with inheritance override - search_model.search['inheritance'] = {'mode': 'any_affected'} - search_model.search['locus'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, skip_genotype_filter=True) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.utils.LIFTOVERS', {'37': None}) - @mock.patch('seqr.utils.search.utils.LiftOver') - @urllib3_responses.activate - def test_get_lifted_grch38_variants(self, mock_liftover): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - Project.objects.filter(guid='R0003_test').update(genome_version='38') - Sample.objects.filter(elasticsearch_index=SECOND_INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - - mock_liftover.side_effect = Exception() - expected_no_lift_grch38_variant = deepcopy(PARSED_HG38_VARIANT) - expected_no_lift_grch38_variant.update({ - 'liftedOverGenomeVersion': None, - 'liftedOverChrom': None, - 'liftedOverPos': None, - }) - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, expected_no_lift_grch38_variant]) - self.assertIsNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - mock_liftover.side_effect = None - mock_liftover.return_value.convert_coordinate.side_effect = lambda chrom, pos: [[chrom, pos - 10]] - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, PARSED_HG38_VARIANT]) - self.assertIsNotNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_NAME_LENGTH', 30) - @urllib3_responses.activate - def test_get_es_variants_create_index_alias(self): - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - setup_search_responses() - urllib3_responses.add_json( - '/{}/_mapping'.format(INDEX_ALIAS), {k: {'mappings': v} for k, v in CORE_INDEX_METADATA.items()}) - urllib3_responses.add_json('/_aliases', {'success': True}, method=urllib3_responses.POST) - - query_variants(results_model, num_results=2) - - self.assertExecutedSearch(index=INDEX_ALIAS, size=8) - self.assertDictEqual(urllib3_responses.call_request_json(index=0), { - 'actions': [{'add': {'indices': [INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME], 'alias': INDEX_ALIAS}}]}) - - @urllib3_responses.activate - def test_get_es_variants_search_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - setup_search_responses() - urllib3_responses.add_json('/{}/_mapping'.format(SECOND_INDEX_NAME), { - k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SUB_INDICES}) - - query_variants(results_model, num_results=2) - - expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - self.assertExecutedSearches([ - dict(index=SUB_INDICES[1], **expected_search), - dict(index=SUB_INDICES[0], **expected_search), - ]) - _set_cache('index_metadata__{}'.format(SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variants_search_multiple_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000011_11'])) - - setup_search_responses() - mappings = {k: {'mappings': INDEX_METADATA[INDEX_NAME]} for k in SUB_INDICES} - mappings.update({k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_mapping'.format(INDEX_NAME, SECOND_INDEX_NAME), mappings) - aliases = {k: {'aliases': {INDEX_NAME: {}}} for k in SUB_INDICES} - aliases.update({k: {'aliases': {SECOND_INDEX_NAME: {}, INDEX_ALIAS: {}}} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_alias'.format(INDEX_NAME, SECOND_INDEX_NAME), aliases) - - query_variants(results_model, num_results=2) - - second_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - first_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}} - ], - '_name': 'F000003_3' - }}] - } - self.assertExecutedSearches([ - dict(index=SECOND_SUB_INDICES[1], **second_alias_expected_search), - dict(index=SECOND_SUB_INDICES[0], **second_alias_expected_search), - dict(index=SUB_INDICES[1], **first_alias_expected_search), - dict(index=SUB_INDICES[0], **first_alias_expected_search), - ]) - _set_cache('index_metadata__{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000240361': PARSED_COMPOUND_HET_VARIANTS}], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 7, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000240361': {'total': 2, 'families': {'F000003_3': 2}}, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], - size=1, index=INDEX_NAME, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 4, 'loaded': 4}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 2, 'loaded': 2}, - }, - 'total_results': 13, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 4, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}} - }) - - expected_search = dict(size=1, start_index=0, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - self.assertExecutedSearches([ - dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], index=SECOND_INDEX_NAME, **expected_search), - dict(filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], index=INDEX_NAME, **expected_search), - ]) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps({'total_results': 5})) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}}, - 'ENSG00000228198': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}} - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts, 'total_results': 5}) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}}, - 'ENSG00000228198': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}} - }) - - self.assertExecutedSearches([dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ), dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - )]) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts}) - - def test_get_family_affected_status(self): - samples_by_id = { - sample_id: Sample.objects.get(sample_id=sample_id, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - for sample_id in ['HG00731', 'HG00732', 'HG00733'] - } - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - self.assertDictEqual(_get_family_affected_status(samples_by_id, {}), { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'N', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_multi_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - @urllib3_responses.activate - def test_sort(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, _ = query_variants(results_model, sort='primate_ai', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], sort=[ - {'primate_ai_score': {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}, 'xpos', 'variantId'], - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4) - self.assertEqual(variants[0]['_sort'][0], maxsize) - self.assertEqual(variants[1]['_sort'][0], -1) - - variants, _ = query_variants(results_model, sort='gnomad', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': {'field': 'gnomad_genomes_AF'}, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - self.assertEqual(variants[0]['_sort'][0], 0.00012925741614425127) - self.assertEqual(variants[1]['_sort'][0], maxsize) - - variants, _ = query_variants(results_model, sort='in_omim', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - results_model.families.set(Family.objects.filter(guid='F000001_1')) - query_variants(results_model, sort='prioritized_gene', num_results=2) - family_sample_filter = {'bool': {'_name': 'F000001_1', 'must': mock.ANY}} - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, family_sample_filter], index=INDEX_NAME, size=2, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'prioritized_ranks_by_gene': {'ENSG00000268903': 1, 'ENSG00000268904': 11} - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - @urllib3_responses.activate - def test_genotype_inheritance_filter(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - cache_key = 'search_results__{}__xpos'.format(results_model.guid) - - def _execute_inheritance_search( - mode=None, inheritance_filter=None, expected_filter=None, expected_comp_het_filter=None, - quality_filter=None, expected_quality_filter=None, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, **kwargs): - _set_cache(cache_key, None) - annotations = {'frameshift': ['frameshift_variant']} if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS \ - else {'structural': ['DEL', 'gCNV_DEL']} - search_model.search = { - 'inheritance': {'mode': mode, 'filter': inheritance_filter}, - 'annotations': annotations, - } - if quality_filter: - search_model.search['qualityFilter'] = quality_filter - search_model.save() - variants, _ = query_variants(results_model, num_results=2) - - if mode not in {'compound_het', 'recessive'}: - self.assertSetEqual( - set(variants[-1]['genotypes'].keys()), - {'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'}, - ) - - index = INDEX_NAME if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else SV_INDEX_NAME - annotation_query = {'terms': {'transcriptConsequenceTerms': next(iter(annotations.values()))}} - if expected_comp_het_filter: - self.assertExecutedSearches([ - dict(gene_aggs=True, start_index=0, size=1, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_comp_het_filter]}} - ]), - dict(start_index=0, size=2, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_filter]}}]) - ]) - else: - filters = [expected_filter] - if expected_quality_filter: - filters.append(expected_quality_filter) - self.assertExecutedSearch(index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': filters}}], **kwargs) - - # custom genotype - _execute_inheritance_search( - inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref', 'I000005_hg00732': 'ref_alt'}}, - expected_filter={ - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}} - ] - } - }) - - # de novo - _execute_inheritance_search(mode='de_novo', expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', dataset_type='SV', expected_filter={'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }, quality_filter={'affected_only': True, 'min_gq': 10}, expected_quality_filter={ - 'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - ]}} - ) - - _execute_inheritance_search( - mode='de_novo', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ], - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ]} - }] - } - }) - - recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - } - sv_recessive_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - } - custom_affected_recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - } - - # homozygous recessive - _execute_inheritance_search(mode='homozygous_recessive', expected_filter=recessive_filter) - - _execute_inheritance_search(mode='homozygous_recessive', dataset_type='SV', expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='homozygous_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_recessive_filter) - - # compound het - com_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - } - - sv_com_het_filter = {'term': {'samples': 'HG00731'}} - - custom_affected_comp_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='compound_het', expected_filter=com_het_filter, gene_aggs=True, size=1) - _execute_inheritance_search( - mode='compound_het', dataset_type='SV', gene_aggs=True, size=1, expected_filter=sv_com_het_filter) - _execute_inheritance_search( - mode='compound_het', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_comp_het_filter, gene_aggs=True, size=1) - - # x-linked recessive - x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - } - - sv_x_linked_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ], - 'must': [{'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}},], - } - } - - custom_affected_x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='x_linked_recessive', expected_filter=x_linked_filter) - - _execute_inheritance_search(mode='x_linked_recessive', dataset_type='SV', expected_filter=sv_x_linked_filter) - - _execute_inheritance_search( - mode='x_linked_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_x_linked_filter) - - # recessive - _execute_inheritance_search(mode='recessive', expected_comp_het_filter=com_het_filter, expected_filter=recessive_filter) - - _execute_inheritance_search( - mode='recessive', dataset_type='SV', expected_comp_het_filter=sv_com_het_filter, expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='recessive', inheritance_filter={'affected': custom_affected}, - expected_comp_het_filter=custom_affected_comp_het_filter, expected_filter=custom_affected_recessive_filter) - - # any affected - _execute_inheritance_search(mode='any_affected', expected_filter={ - 'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - } - }) - - _execute_inheritance_search( - mode='any_affected', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732']}}, - {'terms': {'samples': ['HG00731', 'HG00732']}}, - ] - } - }) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py deleted file mode 100644 index 72e25a157b..0000000000 --- a/seqr/utils/search/search_utils_tests.py +++ /dev/null @@ -1,749 +0,0 @@ -from datetime import timedelta -from django.contrib.auth.models import User -from django.test import TestCase -import json -import mock - -from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant, \ - GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, SV_VARIANT1 -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.utils.search.utils import get_single_variant, get_variant_query_gene_counts, \ - query_variants, variant_lookup, InvalidSearchException -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, \ - PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS - - -SV_SAMPLES = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] -MITO_SAMPLES = ['S000149_hg00733'] -NON_SNP_INDEL_SAMPLES = SV_SAMPLES + MITO_SAMPLES -FAMILY_3_SAMPLE = 'S000135_na20870' - -GENE_COUNTS = { - **GENE_COUNTS, - 'ENSG00000177000': {**GENE_COUNTS['ENSG00000177000'], 'total': 3}, - 'ENSG00000277258': {**GENE_COUNTS['ENSG00000277258'], 'total': 2}, -} - -class SearchTestHelper(object): - - def set_up(self): - patcher = mock.patch('seqr.utils.redis_utils.redis.StrictRedis') - self.mock_redis = patcher.start().return_value - self.addCleanup(patcher.stop) - - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - self.user = User.objects.get(username='test_user') - - self.search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': {'callset': {'ac': 1000}}}) - self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) - self.results_model.families.set(self.families) - - def set_cache(self, cached): - self.mock_redis.get.return_value = json.dumps(cached) - - def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): - cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' - self.mock_redis.set.assert_called_with(cache_key, mock.ANY) - self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) - self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - - -class SearchUtilsTests(SearchTestHelper): - - CACHED_VARIANTS = PARSED_VARIANTS + PARSED_VARIANTS - HAS_GENE_AGG = False - - def set_up(self): - super(SearchUtilsTests, self).set_up() - - self.non_affected_search_samples = Sample.objects.filter(guid__in=[ - 'S000137_na20874', - ]) - self.affected_search_samples = Sample.objects.filter(guid__in=[ - 'S000132_hg00731', 'S000133_hg00732', 'S000134_hg00733', FAMILY_3_SAMPLE, - 'S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733', - ]) - self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) - - def test_variant_lookup(self, mock_variant_lookup): - mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] - variants = variant_lookup(self.user, '1-10439-AC-A', '38', affected_only=True) - self.assertListEqual(variants, [VARIANT_LOOKUP_VARIANT]) - mock_variant_lookup.assert_called_with(self.user, '1-10439-AC-A', ('1', 10439, 'AC', 'A'), 'SNV_INDEL', None, '38', True, False) - cache_key = "variant_lookup_results__1-10439-AC-A__38" - self.assert_cached_results(variants, cache_key=f'{cache_key}__affected') - - mock_variant_lookup.reset_mock() - self.set_cache(variants) - cached_variant = variant_lookup(self.user, '1-10439-AC-A', '38') - self.assertListEqual(variants, cached_variant) - mock_variant_lookup.assert_not_called() - self.mock_redis.get.assert_called_with(cache_key) - - self.set_cache(None) - mock_variant_lookup.reset_mock() - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') - self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') - - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') - self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') - - mock_variant_lookup.return_value = [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT] - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', hom_only=True) - self.assertListEqual(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) - mock_variant_lookup.assert_called_with( - self.user, 'phase2_DEL_chr14_4640', None, 'SV', 'WGS', '38', False, True) - cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' - self.assert_cached_results(variants, cache_key=f'{cache_key}__hom') - - mock_variant_lookup.reset_mock() - self.set_cache(variants) - cached_variant = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self.assertListEqual(variants, cached_variant) - mock_variant_lookup.assert_not_called() - self.mock_redis.get.assert_called_with(cache_key) - - def test_get_single_variant(self, mock_get_variants_for_ids): - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self.assertDictEqual(variant, PARSED_VARIANTS[0]) - expected_samples = {s for s in self.affected_search_samples if s.guid not in NON_SNP_INDEL_SAMPLES and s.guid != FAMILY_3_SAMPLE} - self._assert_expected_get_single_variant_call( - mock_get_variants_for_ids, '2-103343353-GAGA-G', ('2', 103343353, 'GAGA', 'G'), expected_samples, - ) - - get_single_variant(family, 'prefix_19107_DEL', user=self.user) - expected_samples = { - s for s in self.search_samples if s.guid in ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] - } - self._assert_expected_get_single_variant_call( - mock_get_variants_for_ids, 'prefix_19107_DEL', None, expected_samples, dataset_type='SV', - ) - - mock_get_variants_for_ids.return_value = [] - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G') - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, variant_id, parsed_variant_id, expected_samples, **kwargs): - mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', [variant_id], self.user, - ) - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - - @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) - @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) - def _test_invalid_search_params(self, search_func): - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user, page=200) - self.assertEqual(str(cm.exception), 'Unable to load more than 10000 variants (20000 requested)') - - self.search_model.search['locus'] = {'rawVariantItems': 'chr2-A-C'} - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid variants: chr2-A-C') - - self.search_model.search['locus']['rawVariantItems'] = 'rs9876,chr2-1234-A-C' - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), self.INVALID_RSID_ERROR) - - self.search_model.search['locus']['rawItems'] = 'chr27:1234-5678,2:40-400000000, ENSG00012345' - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345') - - self.search_model.search['locus']['rawItems'] = '1:1-1000, 2:2000-3000, 3:4000-5000' - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Too many genes/intervals') - - build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' - self.search_model.search['locus']['rawItems'] = build_specific_genes - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') - - self.search_model.search['exclude'] = self.search_model.search['locus'] - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Cannot specify both Location and Excluded Genes/Intervals') - - self.search_model.search['locus'] = {} - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') - - self.search_model.search['pathogenicity'] = {'clinvar': ['pathogenic', 'vus']} - self.search_model.search['exclude'] = {'clinvar': ['benign', 'vus']} - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'ClinVar pathogenicity vus is both included and excluded') - - self.search_model.search['exclude'] = {} - self.search_model.search['inheritance'] = {'mode': 'recessive'} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Annotations must be specified to search for compound heterozygous variants') - - self.search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Location must be specified to search for compound heterozygous variants across many families') - - self.results_model.families.set([family for family in self.families if family.guid == 'F000005_5']) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Inheritance based search is disabled in families with no data loaded for affected individuals', - ) - - self.results_model.families.set([family for family in self.families if family.guid == 'F000003_3']) - self.search_model.search['annotations'] = {'structural': ['DEL']} - self.search_model.search['pathogenicity'] = {} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Unable to search against dataset type "SV"') - - self.search_model.search['annotations_secondary'] = {'frameshift': ['frameshift_variant']} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals', - ) - - self.search_model.search['inheritance']['filter'] = {'affected': {'I000007_na20870': 'N'}} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Inheritance based search is disabled in families with no data loaded for affected individuals', - ) - - self.search_model.search['inheritance']['mode'] = None - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Inheritance must be specified if custom affected status is set') - - self.search_model.search['inheritance']['filter'] = {'genotype': {'I000004_hg00731': 'ref_ref'}} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Invalid custom inheritance') - - self.results_model.families.set(Family.objects.filter(family_id='no_individuals')) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'No search data found for families no_individuals') - - self.results_model.families.set(Family.objects.all()) - self.search_model.search['annotations'] = None - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual( - str(cm.exception), - 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde, Test Reprocessed Project; 38 - Non-Analyst Project', - ) - - self._test_invalid_no_location_search_params() - - self.results_model.families.set(Family.objects.filter(guid='F000014_14')) - self.search_model.search['locus']['rawItems'] = build_specific_genes - with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') - - def _test_invalid_no_location_search_params(self): - self.search_model.search['inheritance'] = {} - self.results_model.families.set(Family.objects.filter(id__in=[2, 11])) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') - - self.search_model.search['freqs'] = {} - self.results_model.families.set(self.families) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'seqr AC frequency of at least 5000 must be specified to search across multiple families') - - - def test_invalid_search_query_variants(self): - with self.assertRaises(InvalidSearchException) as se: - query_variants(self.results_model, sort='prioritized_gene', num_results=2) - self.assertEqual(str(se.exception), 'Phenotype sort is only supported for single-family search.') - - self.set_cache({'total_results': 20000}) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, page=1, num_results=2, load_all=True) - self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') - - self._test_invalid_search_params(query_variants) - - def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, - rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', - dataset_type=None, secondary_dataset_type=None, exclude_keys=None, exclude_key_pairs=None, - exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): - genes = intervals = None - if has_gene_search: - genes = {'ENSG00000186092': mock.ANY, 'ENSG00000227232': mock.ANY} - intervals = [{'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}] - if single_gene_search: - genes = {'ENSG00000227232': mock.ANY} - intervals = intervals[2:3] - - expected_search = { - 'inheritance_mode': inheritance_mode, - 'inheritance_filter': {}, - 'skipped_samples': mock.ANY, - 'dataset_type': dataset_type, - 'secondary_dataset_type': secondary_dataset_type, - 'exclude_locations': exclude_locations, - 'genes': genes, - 'intervals': intervals, - 'freqs': {'callset': {'ac': 1000}}, - } - if not genes: - expected_search.update({ - 'parsed_variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, - 'variant_ids': variant_ids, - }) - expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) - if exclude: - expected_search['exclude'] = exclude - if annotations is not None: - expected_search['annotations'] = annotations - if annotations_secondary is not None: - expected_search['annotations_secondary'] = annotations_secondary - if exclude_keys is not None: - expected_search['exclude_keys'] = exclude_keys - if exclude_key_pairs is not None: - expected_search['exclude_key_pairs'] = exclude_key_pairs - - return (mock.ANY, expected_search, self.user, results_cache, '37'), kwargs, genes - - def _test_expected_search_call(self, mock_get_variants, *args, has_gene_search=False, exclude_locations=False, omitted_sample_guids=None, **kwargs): - call_args, call_kwargs, genes = self._get_expected_search_call(*args, has_gene_search=has_gene_search, exclude_locations=exclude_locations, **kwargs) - mock_get_variants.assert_called_with(*call_args, **call_kwargs) - self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - - if genes: - parsed_genes = mock_get_variants.call_args.args[1]['genes'] - for gene in parsed_genes.values(): - self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) - self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') - if len(genes) > 1: - self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') - - - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): - searched_samples = self.affected_search_samples - non_affected_search_samples = self.non_affected_search_samples - if omitted_sample_guids: - searched_samples = searched_samples.exclude(guid__in=omitted_sample_guids) - non_affected_search_samples = non_affected_search_samples.exclude(guid__in=omitted_sample_guids) - if has_gene_search: - searched_samples = searched_samples.exclude(guid__in=MITO_SAMPLES) - self.assertSetEqual(set(mock_get_variants.call_args.args[0]), set(searched_samples)) - self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - - - def test_query_variants(self, mock_get_variants): - parsed_variants = [{**v, 'key': (i+1) * 1000 } for i, v in enumerate(PARSED_VARIANTS)] - def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] =parsed_variants - previous_search_results['total_results'] = 5 - return parsed_variants - mock_get_variants.side_effect = _mock_get_variants - - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, parsed_variants) - self.assertEqual(total, 5) - results_cache = {'all_results': parsed_variants, 'total_results': 5} - self.assert_cached_results(results_cache) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - ) - - query_variants( - self.results_model, user=self.user, sort='cadd', skip_genotype_filter=True, page=3, num_results=10, - ) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='cadd', page=3, num_results=10, skip_genotype_filter=True, - ) - - query_variants(self.results_model, user=self.user, load_all=True) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=1000, skip_genotype_filter=False, - ) - - with mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 4): - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user, load_all=True) - self.assertEqual(str(cm.exception), 'Unable to export more than 4 variants (5 requested)') - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=4, skip_genotype_filter=False, - ) - - self.set_cache({'total_results': 22}) - query_variants(self.results_model, user=self.user, load_all=True) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=22, skip_genotype_filter=False, - ) - - self._test_locus_query_variants(mock_get_variants, results_cache) - - locus_items = self.search_model.search['locus']['rawItems'] - del self.search_model.search['locus'] - self.search_model.search['exclude'] = {'clinvar': ['benign'], 'rawItems': locus_items} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, exclude_locations=True, exclude={'clinvar': ['benign']}, - ) - - del self.search_model.search['exclude']['rawItems'] - self.search_model.search.update({'pathogenicity': {'clinvar': ['pathogenic', 'likely_pathogenic']}, - 'annotations': {'frameshift': [], 'structural': []}}) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - search_fields=['exclude', 'pathogenicity'], annotations={}, dataset_type='SNV_INDEL', omitted_sample_guids=SV_SAMPLES, - ) - - self.search_model.search = { - 'inheritance': {'mode': 'recessive'}, 'annotations': {'frameshift': ['frameshift_variant']}, - 'freqs': self.search_model.search['freqs'], - } - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type=None, - search_fields=['annotations'], omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'], - ) - - self.search_model.search['annotations_secondary'] = {'structural_consequence': ['LOF']} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SV', - search_fields=['annotations', 'annotations_secondary'] - ) - - screen_annotations = {'SCREEN': ['dELS', 'DNase-only']} - self.search_model.search['annotations_secondary'].update({'SCREEN': ['dELS', 'DNase-only']}) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='ALL', - search_fields=['annotations', 'annotations_secondary'] - ) - - self.search_model.search['annotations_secondary']['structural_consequence'] = [] - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, - omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'], - ) - - self.set_cache(None) - mock_get_variants.reset_mock() - self.search_model.search = { - 'inheritance': {'mode': 'any_affected'}, - 'exclude': {'previousSearch': True, 'previousSearchHash': 'abc1234', 'clinvar': ['benign']}, - 'freqs': self.search_model.search['freqs'], - } - previous_search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': self.search_model.search['freqs']}) - previous_results_model = VariantSearchResults.objects.create(variant_search=previous_search_model, search_hash='abc1234') - previous_results_model.families.set(self.families) - query_variants(self.results_model, user=self.user) - self._test_exclude_previous_search( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], - ) - - del self.search_model.search['exclude'] - self.search_model.search['exclude_svs'] = True - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', omitted_sample_guids=SV_SAMPLES, dataset_type='SNV_INDEL', - ) - - self.search_model.search['locus'] = {'rawItems': 'WASH7P'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', has_gene_search=True, single_gene_search=True, - omitted_sample_guids=NON_SNP_INDEL_SAMPLES, dataset_type='SNV_INDEL_only', - ) - - def _test_locus_query_variants(self, mock_get_variants, results_cache): - self.search_model.search['locus'] = {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=2, skip_genotype_filter=False, - rs_ids=[], variant_ids=['1-248367227-TC-T', '2-103343353-GAGA-G'], - parsed_variant_ids=[('1', 248367227, 'TC', 'T'), ('2', 103343353, 'GAGA', 'G')], dataset_type='SNV_INDEL', - omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733'], - ) - - self.search_model.search['locus']['rawItems'] = 'WASH7P' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, single_gene_search=True, omitted_sample_guids=MITO_SAMPLES, - ) - - self.search_model.search['locus']['rawItems'] = 'WASH7P, chr2:1234-5678, chr7:100-10100%10, ENSG00000186092' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, omitted_sample_guids=MITO_SAMPLES, - ) - - def _test_exclude_previous_search(self, mock_get_variants, *args, num_searches=1, **kwargs): - self._test_expected_search_call(mock_get_variants, *args, **kwargs) - self.assertEqual(mock_get_variants.call_count, num_searches) - - def test_cached_query_variants(self): - self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS}) - variants, total = query_variants(self.results_model, user=self.user) - self._assert_expected_cached_variants(variants, 4) - self.assertEqual(total, 4) - - variants, total = query_variants(self.results_model, user=self.user, num_results=2) - self._assert_expected_cached_variants(variants, 2) - self.assertEqual(total, 4) - - def _assert_expected_cached_variants(self, variants, num_results): - self.assertListEqual(variants, self.CACHED_VARIANTS[:num_results]) - - def test_invalid_search_get_variant_query_gene_counts(self): - self._test_invalid_search_params(get_variant_query_gene_counts) - - def test_get_variant_query_gene_counts(self, mock_get_variants): - def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = self.GENE_AGG_ALL_RESULTS - return None - def _mock_get_gene_counts(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['gene_aggs'] = GENE_COUNTS - return GENE_COUNTS - - mock_get_variants.side_effect = _mock_get_variants if hasattr(self, 'GENE_AGG_ALL_RESULTS') else _mock_get_gene_counts - - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, GENE_COUNTS) - results_cache = {'all_results': self.GENE_AGG_ALL_RESULTS} if hasattr(self, 'GENE_AGG_ALL_RESULTS') else {'gene_aggs': gene_counts} - self.assert_cached_results(results_cache) - kwargs = dict(sort=None, num_results=100) - if self.HAS_GENE_AGG: - kwargs['gene_agg'] = True - self._test_expected_search_call( - mock_get_variants, results_cache, **kwargs, - ) - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - HAS_GENE_AGG = True - INVALID_RSID_ERROR = 'Invalid variant notation: found both variant IDs and rsIDs' - - def setUp(self): - self.set_up() - - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): - return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) - - def test_variant_lookup(self, *args, **kwargs): - with self.assertRaises(ValueError) as cm: - super().test_variant_lookup(mock.MagicMock()) - self.assertEqual(str(cm.exception), 'variant_lookup is disabled without the clickhouse backend') - - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') - def test_get_single_variant(self, mock_get_variants_for_ids): - mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] - super(ElasticsearchSearchUtilsTests, self).test_get_single_variant(mock_get_variants_for_ids) - - @mock.patch('seqr.utils.search.utils.get_es_variants') - def test_query_variants(self, mock_get_variants): - super(ElasticsearchSearchUtilsTests, self).test_query_variants(mock_get_variants) - - def _test_locus_query_variants(self, mock_get_variants, results_cache): - self.search_model.search['locus'] = {'rawVariantItems': 'rs9876'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - rs_ids=['rs9876'], variant_ids=[], parsed_variant_ids=[], omitted_sample_guids=SV_SAMPLES, dataset_type='SNV_INDEL', - ) - - super()._test_locus_query_variants(mock_get_variants, results_cache) - - def test_cached_query_variants(self): - super(ElasticsearchSearchUtilsTests, self).test_cached_query_variants() - - self.set_cache({ - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}, - {'null': [PARSED_VARIANTS[1]]} - ], - 'total_results': 3, - }) - - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, [PARSED_VARIANTS[0]] + [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT] + [PARSED_VARIANTS[1]]) - self.assertEqual(total, 3) - - variants, total = query_variants(self.results_model, user=self.user, num_results=2) - self.assertListEqual(variants, [PARSED_VARIANTS[0]] + [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) - self.assertEqual(total, 3) - - @mock.patch('seqr.utils.search.utils.get_es_variants') - def test_get_variant_query_gene_counts(self, mock_get_variants): - super(ElasticsearchSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_get_variants) - - def test_cached_get_variant_query_gene_counts(self): - cached_gene_counts = { - 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} - } - self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, cached_gene_counts) - - self.set_cache({'all_results': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, 'total_results': 2}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 1, 'families': {'F000003_3': 1, 'F000011_11': 1}}, - 'ENSG00000228198': {'total': 1, 'families': {'F000003_3': 1, 'F000011_11': 1}}, - }) - - self.set_cache({ - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}, - {'null': [PARSED_VARIANTS[1]]} - ], - 'loaded_variant_counts': { - 'test_index_second': {'loaded': 1, 'total': 1}, - 'test_index_second_compound_het': {'total': 0, 'loaded': 0}, - 'test_index': {'loaded': 1, 'total': 1}, - 'test_index_compound_het': {'total': 2, 'loaded': 2}, - }, - 'total_results': 4, - }) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000002_2': 1}}, - 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}} - }) - - def _test_invalid_no_location_search_params(self): - # Elasticsearch has no limits on no-location searches - pass - - -class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): - databases = '__all__' - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] - - PARSED_CACHED_VARIANTS = [VARIANT1, VARIANT2, VARIANT3, VARIANT4] - CACHED_VARIANTS = [format_cached_variant(v) for v in PARSED_CACHED_VARIANTS] - GENE_AGG_ALL_RESULTS = CACHED_VARIANTS + [format_cached_variant(PROJECT_2_VARIANT2)] - INVALID_RSID_ERROR = 'Invalid variants: rs9876' - - def setUp(self): - self.set_up() - - def _assert_expected_cached_variants(self, variants, num_results): - self.assertListEqual( - json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), - self.PARSED_CACHED_VARIANTS[:num_results], - ) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') - def test_get_single_variant(self, mock_call): - mock_call.return_value = PARSED_VARIANTS[0] - super().test_get_single_variant(mock_call) - - def _assert_expected_get_single_variant_call(self, mock_call, variant_id, parsed_variant_id, expected_samples, dataset_type='SNV_INDEL', **kwargs): - mock_call.assert_called_with(variant_id, parsed_variant_id, mock.ANY, '37', dataset_type) - self.assertSetEqual(set(mock_call.call_args.args[2]), expected_samples) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_query_variants(self, mock_call): - super().test_query_variants(mock_call) - - def _test_exclude_previous_search(self, mock_get_variants, *args, **kwargs): - super()._test_exclude_previous_search( - mock_get_variants, *args, **kwargs, - exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, num_searches=2, - ) - call_args, call_kwargs, _ = self._get_expected_search_call(*args, inheritance_mode='de_novo', sort=None, num_results=100) - mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)]) - - # Test when previous results are cached - mock_get_variants.reset_mock() - self.mock_redis.get.side_effect = [ - None, - json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), - ] - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] - query_variants(self.results_model, user=self.user) - super()._test_exclude_previous_search( - mock_get_variants, *args, **kwargs, num_searches=1, - exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]}, - exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]}, - ) - - def test_cached_query_variants(self): - Project.objects.filter(id=1).update(genome_version='38') - super().test_cached_query_variants() - - cache_key_prefix = f'search_results__{self.results_model.guid}' - self.mock_redis.get.side_effect = [None, json.dumps({'total_results': 4, 'all_results': self.CACHED_VARIANTS})] - self.mock_redis.keys.return_value = [f'{cache_key_prefix}__xpos', f'{cache_key_prefix}__gnomad'] - - variants, total = query_variants(self.results_model, user=self.user, sort='cadd') - self.assertEqual(total, 4) - self.assertListEqual( - json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), - [VARIANT4, VARIANT3, VARIANT2, VARIANT1] - ) - self.mock_redis.get.assert_has_calls([ - mock.call(f'{cache_key_prefix}__cadd'), - mock.call(f'{cache_key_prefix}__xpos'), - ]) - self.mock_redis.keys.assert_called_with(pattern=f'{cache_key_prefix}__*') - self.assert_cached_results( - {'all_results': [format_cached_variant(v) for v in [VARIANT4, VARIANT3, VARIANT2, VARIANT1]], 'total_results': 4}, - sort='cadd', - ) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_get_variant_query_gene_counts(self, mock_call): - super().test_get_variant_query_gene_counts(mock_call) - - def test_cached_get_variant_query_gene_counts(self): - self.set_cache({'all_results': self.CACHED_VARIANTS + [SV_VARIANT1], 'total_results': 5}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, - }) - - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_variant_lookup(self, mock_call): - super().test_variant_lookup(mock_call) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f10eb0ec09..7bef21e164 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -1,114 +1,29 @@ from collections import defaultdict from copy import deepcopy from datetime import timedelta + from pyliftover.liftover import LiftOver -from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ - get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ + get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Individual, Project, VariantSearchResults +from seqr.models import Sample, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json -from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ - MAX_VARIANTS -from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ - get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ - es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS +from clickhouse_search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ + SV_ANNOTATION_TYPES, X_LINKED_RECESSIVE, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom +from seqr.views.utils.permissions_utils import user_is_analyst logger = SeqrLogger(__name__) -class InvalidSearchException(Exception): - pass - - -SEARCH_EXCEPTION_ERROR_MAP = { - InvalidSearchException: 400, -} -SEARCH_EXCEPTION_ERROR_MAP.update(ES_EXCEPTION_ERROR_MAP) - -SEARCH_EXCEPTION_MESSAGE_MAP = {} -SEARCH_EXCEPTION_MESSAGE_MAP.update(ES_EXCEPTION_MESSAGE_MAP) - -ERROR_LOG_EXCEPTIONS = set() -ERROR_LOG_EXCEPTIONS.update(ES_ERROR_LOG_EXCEPTIONS) -DATASET_TYPES_LOOKUP = { - data_types[0]: data_types for data_types in [ - [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS], - [Sample.DATASET_TYPE_MITO_CALLS], - [Sample.DATASET_TYPE_SV_CALLS], - ] -} -DATASET_TYPES_LOOKUP[ALL_DATA_TYPES] = [dt for dts in DATASET_TYPES_LOOKUP.values() for dt in dts] -DATASET_TYPE_SNP_INDEL_ONLY = f'{Sample.DATASET_TYPE_VARIANT_CALLS}_only' -DATASET_TYPES_LOOKUP[DATASET_TYPE_SNP_INDEL_ONLY] = [Sample.DATASET_TYPE_VARIANT_CALLS] -DATASET_TYPE_NO_MITO = f'{Sample.DATASET_TYPE_MITO_CALLS}_missing' -DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] MAX_GENES_FOR_FILTER = 10000 MIN_MULTI_FAMILY_SEQR_AC = 5000 - - -def es_only(func): - def _wrapped(*args, **kwargs): - if not es_backend_enabled(): - raise ValueError(f'{func.__name__} is disabled without the elasticsearch backend') - return func(*args, **kwargs) - return _wrapped - - -def clickhouse_only(func): - def _wrapped(*args, **kwargs): - if es_backend_enabled(): - raise ValueError(f'{func.__name__} is disabled without the clickhouse backend') - return func(*args, **kwargs) - return _wrapped - - -def backend_specific_call(es_func, clickhouse_func): - if es_backend_enabled(): - return es_func - else: - return clickhouse_func - - -def ping_search_backend(): - # Clickhouse backend does not need special uptime testing, will be checked with the other database connection pings - backend_specific_call(ping_elasticsearch, lambda: None)() - - -def ping_search_backend_admin(): - backend_specific_call(ping_kibana, lambda: True)() - - -def _get_filtered_search_samples(search_filter, active_only=True): - samples = Sample.objects.filter(**search_filter) - if active_only: - samples = samples.filter(is_active=True) - return samples - - -def get_search_samples(projects, active_only=True): - return _get_filtered_search_samples({'individual__family__project__in': projects}, active_only=active_only) - - -def _get_families_search_data(families, dataset_type, include_no_access_projects=False, sample_filter=None): - samples = _get_filtered_search_samples(sample_filter or {'individual__family__in': families}) - if len(samples) < 1: - if include_no_access_projects: - return samples - raise InvalidSearchException('No search data found for families {}'.format( - ', '.join([f.family_id for f in families]))) - - if dataset_type: - samples = samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dataset_type]) - if not samples: - raise InvalidSearchException(f'Unable to search against dataset type "{dataset_type}"') - - return samples +MAX_EXPORT_VARIANTS = 1000 +MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 def _get_search_genome_version(search_model): @@ -129,31 +44,18 @@ def _get_search_genome_version(search_model): return next(iter(project_versions.keys())) -def get_single_variant(family, variant_id, user=None): +def get_single_variant(family, variant_id, user): parsed_variant_id = parse_variant_id(variant_id) - dataset_type = _variant_ids_dataset_type([parsed_variant_id]) - samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) - variant = backend_specific_call( - _get_es_variant_by_id, - _get_clickhouse_variant_by_id, - )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type=dataset_type, user=user) - if not variant: + dataset_type = _variant_id_dataset_type(parsed_variant_id) + parsed_variant_ids = [parsed_variant_id] if parsed_variant_id else None + search = {'parsed_variant_ids': parsed_variant_ids, 'variant_ids': [variant_id]} + genome_version = family.project.genome_version + variants = get_clickhouse_variants([family], [dataset_type], search, user, genome_version) + if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return variant - - -def _get_es_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, user=None, **kwargs): - variants = get_es_variants_for_variant_ids(samples, genome_version, [variant_id], user) - return variants[0] if variants else None + return format_clickhouse_results(variants, genome_version)[0] -def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type=None, **kwargs): - return get_clickhouse_variant_by_id( - variant_id, parsed_variant_id, samples, genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], - ) - - -@clickhouse_only def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): cache_fields = ['variant_lookup_results', variant_id, genome_version] if affected_only: @@ -166,7 +68,7 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ return variants parsed_variant_id = parse_variant_id(variant_id) - dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] + dataset_type = _variant_id_dataset_type(parsed_variant_id) _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) variants = clickhouse_variant_lookup(user, variant_id, parsed_variant_id, dataset_type, sample_type, genome_version, affected_only, hom_only) @@ -186,96 +88,64 @@ def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) -def _get_any_sort_cached_results(search_model): - cache_key = _get_search_cache_key(search_model, sort='*') - return safe_redis_get_wildcard_json(cache_key) - - -def _get_cached_search_results(search_model, sort=None): - cache_key = _get_search_cache_key(search_model, sort=sort) - return safe_redis_get_json(cache_key) or {} - - -def _validate_export_variant_count(total_variants): +def export_variants(search_model, user): + search_results, genome_version = _query_variants(search_model, user) + total_variants = len(search_results) if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + return format_clickhouse_export_results(search_results, genome_version) -def _get_elasticsearch_previous_search_results(search_model, sort, page, num_results, load_all, **kwargs): - previous_search_results = _get_cached_search_results(search_model, sort=sort) - start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) - - cached_page = None - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - cached_page = loaded_results[start_index:end_index] - - if not cached_page: - cached_page = process_es_previously_loaded_results(previous_search_results, start_index, end_index) - - return previous_search_results, cached_page, num_results - - -def _get_clickhouse_previous_search_results(search_model, sort, page, num_results, load_all, genome_version=None): - previous_search_results = _get_cached_search_results(search_model, sort=sort) +def _get_previous_search_results(search_model, sort): + previous_search_results = None + if sort: + cache_key = _get_search_cache_key(search_model, sort=sort) + previous_search_results = safe_redis_get_json(cache_key) or [] if not previous_search_results: - unsorted_results = _get_any_sort_cached_results(search_model) - if unsorted_results: - previous_search_results = get_clickhouse_cache_results( - unsorted_results['all_results'], sort, family_guid=search_model.families.first().guid, + wildcard_cache_key = _get_search_cache_key(search_model, sort='*') + previous_search_results = safe_redis_get_wildcard_json(wildcard_cache_key) + if previous_search_results and sort: + previous_search_results = get_sorted_search_results( + previous_search_results, sort, family_guid=search_model.families.first().guid, ) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) + return previous_search_results - start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) - cached_page = None - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) - - return previous_search_results, cached_page, num_results +def query_variants(search_model, sort, page, num_results, user): + if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(user): + sort = PATHOGENICTY_HGMD_SORT_KEY + all_results, genome_version = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) -def _get_result_range(page, num_results, total_results, load_all): - if load_all: - num_results = total_results or MAX_EXPORT_VARIANTS - _validate_export_variant_count(num_results) - start_index = (page - 1) * num_results - end_index = page * num_results - if total_results is not None: - end_index = min(end_index, total_results) + results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results], genome_version) - if end_index > MAX_VARIANTS: - raise InvalidSearchException(f'Unable to load more than {MAX_VARIANTS} variants ({end_index} requested)') + return results_page, len(all_results) - return start_index, end_index, num_results - -def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): +def _query_variants(search_model, user, sort=None): genome_version = _get_search_genome_version(search_model) - previous_search_results, cached_page, num_results = backend_specific_call( - _get_elasticsearch_previous_search_results, - _get_clickhouse_previous_search_results, - )(search_model, sort, page, num_results, load_all, genome_version=genome_version) - if cached_page is not None: - return cached_page, previous_search_results.get('total_results') + previous_search_results = _get_previous_search_results(search_model, sort) or {} + if previous_search_results: + return previous_search_results, genome_version - variants, total_results = _query_variants( - search_model, user, previous_search_results, genome_version, sort=sort, page=page, num_results=num_results, - skip_genotype_filter=skip_genotype_filter) + search = deepcopy(search_model.variant_search.search) + families = search_model.families.all() + _validate_sort(sort, families) - if load_all: - _validate_export_variant_count(total_results) + parsed_search = _parse_search(search, genome_version, user) + dataset_types, secondary_dataset_types = _search_dataset_type(parsed_search, genome_version) + _validate_search(parsed_search, families) - return variants, total_results + results = get_clickhouse_variants(families, dataset_types, parsed_search, user, genome_version, sort=sort, secondary_dataset_types=secondary_dataset_types) + cache_key = _get_search_cache_key(search_model, sort=sort) + safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) -def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, **kwargs): - search = deepcopy(search_model.variant_search.search) + return results, genome_version - families = search_model.families.all() - _validate_sort(sort, families) +def _parse_search(search, genome_version, user): no_access_project_genome_version = search.get('no_access_project_genome_version') locus = search.pop('locus', None) or {} exclude = search.get('exclude', None) or {} @@ -283,7 +153,6 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if locus and exclude_locations: raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - variant_ids = None parsed_search = {**search} genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: @@ -294,15 +163,10 @@ def _query_variants(search_model, user, previous_search_results, genome_version, raise InvalidSearchException('Too many genes/intervals') parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) if not (genes or intervals): - rs_ids, variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) + variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) if invalid_items: raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) - if rs_ids and variant_ids: - raise InvalidSearchException('Invalid variant notation: found both variant IDs and rsIDs') - parsed_search.update({'rs_ids': rs_ids, 'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) - - if variant_ids: - num_results = len(variant_ids) + parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) exclude.pop('rawItems', None) if exclude.get('clinvar') and (search.get('pathogenicity') or {}).get('clinvar'): @@ -313,47 +177,21 @@ def _query_variants(search_model, user, previous_search_results, genome_version, exclude.pop('previousSearch', None) exclude_previous_hash = exclude.pop('previousSearchHash', None) if exclude_previous_hash: - parsed_search.update(backend_specific_call( - lambda *args: {}, _get_clickhouse_exclude_keys, - )(exclude_previous_hash, user, genome_version)) + parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user)) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) - parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) - search_dataset_type = None - if dataset_type and dataset_type != ALL_DATA_TYPES: - if secondary_dataset_type is None or secondary_dataset_type == dataset_type: - search_dataset_type = lookup_dataset_type or dataset_type - elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: - search_dataset_type = DATASET_TYPE_NO_MITO - - samples = _get_families_search_data(families, dataset_type=search_dataset_type, include_no_access_projects=bool(no_access_project_genome_version)) if parsed_search.get('inheritance'): - samples = _parse_inheritance(parsed_search, samples) - - _validate_search(parsed_search, samples, previous_search_results) + _parse_inheritance(parsed_search) - variant_results = backend_specific_call(get_es_variants, get_clickhouse_variants)( - samples, parsed_search, user, previous_search_results, genome_version, - sort=sort, num_results=num_results, **kwargs, - ) - - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) + return parsed_search - return variant_results, previous_search_results.get('total_results') - -def _get_clickhouse_exclude_keys(search_hash, user, genome_version): +def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - cached_results = _get_any_sort_cached_results(previous_search_model) - if not cached_results: - cached_results = {} - _query_variants(previous_search_model, user, cached_results, genome_version) - results = cached_results['all_results'] + results, _ = _query_variants(previous_search_model, user) exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) for variant in results: @@ -376,50 +214,13 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - return backend_specific_call( - _get_es_variant_query_gene_counts, - _get_clickhouse_variant_query_gene_counts, - )(search_model, user) - - -def _get_es_variant_query_gene_counts(search_model, user): - previous_search_results = _get_cached_search_results(search_model) - if previous_search_results.get('gene_aggs'): - return previous_search_results['gene_aggs'] - - if len(previous_search_results.get('all_results', [])) == previous_search_results.get('total_results'): - return _get_gene_aggs_for_cached_variants( - previous_search_results['all_results'], - lambda v: next(( - [gene_id] for gene_id, transcripts in v['transcripts'].items() - if any(t['transcriptId'] == v['mainTranscriptId'] for t in transcripts) - ), []) if v['mainTranscriptId'] else [], - ) - - previously_loaded_results = process_es_previously_loaded_gene_aggs(previous_search_results) - if previously_loaded_results is not None: - return previously_loaded_results - - genome_version = _get_search_genome_version(search_model) - gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) - return gene_counts - - -def _get_clickhouse_variant_query_gene_counts(search_model, user): - previous_search_results = _get_any_sort_cached_results(search_model) or {} - if len(previous_search_results.get('all_results', [])) != previous_search_results.get('total_results'): - genome_version = _get_search_genome_version(search_model) - _query_variants(search_model, user, previous_search_results, genome_version) - - return _get_gene_aggs_for_cached_variants([ - v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) - ], lambda v: v['transcripts'].keys() if 'transcripts' in v else {t['geneId'] for t in v['sortedTranscriptConsequences']}) - - -def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): + results, _ = _query_variants(search_model, user) + flat_variants = [ + v for variants in results for v in (variants if isinstance(variants, list) else [variants]) + ] gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - for var in variants: - gene_ids = get_variant_genes(var) + for var in flat_variants: + gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} for gene_id in gene_ids: gene_aggs[gene_id]['total'] += 1 for family_guid in var['familyGuids']: @@ -430,25 +231,21 @@ def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): def _parse_variant_items(search_json): raw_items = search_json.get('rawVariantItems') if not raw_items: - return None, None, None, None + return None, None, None invalid_items = [] variant_ids = [] parsed_variant_ids = [] - rs_ids = [] for item in raw_items.replace(',', ' ').split(): - if item.startswith('rs') and backend_specific_call(True, False): - rs_ids.append(item) + variant_id = item.lstrip('chr') + parsed_variant_id = parse_variant_id(variant_id) + if parsed_variant_id: + parsed_variant_ids.append(parsed_variant_id) + variant_ids.append(variant_id) else: - variant_id = item.lstrip('chr') - parsed_variant_id = parse_variant_id(variant_id) - if parsed_variant_id: - parsed_variant_ids.append(parsed_variant_id) - variant_ids.append(variant_id) - else: - invalid_items.append(item) + invalid_items.append(item) - return rs_ids, variant_ids, parsed_variant_ids, invalid_items + return variant_ids, parsed_variant_ids, invalid_items def parse_variant_id(variant_id): @@ -473,53 +270,63 @@ def _validate_sort(sort, families): def _search_dataset_type(search, genome_version): parsed_variant_ids = search.get('parsed_variant_ids') - rsids = search.get('rs_ids') - if parsed_variant_ids or rsids: - lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) - return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type + secondary_dataset_types = None + if parsed_variant_ids: + dataset_types = _chromosome_filter_dataset_types([vid[0] for vid in parsed_variant_ids]) + else: + chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ + interval['chrom'] for interval in (search.get('intervals') or []) + ] if not search.get('exclude_locations') else None + dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) + secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None + if secondary_dataset_types and len(dataset_types or []) == 1 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: + secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] - chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ - interval['chrom'] for interval in (search.get('intervals') or []) - ] if not search.get('exclude_locations') else None - dataset_type = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) - secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None + if search.get('inheritance_mode') == X_LINKED_RECESSIVE: + if not dataset_types: + dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] + elif Sample.DATASET_TYPE_MITO_CALLS in dataset_types: + dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) - return dataset_type, secondary_dataset_type, None + return dataset_types, secondary_dataset_types -def _variant_ids_dataset_type(all_variant_ids): - variant_ids = [v for v in all_variant_ids if v] - any_sv = len(variant_ids) < len(all_variant_ids) - if len(variant_ids) == 0: +def _variant_id_dataset_type(parsed_variant_id): + if not parsed_variant_id: return Sample.DATASET_TYPE_SV_CALLS - return _chromosome_filter_dataset_type([vid[0] for vid in variant_ids], any_sv) + if parsed_variant_id[0].replace('chr', '').startswith('M'): + return Sample.DATASET_TYPE_MITO_CALLS + return Sample.DATASET_TYPE_VARIANT_CALLS + -def _chromosome_filter_dataset_type(chroms, any_sv): +def _chromosome_filter_dataset_types(chroms): has_mito = [chrom for chrom in chroms if chrom.replace('chr', '').startswith('M')] if len(has_mito) == len(chroms): - return Sample.DATASET_TYPE_MITO_CALLS - elif not has_mito: - return DATASET_TYPE_NO_MITO if any_sv else DATASET_TYPE_SNP_INDEL_ONLY - return ALL_DATA_TYPES if any_sv else Sample.DATASET_TYPE_VARIANT_CALLS + return [Sample.DATASET_TYPE_MITO_CALLS] + dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS] + if has_mito: + dataset_types.append(Sample.DATASET_TYPE_MITO_CALLS) + return dataset_types def _annotation_dataset_type(annotations, chroms, pathogenicity=None, exclude_svs=False): if not (annotations or chroms): - return Sample.DATASET_TYPE_VARIANT_CALLS if (pathogenicity or exclude_svs) else None + return [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] if (pathogenicity or exclude_svs) else None annotation_types = set((annotations or {}).keys()) if annotations and annotation_types.issubset(SV_ANNOTATION_TYPES) and not pathogenicity: - return Sample.DATASET_TYPE_SV_CALLS + return [Sample.DATASET_TYPE_SV_CALLS] + + dataset_types = _chromosome_filter_dataset_types(chroms) if chroms else [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] no_svs = exclude_svs or (annotations and annotation_types.isdisjoint(SV_ANNOTATION_TYPES)) - if chroms: - return _chromosome_filter_dataset_type(chroms, any_sv=not no_svs) - elif no_svs: - return Sample.DATASET_TYPE_VARIANT_CALLS - return ALL_DATA_TYPES + if not no_svs: + dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) + return dataset_types -def _parse_inheritance(search, samples): + +def _parse_inheritance(search): inheritance = search.pop('inheritance') inheritance_mode = inheritance.get('mode') inheritance_filter = inheritance.get('filter') or {} @@ -528,90 +335,34 @@ def _parse_inheritance(search, samples): inheritance_mode = None search.update({'inheritance_mode': inheritance_mode, 'inheritance_filter': inheritance_filter}) - if not (inheritance_mode or inheritance_filter): - return samples if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: raise InvalidSearchException('Inheritance must be specified if custom affected status is set') - if inheritance_mode == X_LINKED_RECESSIVE: - samples = samples.exclude(dataset_type=Sample.DATASET_TYPE_MITO_CALLS) - - samples = samples.select_related('individual') - if samples: - skipped_samples = _filter_inheritance_family_samples(samples, inheritance_filter) - if skipped_samples: - search['skipped_samples'] = skipped_samples - samples = samples.exclude(id__in=[s.id for s in skipped_samples]) - - return samples - -def _validate_search(search, samples, previous_search_results): - has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} and not previous_search_results.get('grouped_results') +def _validate_search(search, families): + has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) if has_comp_het_search: if not search.get('annotations'): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') - family_ids = {s.individual.family_id for s in samples.select_related('individual')} - if not has_location_filter and len(family_ids) > MAX_NO_LOCATION_COMP_HET_FAMILIES: + if not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: raise InvalidSearchException( 'Location must be specified to search for compound heterozygous variants across many families') - if search['secondary_dataset_type']: - invalid_type = next(( - dt for dt in [search['dataset_type'], search['secondary_dataset_type']] - if dt and dt != ALL_DATA_TYPES and samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dt]).count() < 1 - ), None) - if invalid_type: - raise InvalidSearchException( - f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' - ) - if search.get('no_access_project_genome_version'): raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - backend_specific_call(lambda *args: None, _validate_clickhouse_search)(samples, has_location_filter, search) - - -def _validate_clickhouse_search(samples, has_location_filter, search): - variant_samples = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - if not has_location_filter and variant_samples.values('individual__family__project_id').distinct().count() > 1: + if not has_location_filter and families.values('project_id').distinct().count() > 1: raise InvalidSearchException('Location must be specified to search across multiple projects') seqr_ac_filter = search.get('freqs', {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) - if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and variant_samples.values('individual__family_id').distinct().count() > 1: + if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: raise InvalidSearchException( f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' ) -def _filter_inheritance_family_samples(samples, inheritance_filter): - family_groups = defaultdict(set) - sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type') - individual_affected_status = inheritance_filter.get('affected') or {} - genotype_filter = None if inheritance_filter.get(Individual.AFFECTED_STATUS_AFFECTED) else inheritance_filter.get('genotype') - for sample in samples: - if genotype_filter: - is_filtered_family = sample.individual.guid in genotype_filter - else: - affected_status = individual_affected_status.get(sample.individual.guid) or sample.individual.affected - is_filtered_family = affected_status == Individual.AFFECTED_STATUS_AFFECTED - - if is_filtered_family: - family_groups[sample.individual.family_id].add(getattr(sample, sample_group_field)) - - if not family_groups: - raise InvalidSearchException( - 'Invalid custom inheritance' if genotype_filter else - 'Inheritance based search is disabled in families with no data loaded for affected individuals' - ) - - return [ - s for s in samples if getattr(s, sample_group_field) not in family_groups[s.individual.family_id] - ] - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index 8112f9ffaf..892985a490 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -5,7 +5,6 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.file_utils import file_iter, does_file_exist, list_files -from seqr.utils.search.constants import VCF_FILE_EXTENSIONS from seqr.models import Sample BLOCK_SIZE = 65536 @@ -31,6 +30,8 @@ REQUIRED_HEADERS = ['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT'] +VCF_FILE_EXTENSIONS = ('.vcf', '.vcf.gz', '.vcf.bgz') + def _validate_vcf_header(header): missing_headers = [h for h in REQUIRED_HEADERS if h not in header] diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 81ce98e2b2..821bff7af3 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -1,4 +1,3 @@ -import base64 from collections import defaultdict import json import os @@ -8,14 +7,11 @@ from django.core.exceptions import PermissionDenied from django.db.models import Max, F, Q from django.http.response import HttpResponse -from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search -from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index -from seqr.utils.search.utils import clickhouse_only, es_only, InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list @@ -29,31 +25,10 @@ from seqr.models import Sample, RnaSample, Individual, Project, PhenotypePrioritization -from settings import KIBANA_SERVER, KIBANA_ELASTICSEARCH_PASSWORD, KIBANA_ELASTICSEARCH_USER, \ - LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT +from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT logger = SeqrLogger(__name__) - -@data_manager_required -@es_only -def elasticsearch_status(request): - return create_json_response(get_elasticsearch_status()) - - -@data_manager_required -@es_only -def delete_index(request): - index = json.loads(request.body)['index'] - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=index) - if active_samples: - projects = set(active_samples.values_list('individual__family__project__name', flat=True)) - raise InvalidSearchException(f'"{index}" is still used by: {", ".join(projects)}') - - updated_indices = delete_es_index(index) - - return create_json_response({'indices': updated_indices}) - RNA = 'RNA' TISSUE_FIELD = 'TissueOfOrigin' AIRTABLE_TISSUE_TYPE_MAP = { @@ -79,21 +54,30 @@ def update_rna_seq(request): ], ) sample_metadata_mapping = {} - misconfigured_samples = {} + misconfigured_samples = defaultdict(list) for sample in airtable_samples: sample_ids = [ sample[sample_id_field] for sample_id_field in ['sample_id', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'] if sample.get(sample_id_field) ] - error_message = None - if len(sample.get(TISSUE_FIELD, [])) != 1: - error_message = 'no tissue specified' if not sample.get(TISSUE_FIELD) else 'multiple tissues specified' - elif len(sample['pdos']) != 1: - error_message = 'multiple conflicting PDOs' - elif sample['pdos'][0]['project_guid'] is None: - error_message = 'no project specified' - if error_message: - misconfigured_samples.update({sample_id: error_message for sample_id in sample_ids}) + error_messages = [] + tissues = sample.get(TISSUE_FIELD, []) + if not tissues: + error_messages.append('no tissue specified') + elif len(tissues) > 1: + error_messages.append('multiple tissues specified') + elif tissues[0] not in TISSUE_TYPE_MAP: + error_messages.append('invalid tissue specified') + + pdos = sample.get('pdos', []) + if len(pdos) > 1: + error_messages.append('multiple conflicting PDOs') + elif not pdos or pdos[0]['project_guid'] is None: + error_messages.append('no project specified') + + if error_messages: + for sample_id in sample_ids: + misconfigured_samples[sample_id].extend(error_messages) else: metadata = { 'tissue': TISSUE_TYPE_MAP[sample[TISSUE_FIELD][0]], @@ -109,10 +93,6 @@ def update_rna_seq(request): return create_json_response(response_json, status=status) -def _get_sample_file_path(file_dir, sample_guid): - return os.path.join(file_dir, f'{sample_guid}.json.gz') - - def _notify_phenotype_prioritization_loaded(project, tool, num_samples): send_project_notification( project, @@ -413,7 +393,6 @@ def _get_valid_search_individuals(project, airtable_samples, vcf_samples, datase @data_manager_required -@clickhouse_only def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') @@ -431,7 +410,6 @@ def trigger_delete_project(request): @data_manager_required -@clickhouse_only def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') @@ -450,25 +428,16 @@ def trigger_delete_family(request): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - @data_manager_required -@csrf_exempt -def proxy_to_kibana(request): - headers = {} - if KIBANA_ELASTICSEARCH_PASSWORD: - token = base64.b64encode('{}:{}'.format(KIBANA_ELASTICSEARCH_USER, KIBANA_ELASTICSEARCH_PASSWORD).encode('utf-8')) - headers['Authorization'] = 'Basic {}'.format(token.decode('utf-8')) - return _proxy_iframe_page(request, 'Kibana', KIBANA_SERVER, additional_headers=headers) - +def proxy_to_luigi(request): + if not LUIGI_UI_SERVICE_HOSTNAME: + return HttpResponse('Loading Pipeline UI is not configured', status=404) -def _proxy_iframe_page(request, page_name, host, additional_headers=None, path_prefix=None): headers = convert_django_meta_to_http_headers(request) + host = f'{LUIGI_UI_SERVICE_HOSTNAME}:{LUIGI_UI_SERVICE_PORT}' headers['Host'] = host - headers.update(additional_headers or {}) - path = request.get_full_path() - if path_prefix: - path = path.replace(path_prefix, '') + path = request.get_full_path().replace('/luigi_ui', '') url = f'http://{host}{path}' request_method = getattr(requests.Session(), request.method.lower()) @@ -496,13 +465,4 @@ def _proxy_iframe_page(request, page_name, host, additional_headers=None, path_p return proxy_response except (ConnectionError, RequestConnectionError) as e: logger.error(str(e), request.user) - return HttpResponse(f'Error: Unable to connect to {page_name} {e}', status=400) - - -@data_manager_required -def proxy_to_luigi(request): - if not LUIGI_UI_SERVICE_HOSTNAME: - return HttpResponse('Loading Pipeline UI is not configured', status=404) - return _proxy_iframe_page( - request, 'Luigi UI', f'{LUIGI_UI_SERVICE_HOSTNAME}:{LUIGI_UI_SERVICE_PORT}', path_prefix='/luigi_ui', - ) + return HttpResponse(f'Error: Unable to connect to Luigi UI {e}', status=400) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3f47f30e63..3b688ab47e 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -7,228 +7,16 @@ import responses from seqr.utils.communication_utils import _set_bulk_notification_stream -from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ - update_rna_seq, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ +from seqr.views.apis.data_manager_api import update_rna_seq, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ get_loaded_projects, load_data, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' NON_ANALYST_PROJECT_GUID = 'R0004_non_analyst_project' -ES_CAT_ALLOCATION=[{ - 'node': 'node-1', - 'shards': '113', - 'disk.used': '67.2gb', - 'disk.avail': '188.6gb', - 'disk.percent': '26' -}, - {'node': 'UNASSIGNED', - 'shards': '2', - 'disk.used': None, - 'disk.avail': None, - 'disk.percent': None - }] - -ES_CAT_NODES=[{ - 'name': 'node-1', - 'heap.percent': '57', -}, - {'name': 'no-disk-node', - 'heap.percent': '83', - }] - -EXPECTED_DISK_ALLOCATION = [{ - 'node': 'node-1', - 'shards': '113', - 'diskUsed': '67.2gb', - 'diskAvail': '188.6gb', - 'diskPercent': '26', - 'heapPercent': '57', -}, - {'node': 'UNASSIGNED', - 'shards': '2', - 'diskUsed': None, - 'diskAvail': None, - 'diskPercent': None - }] - -EXPECTED_NODE_STATS = [{'name': 'no-disk-node', 'heapPercent': '83'}] - -ES_CAT_INDICES = [{ - "index": "test_index", - "docs.count": "122674997", - "store.size": "14.9gb", - "creation.date.string": "2019-11-04T19:33:47.522Z" -}, - { - "index": "test_index_alias_1", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_alias_2", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_no_project", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_sv", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_sv_wgs", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, -] - -ES_CAT_ALIAS = [ - { - "alias": "test_index_second", - "index": "test_index_alias_1" - }, - { - "alias": "test_index_second", - "index": "test_index_alias_2" - }] - -ES_INDEX_MAPPING = { - "test_index": { - "mappings": { - "_meta": { - "gencodeVersion": "25", - "genomeVersion": "38", - "sampleType": "WES", - "sourceFilePath": "test_index_file_path", - }, - "_all": { - "enabled": False - } - } - }, - "test_index_alias_1": { - "mappings": { - "_meta": { - "gencodeVersion": "25", - "hail_version": "0.2.24", - "genomeVersion": "37", - "sampleType": "WGS", - "sourceFilePath": "test_index_alias_1_path", - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_alias_2": { - "mappings": { - "_meta": { - "gencodeVersion": "19", - "genomeVersion": "37", - "sampleType": "WES", - "datasetType": "VARIANTS", - "sourceFilePath": "test_index_alias_2_path" - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_no_project": { - "mappings": { - "_meta": { - "gencodeVersion": "19", - "genomeVersion": "37", - "sampleType": "WGS", - "datasetType": "VARIANTS", - "sourceFilePath": "test_index_no_project_path" - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_sv": { - "mappings": { - "_meta": { - "gencodeVersion": "29", - "genomeVersion": "38", - "sampleType": "WES", - "datasetType": "SV", - "sourceFilePath": "test_sv_index_path" - }, - } - }, - "test_index_sv_wgs": { - "mappings": { - "_meta": { - "gencodeVersion": "29", - "genomeVersion": "38", - "sampleType": "WGS", - "datasetType": "SV", - "sourceFilePath": "test_sv_wgs_index_path" - }, - } - }, -} - -TEST_INDEX_EXPECTED_DICT = { - "index": "test_index", - "sampleType": "WES", - "genomeVersion": "38", - "sourceFilePath": "test_index_file_path", - "docsCount": "122674997", - "storeSize": "14.9gb", - "creationDateString": "2019-11-04T19:33:47.522Z", - "gencodeVersion": "25", - "projects": [{'projectName': '1kg project n\xe5me with uni\xe7\xf8de', 'projectGuid': 'R0001_1kg'}] -} - -TEST_SV_INDEX_EXPECTED_DICT = { - "index": "test_index_sv", - "sampleType": "WES", - "genomeVersion": "38", - "sourceFilePath": "test_sv_index_path", - "docsCount": "672312", - "storeSize": "233.4mb", - "creationDateString": "2019-10-03T19:53:53.846Z", - "gencodeVersion": "29", - "datasetType": "SV", - "projects": [{'projectName': '1kg project n\xe5me with uni\xe7\xf8de', 'projectGuid': 'R0001_1kg'}] -} - -TEST_INDEX_NO_PROJECT_EXPECTED_DICT = { - "index": "test_index_no_project", - "sampleType": "WGS", - "genomeVersion": "37", - "sourceFilePath": "test_index_no_project_path", - "docsCount": "672312", - "storeSize": "233.4mb", - "creationDateString": "2019-10-03T19:53:53.846Z", - "datasetType": "VARIANTS", - "gencodeVersion": "19", - "projects": [] -} - -EXPECTED_ERRORS = [ - 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', - 'test_index_old does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', -] - RNA_TPM_MUSCLE_SAMPLE_GUID = 'RS000162_T_na19675_d2' RNA_OUTLIER_MUSCLE_SAMPLE_GUID = 'RS000172_E_na19675_d2' RNA_SPLICE_SAMPLE_GUID = 'RS000151_S_na19675_1' @@ -505,9 +293,8 @@ 'CollaboratorSampleID': 'NA12346', 'SeqrProject': [ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', - 'https://seqr.broadinstitute.org/project/R0004_non_analyst_project/project_page', ], - 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], + 'PDOStatus': ['RNA ready to load'], } }, { @@ -528,7 +315,7 @@ 'CollaboratorSampleID': 'NA12348', 'SeqrProject': ['R0003_test'], 'PDOStatus': ['RNA ready to load'], - 'TissueOfOrigin': ['Muscle'], + 'TissueOfOrigin': ['Custom Tissue'], } }, { @@ -559,96 +346,14 @@ class DataManagerAPITest(AirtableTest): VCF_SAMPLES = VCF_SAMPLES SKIP_TDR = False - @urllib3_responses.activate - def test_elasticsearch_status(self): - url = reverse(elasticsearch_status) - self.check_data_manager_login(url) - - urllib3_responses.add_json( - '/_cat/allocation?format=json&h=node,shards,disk.avail,disk.used,disk.percent', ES_CAT_ALLOCATION) - urllib3_responses.add_json( - '/_cat/nodes?format=json&h=name,heap.percent', ES_CAT_NODES) - urllib3_responses.add_json( - '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) - urllib3_responses.add_json('/_cat/aliases?format=json&h=alias,index', ES_CAT_ALIAS) - urllib3_responses.add_json('/_all/_mapping', ES_INDEX_MAPPING) - - response = self.client.get(url) - self._assert_expected_es_status(response) - - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'indices', 'errors', 'diskStats', 'nodeStats'}) - - self.assertEqual(len(response_json['indices']), 6) - self.assertDictEqual(response_json['indices'][0], TEST_INDEX_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][3], TEST_INDEX_NO_PROJECT_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][4], TEST_SV_INDEX_EXPECTED_DICT) - - self.assertListEqual(response_json['errors'], EXPECTED_ERRORS) - - self.assertListEqual(response_json['diskStats'], EXPECTED_DISK_ALLOCATION) - self.assertListEqual(response_json['nodeStats'], EXPECTED_NODE_STATS) - - @urllib3_responses.activate - def test_delete_index(self): - url = reverse(delete_index) - self.check_data_manager_login(url) - - response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'test_index'})) - self._assert_expected_delete_active_index_response(response) - - urllib3_responses.add_json( - '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) - urllib3_responses.add_json('/_cat/aliases?format=json&h=alias,index', ES_CAT_ALIAS) - urllib3_responses.add_json('/_all/_mapping', ES_INDEX_MAPPING) - urllib3_responses.add(urllib3_responses.DELETE, '/unused_index') - - response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'unused_index'})) - self._assert_expected_delete_index_response(response) - - def _assert_expected_delete_active_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') - self.assertEqual(len(urllib3_responses.calls), 0) - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'indices'}) - self.assertEqual(len(response_json['indices']), 6) - self.assertDictEqual(response_json['indices'][0], TEST_INDEX_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][3], TEST_INDEX_NO_PROJECT_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][4], TEST_SV_INDEX_EXPECTED_DICT) - - self.assertEqual(urllib3_responses.calls[0].request.method, 'DELETE') - - @mock.patch('seqr.views.apis.data_manager_api.KIBANA_ELASTICSEARCH_PASSWORD', 'abc123') - @responses.activate - def test_kibana_proxy(self): - url = '/api/kibana/random/path' - self.check_data_manager_login(url) - - self._test_request_proxy('localhost:5601', url, auth_header='Basic a2liYW5hOmFiYzEyMw==') - - # Test with error response - response = self.client.get('{}/bad_response'.format(url)) - self.assertEqual(response.status_code, 500) - - # Test with connection error - response = self.client.get('{}/bad_path'.format(url)) - self.assertContains(response, 'Error: Unable to connect to Kibana', status_code=400) - - def _test_request_proxy(self, host, url, auth_header=None, proxy_path=None): + def _test_request_proxy(self, host, url, proxy_path): response_args = { 'stream': True, 'body': 'Test response', 'content_type': 'text/custom', 'headers': {'x-test-header': 'test', 'keep-alive': 'true'}, } - proxy_url = f'http://{host}{proxy_path or url}' + proxy_url = f'http://{host}{proxy_path}' responses.add(responses.GET, proxy_url, status=200, **response_args) responses.add(responses.POST, proxy_url, status=201, **response_args) responses.add(responses.GET, '{}/bad_response'.format(proxy_url), body=HTTPError()) @@ -669,20 +374,14 @@ def _test_request_proxy(self, host, url, auth_header=None, proxy_path=None): get_request = responses.calls[0].request self.assertEqual(get_request.headers['Host'], host) self.assertEqual(get_request.headers['Test-Header'], 'some/value') - if auth_header: - self.assertEqual(get_request.headers['Authorization'], auth_header) - else: - self.assertFalse('Authorization' in get_request.headers) + self.assertFalse('Authorization' in get_request.headers) post_request = responses.calls[1].request self.assertEqual(post_request.headers['Host'], host) self.assertEqual(post_request.headers['Content-Type'], 'application/json') self.assertEqual(post_request.headers['Content-Length'], '26') self.assertEqual(post_request.body, data.encode('utf-8')) - if auth_header: - self.assertEqual(get_request.headers['Authorization'], auth_header) - else: - self.assertFalse('Authorization' in get_request.headers) + self.assertFalse('Authorization' in get_request.headers) @mock.patch('seqr.views.apis.data_manager_api.LUIGI_UI_SERVICE_HOSTNAME') @responses.activate @@ -807,11 +506,8 @@ def _has_expected_file_loading_logs(self, file, file_path, user, info=None, warn (warn_log, {'severity': 'WARNING'}) for warn_log in warnings or [] ] if additional_logs: - if additional_logs_offset: - for log in reversed(additional_logs): - expected_logs.insert(additional_logs_offset, log) - else: - expected_logs += additional_logs + for log in reversed(additional_logs): + expected_logs.insert(additional_logs_offset, log) self.assert_json_logs(user, expected_logs) @@ -937,10 +633,11 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to load the following samples from Airtable with no corresponding seqr ID: NA11111', + 'Unable to load the following samples that are improperly configured in Airtable with invalid tissue specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345, NA12347', 'Unable to load the following samples that are improperly configured in Airtable with multiple tissues specified: NA12345', - 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', - 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12347', 'Unable to load the following samples that are improperly configured in Airtable with no project specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', ]) _set_gzip_file_iter_stdout(mock_files[f'tmp/temp_uploads/{file_path}/muscle_samples.tsv.gz'], [header, loaded_data_row]) @@ -1016,6 +713,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s self.reset_logs() body.update({'ignoreExtraSamples': True, 'file': RNA_FILE_ID}) warnings = [ + 'Skipped loading for the following 1 samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345', 'Skipped loading for the following 1 samples that are improperly configured in Airtable with multiple tissues specified: NA12345', f'Skipped loading for the following {len(params["skipped_samples"].split(","))} ' f'samples with no match: {params["skipped_samples"]}'] @@ -1573,11 +1271,22 @@ def test_trigger_delete_family(self): Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') response = self.client.post(url, content_type='application/json', data=json.dumps({'family': 'F000002_2'})) - self._assert_expected_delete_family(response) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': [ + 'Disabled search for 7 samples in the following 1 families: 2', + 'Triggered delete family data', + ], + }) - def _assert_expected_delete_family(self, response): - self.assertEqual(response.status_code, 500) - self.assertDictEqual(response.json(), {'error': 'trigger_delete_family is disabled without the clickhouse backend'}) + family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) + self.assertEqual(family_samples.count(), 0) + + self.assertEqual(len(responses.calls), 1) + self.assertDictEqual(json.loads(responses.calls[-1].request.body), { + 'project_guid': 'R0001_1kg', + 'family_guids': ['F000002_2'], + }) class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): @@ -1791,25 +1500,6 @@ def _add_file_list_iter(self, file_list, stdout): self.mock_does_file_exist, self.mock_does_file_exist, self.mock_file_iter, self.mock_does_file_exist, self.mock_does_file_exist, self.mock_file_iter, ] - def _get_expected_read_file_subprocess_calls(self, file_name, sample_guid): - gsutil_cat = f'gsutil cat gs://seqr-scratch-temp/{file_name}/{sample_guid}.json.gz | gunzip -c -q - ' - self.mock_subprocess.assert_called_with(gsutil_cat, stdout=-1, stderr=-2, shell=True) # nosec - return [ - (f'==> gsutil ls gs://seqr-scratch-temp/{file_name}/{sample_guid}.json.gz', None), - (f'==> {gsutil_cat}', None), - ] - - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'elasticsearch_status is disabled without the elasticsearch backend') - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'delete_index is disabled without the elasticsearch backend') - - def _assert_expected_delete_active_index_response(self, response): - self._assert_expected_delete_index_response(response) - def _assert_expected_get_projects_requests(self): pdo_filter = "OR(SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';')))" expected_filters = [ @@ -2029,24 +1719,6 @@ def _test_validate_dataset_type(self, url): self._set_file_not_found() - def _assert_expected_delete_family(self, response): - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'info': [ - 'Disabled search for 7 samples in the following 1 families: 2', - 'Triggered delete family data', - ], - }) - - family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) - self.assertEqual(family_samples.count(),0) - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[-1].request.body), { - 'project_guid': 'R0001_1kg', - 'family_guids': ['F000002_2'], - }) - def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', diff --git a/seqr/views/apis/dataset_api.py b/seqr/views/apis/dataset_api.py deleted file mode 100644 index 360d861736..0000000000 --- a/seqr/views/apis/dataset_api.py +++ /dev/null @@ -1,94 +0,0 @@ -import json - -from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded -from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples -from seqr.utils.search.utils import es_only -from seqr.models import Individual, Family, Sample -from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file -from seqr.views.utils.json_utils import create_json_response -from seqr.views.utils.orm_to_json_utils import get_json_for_samples -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, data_manager_required - - -@data_manager_required -@es_only -def add_variants_dataset_handler(request, project_guid): - """Create or update samples for the given variant dataset - - Args: - request: Django request object - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'elasticsearchIndex': (required) - 'ignoreExtraSamplesInCallset': - 'mappingFilePath': - } - - Response body - will contain the following structure: - - """ - - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) - request_json = json.loads(request.body) - - try: - inactivated_sample_guids, updated_family_guids, updated_samples = _add_new_es_search_samples( - request_json, project, request.user, notify=True) - except ValueError as e: - return create_json_response({'errors': [str(e)]}, status=400) - - response_json = _get_samples_json(updated_samples, inactivated_sample_guids, project_guid) - response_json['familiesByGuid'] = {family_guid: {'analysisStatus': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS} - for family_guid in updated_family_guids} - - return create_json_response(response_json) - - -def _add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): - dataset_type = request_json.get('datasetType') - if dataset_type not in Sample.DATASET_TYPE_LOOKUP: - raise ValueError(f'Invalid dataset type "{dataset_type}"') - - sample_ids, sample_type, sample_data = validate_es_index_metadata_and_get_samples(request_json, project) - if not sample_ids: - raise ValueError('No samples found. Make sure the specified caller type is correct') - - sample_id_to_individual_id_mapping = load_mapping_file( - request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} - ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') - sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] - new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( - projects=[project], - user=user, - sample_project_tuples=sample_project_tuples, - sample_data=sample_data, - sample_type=sample_type, - dataset_type=dataset_type, - expected_families=expected_families, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' - ) - - if notify: - basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id', flat=True)) - - return inactivated_sample_guids, updated_family_guids, updated_samples - - -def _get_samples_json(updated_samples, inactivated_sample_guids, project_guid): - updated_sample_json = get_json_for_samples(updated_samples, project_guid=project_guid) - sample_response = {sample_guid: {'isActive': False} for sample_guid in inactivated_sample_guids} - sample_response.update({s['sampleGuid']: s for s in updated_sample_json}) - response = { - 'samplesByGuid': sample_response - } - updated_individuals = {s['individualGuid'] for s in updated_sample_json} - if updated_individuals: - individuals = Individual.objects.filter(guid__in=updated_individuals).prefetch_related('sample_set') - response['individualsByGuid'] = { - ind.guid: {'sampleGuids': [s.guid for s in ind.sample_set.all()]} for ind in individuals - } - return response diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py deleted file mode 100644 index 02dccf0297..0000000000 --- a/seqr/views/apis/dataset_api_tests.py +++ /dev/null @@ -1,467 +0,0 @@ -import json -import mock -from copy import deepcopy -from datetime import datetime -from django.urls.base import reverse -from io import StringIO - -from seqr.models import Sample, Family -from seqr.views.apis.dataset_api import add_variants_dataset_handler -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses - -SEQR_URL = 'https://seqr.broadinstitute.org' -PROJECT_GUID = 'R0001_1kg' -NON_ANALYST_PROJECT_GUID = 'R0004_non_analyst_project' -INDEX_NAME = 'test_index' -SV_INDEX_NAME = 'test_new_sv_index' -NEW_SAMPLE_TYPE_INDEX_NAME = 'test_new_index' -ADD_DATASET_PAYLOAD = json.dumps({'elasticsearchIndex': INDEX_NAME, 'datasetType': 'SNV_INDEL'}) -MAPPING_PROPS_SAMPLES_NUM_ALT_1 = { - "samples_num_alt_1": {"type": "keyword"}, -} -MAPPING_PROPS_WITH_SAMPLES = { - "samples": {"type": "keyword"}, -} -MAPPING_JSON = { - INDEX_NAME: { - 'mappings': { - '_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, - "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1 - }}} -MAPPING_JSON_38 = deepcopy(MAPPING_JSON) -MAPPING_JSON_38[INDEX_NAME]['mappings']['_meta']['genomeVersion'] = '38' - -MOCK_REDIS = mock.MagicMock() -MOCK_OPEN = mock.MagicMock() -MOCK_FILE_ITER = MOCK_OPEN.return_value.__enter__.return_value.__iter__ - - -@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) -@mock.patch('seqr.utils.file_utils.open', MOCK_OPEN) -class DatasetAPITest(object): - - @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda path: True) - @mock.patch('seqr.models.random.randint') - @mock.patch('seqr.utils.communication_utils.send_html_email') - @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://seqr.broadinstitute.org/') - @urllib3_responses.activate - def test_add_variants_dataset(self, mock_send_email, mock_random): - url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) - self.check_data_manager_login(url) - - # Confirm test DB is as expected - existing_index_sample = Sample.objects.get(sample_id='NA19675') - self.assertEqual(existing_index_sample.elasticsearch_index, INDEX_NAME) - self.assertTrue(existing_index_sample.is_active) - existing_index_sample_guid = existing_index_sample.guid - existing_old_index_sample = Sample.objects.get(guid='S000130_na19678') - self.assertNotEqual(existing_old_index_sample.elasticsearch_index, INDEX_NAME) - self.assertTrue(existing_old_index_sample.is_active) - existing_old_index_sample_guid = existing_old_index_sample.guid - existing_sample = Sample.objects.get(sample_id='NA19679') - self.assertEqual(existing_sample.elasticsearch_index, INDEX_NAME) - self.assertFalse(existing_sample.is_active) - existing_sample_guid = existing_sample.guid - self.assertEqual(Sample.objects.filter(sample_id='NA19678_1').count(), 0) - self.assertEqual(Sample.objects.filter(sample_id='NA20878').count(), 0) - - mock_random.return_value = 98765432101234567890 - - urllib3_responses.add_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON) - urllib3_responses.add_json('/{}/_search?size=0'.format(INDEX_NAME), {'aggregations': { - 'sample_ids': {'buckets': [{'key': 'NA19675'}, {'key': 'NA19679'}, {'key': 'NA19678_1'}, {'key': 'NA20878'}]} - }}, method=urllib3_responses.POST) - MOCK_FILE_ITER.return_value = StringIO('NA19678_1,NA19678\n') - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - MOCK_OPEN.assert_called_with('mapping.csv', 'r') - MOCK_REDIS.get.assert_called_with('index_metadata__test_index') - MOCK_REDIS.set.assert_called_with('index_metadata__test_index', '{"test_index": {"sampleType": "WES", "genomeVersion": "37", "sourceFilePath": "test_data.vcf", "fields": {"samples_num_alt_1": "keyword"}}}') - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - - new_sample_guid = 'S98765432101234567890_na20878' - replaced_sample_guid = 'S98765432101234567890_na19678_' - self.assertSetEqual( - set(response_json['samplesByGuid'].keys()), - {existing_sample_guid, existing_old_index_sample_guid, replaced_sample_guid, new_sample_guid} - ) - self.assertDictEqual(response_json['individualsByGuid'], { - 'I000002_na19678': {'sampleGuids': mock.ANY}, - 'I000003_na19679': {'sampleGuids': mock.ANY}, - 'I000013_na20878': {'sampleGuids': [new_sample_guid]}, - }) - self.assertSetEqual( - set(response_json['individualsByGuid']['I000002_na19678']['sampleGuids']), - {replaced_sample_guid, existing_old_index_sample_guid} - ) - self.assertSetEqual( - set(response_json['individualsByGuid']['I000003_na19679']['sampleGuids']), - {existing_sample_guid} - ) - - self.assertDictEqual(response_json['familiesByGuid'], { - 'F000001_1': {'analysisStatus': 'I'}, - 'F000009_9': {'analysisStatus': 'I'}, - }) - updated_family = Family.objects.get(guid='F000001_1') - self.assertEqual(updated_family.analysis_status, 'I') - self.assertIsNone(updated_family.analysis_status_last_modified_date) - self.assertIsNone(updated_family.analysis_status_last_modified_by) - - updated_samples = [sample for sample_guid, sample in response_json['samplesByGuid'].items() if sample_guid != existing_old_index_sample_guid] - self.assertSetEqual( - {'WES'}, - {sample['sampleType'] for sample in updated_samples} - ) - self.assertSetEqual( - {True}, - {sample['isActive'] for sample in updated_samples} - ) - self.assertSetEqual( - {datetime.now().strftime('%Y-%m-%d')}, - {sample['loadedDate'][:10] for sample in updated_samples} - ) - self.assertDictEqual(response_json['samplesByGuid'][existing_old_index_sample_guid], {'isActive': False}) - - updated_sample_models = Sample.objects.filter(guid__in=[sample['sampleGuid'] for sample in updated_samples]) - self.assertEqual(len(updated_sample_models), 3) - self.assertSetEqual({INDEX_NAME}, {sample.elasticsearch_index for sample in updated_sample_models}) - - existing_index_sample_model = Sample.objects.get(guid=existing_index_sample_guid) - self.assertEqual(existing_index_sample_model.sample_type, 'WES') - self.assertTrue(existing_index_sample_model.is_active) - self.assertTrue(str(existing_index_sample_model.loaded_date).startswith('2017-02-05')) - - self._assert_expected_notification(mock_send_email, sample_type='WES', count=2) - - # Adding an SV index works additively with the regular variants index - mock_random.return_value = 1234567 - mock_send_email.reset_mock() - urllib3_responses.add_json('/{}/_mapping'.format(SV_INDEX_NAME), { - SV_INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.bed', - 'datasetType': 'SV', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}}) - urllib3_responses.add_json('/{}/_search?size=0'.format(SV_INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19675_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': SV_INDEX_NAME, - 'datasetType': 'SV', - })) - self.assertEqual(response.status_code, 200) - - self.assertDictEqual( - urllib3_responses.call_request_json(), - {'aggs': {'sample_ids': {'terms': {'field': 'samples', 'size': 10000}}}} - ) - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - sv_sample_guid = 'S0001234567_na19675_1' - self.assertDictEqual(response_json['familiesByGuid'], {}) - self.assertListEqual(list(response_json['samplesByGuid'].keys()), [sv_sample_guid]) - self.assertEqual(response_json['samplesByGuid'][sv_sample_guid]['datasetType'], 'SV') - self.assertEqual(response_json['samplesByGuid'][sv_sample_guid]['sampleType'], 'WES') - self.assertTrue(response_json['samplesByGuid'][sv_sample_guid]['isActive']) - self.assertListEqual(list(response_json['individualsByGuid'].keys()), ['I000001_na19675']) - self.assertListEqual(list(response_json['individualsByGuid']['I000001_na19675'].keys()), ['sampleGuids']) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675']['sampleGuids']), - {sv_sample_guid, existing_index_sample_guid}) - - # Regular variant sample should still be active - sample_models = Sample.objects.filter(individual__guid='I000001_na19675') - self.assertEqual(len(sample_models), 2) - self.assertSetEqual({sv_sample_guid, existing_index_sample_guid}, - {sample.guid for sample in sample_models}) - self.assertSetEqual({True}, {sample.is_active for sample in sample_models}) - - self._assert_expected_notification(mock_send_email, sample_type='WES SV', count=1) - - # Adding an index for a different sample type works additively - mock_random.return_value = 987654 - mock_send_email.reset_mock() - urllib3_responses.add_json('/{}/_mapping'.format(NEW_SAMPLE_TYPE_INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data_1.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data_2.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - }) - urllib3_responses.add_json('/{}/_search?size=0'.format(NEW_SAMPLE_TYPE_INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19675_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': NEW_SAMPLE_TYPE_INDEX_NAME, - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - new_sample_type_sample_guid = 'S0000987654_na19675_1' - self.assertDictEqual(response_json['familiesByGuid'], {}) - self.assertListEqual(list(response_json['samplesByGuid'].keys()), [new_sample_type_sample_guid]) - self.assertEqual(response_json['samplesByGuid'][new_sample_type_sample_guid]['datasetType'], 'SNV_INDEL') - self.assertEqual(response_json['samplesByGuid'][new_sample_type_sample_guid]['sampleType'], 'WGS') - self.assertTrue(response_json['samplesByGuid'][new_sample_type_sample_guid]['isActive']) - self.assertListEqual(list(response_json['individualsByGuid'].keys()), ['I000001_na19675']) - self.assertListEqual(list(response_json['individualsByGuid']['I000001_na19675'].keys()), ['sampleGuids']) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675']['sampleGuids']), - {sv_sample_guid, existing_index_sample_guid, new_sample_type_sample_guid}) - - self._assert_expected_notification(mock_send_email, sample_type='WGS', count=1) - - # Previous variant samples should still be active - sample_models = Sample.objects.filter(individual__guid='I000001_na19675') - self.assertEqual(len(sample_models), 3) - self.assertSetEqual( - {sv_sample_guid, existing_index_sample_guid, new_sample_type_sample_guid}, - {sample.guid for sample in sample_models}) - self.assertSetEqual({True}, {sample.is_active for sample in sample_models}) - - # Test sending email for adding dataset to a non-analyst project - url = reverse(add_variants_dataset_handler, args=[NON_ANALYST_PROJECT_GUID]) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON_38) - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), {'aggregations': { - 'sample_ids': {'buckets': [{'key': 'NA21234'}]} - }}, method=urllib3_responses.POST) - - self.reset_logs() - mock_send_email.reset_mock() - mock_send_email.side_effect = Exception('Email server is not configured') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - - self._assert_expected_notification( - mock_send_email, sample_type='WES', count=1, project_guid=NON_ANALYST_PROJECT_GUID, - project_name='Non-Analyst Project', recipient='test_user_collaborator@test.com', - ) - self.assert_json_logs(user=None, offset=6, expected=[( - 'Error sending project email for R0004_non_analyst_project: Email server is not configured', {'detail': { - 'email_body': mock.ANY, - 'subject': 'New WES data available in seqr', 'to': ['test_user_collaborator@test.com'], - }, - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - 'severity': 'ERROR', - }) - ]) - - def _assert_expected_notification(self, mock_send_email, sample_type, count, email_content=None, - project_guid=PROJECT_GUID, project_name='1kg project nåme with uniçøde', - recipient='test_user_manager@test.com'): - if not email_content: - email_content = f'This is to notify you that data for {count} new {sample_type} samples has been loaded in seqr project {project_name}' - mock_send_email.assert_called_once_with( - email_body=f'Dear seqr user,\n\n{email_content}\n\nAll the best,\nThe seqr team', - subject=f'New {sample_type} data available in seqr', to=[recipient], process_message=mock.ANY, - ) - - @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda path: True) - @urllib3_responses.activate - def test_add_variants_dataset_errors(self): - url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) - self.check_data_manager_login(url) - - self._assert_expected_add_dataset_errors(url) - - def _assert_expected_add_dataset_errors(self, url): - response = self.client.post(url, content_type='application/json', data=json.dumps({})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "None"']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, 'datasetType': 'NOT_A_TYPE'})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "NOT_A_TYPE"']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({'datasetType': 'SV'})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['request must contain field: "elasticsearchIndex"']}) - - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], 'test_index - Error accessing index: Connection refused: GET /test_index/_mapping') - - urllib3_responses.add_json('/{}/_mapping'.format(INDEX_NAME), - {INDEX_NAME: {'mappings': {"properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'errors': ['Index metadata must contain fields: genomeVersion, sampleType, sourceFilePath']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'NOT_A_TYPE', - 'genomeVersion': '37', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Sample type not supported: NOT_A_TYPE']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '38', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Index "test_index" has genome version 38 but this project uses version 37']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Variant call dataset path must end with .vcf or .vcf.gz or .bgz or .bed or .mt']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - 'datasetType': 'SV', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Index "test_index" has dataset type SV but expects SNV_INDEL']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - }) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Found mismatched sample fields for indices in alias']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - }) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Found mismatched sample types for indices in alias']}) - - urllib3_responses.add_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': []}} - }, method=urllib3_responses.POST) - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'errors': ['No samples found. Make sure the specified caller type is correct']}) - - self.assertDictEqual( - urllib3_responses.call_request_json(), - {'aggs': {'sample_ids': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}}} - ) - - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19679'}, {'key': 'NA19678_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Matches not found for sample ids: NA19678_1. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - 'ignoreExtraSamplesInCallset': True, - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['The following families are included in the callset but are missing some family members: 1 (NA19675_1, NA19678).']}) - - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19673'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - 'ignoreExtraSamplesInCallset': True, - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': [ - 'None of the individuals or samples in the project matched the 1 expected sample id(s)']}) - - MOCK_FILE_ITER.return_value = StringIO('NA19678_1,NA19678,metadata\n') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Must contain 2 columns: NA19678_1, NA19678, metadata']}) - - MOCK_FILE_ITER.side_effect = Exception('Unhandled base exception') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'Unhandled base exception') - MOCK_FILE_ITER.side_effect = None - - -# Tests for AnVIL access disabled -class LocalDatasetAPITest(AuthenticationTestCase, DatasetAPITest): - fixtures = ['users', '1kg_project'] - - -# Test for permissions from AnVIL only -class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): - fixtures = ['users', 'social_auth', '1kg_project'] - - def _assert_expected_add_dataset_errors(self, url): - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'add_variants_dataset_handler is disabled without the elasticsearch backend') - - def test_add_variants_dataset(self, *args): - # Adding dataset is always disabled when ES is disabled, which is tested in test_add_variants_dataset_errors - pass diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index a273ad34a1..d3254d5203 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -12,7 +12,6 @@ from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json @@ -48,7 +47,7 @@ def family_page_data(request, family_guid): sample_models = Sample.objects.filter(individual__family=family) samples = get_json_for_samples( - sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst + sample_models, project_guid=project.guid, family_guid=family_guid, is_analyst=is_analyst ) response = { 'samplesByGuid': {s['sampleGuid']: s for s in samples} @@ -57,10 +56,9 @@ def family_page_data(request, family_guid): add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] - additional_fields = backend_specific_call([],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( - 'xpos', 'xpos_end', 'gene_ids', *additional_fields, - svType=F('saved_variant_json__svType') + 'xpos', 'xpos_end', 'gene_ids', 'key', 'dataset_type', + svType=F('sv_type') ) gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} discovery_variant_intervals = [dict(zip( @@ -129,7 +127,7 @@ def family_variant_tag_summary(request, family_guid): project = family.project check_project_permissions(project, request.user) - response = families_discovery_tags([{'familyGuid': family_guid}], genome_version=project.genome_version) + response = families_discovery_tags([{'familyGuid': family_guid}]) tags = VariantTag.objects.filter(saved_variants__family=family) family_tag_type_counts = tags.values('variant_tag_type__name').annotate(count=Count('*')) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index fdf1045bb8..67fc881917 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -16,7 +16,7 @@ FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \ SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS -from seqr.models import FamilyAnalysedBy, AnalysisGroup +from seqr.models import FamilyAnalysedBy, AnalysisGroup, Sample FAMILY_GUID = 'F000001_1' FAMILY_GUID2 = 'F000002_2' @@ -250,6 +250,12 @@ def test_edit_families_handler(self, mock_pm_group): self.assertEqual(response.status_code, 400) self.assertEqual(response.reason_phrase, "'families' not specified") + response = self.client.post(url, content_type='application/json', data=json.dumps({'families': [ + {'familyGuid': '2', 'familyId': '2', 'description': 'Test description 1'} + ]})) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['error'], 'Invalid family guids: 2') + response = self.client.post(url, content_type='application/json', data=json.dumps({'families': [ {'familyGuid': FAMILY_GUID, 'familyId': '2', 'description': 'Test description 1'} ]})) @@ -291,7 +297,6 @@ def test_edit_families_handler(self, mock_pm_group): 'families': [{'familyGuid': 'F000012_12'}]})) self.assertEqual(response.status_code, 200) - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') @mock.patch('seqr.views.utils.permissions_utils.INTERNAL_NAMESPACES') @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): @@ -312,11 +317,6 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 400) - self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active MME submission: NA19675_1']) - - with mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', ''): - response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) - self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to delete individuals with active MME submission: NA19675_1', 'Unable to delete individuals with active search sample: HG00731, HG00732, HG00733, NA19675_1, NA19678', @@ -324,6 +324,7 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): # Test success MatchmakerSubmission.objects.update(deleted_date=datetime.now()) + Sample.objects.update(is_active=False) response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 200) @@ -495,7 +496,7 @@ def test_update_family_fields(self): self.assertEqual(response_json['F000014_14']['displayName'], expected_id) def _anvil_enabled(self): - return not self.ES_HOSTNAME + return bool(self.MOCK_AIRTABLE_KEY) @mock.patch('seqr.views.utils.file_utils.anvil_enabled', lambda: False) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index df12dce8ee..75846a1bb1 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -12,6 +12,7 @@ from seqr.models import Individual, Family, CAN_VIEW from seqr.utils.file_utils import file_iter from seqr.utils.gene_utils import get_genes, get_gene_ids_for_gene_symbols +from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.anvil_metadata_utils import PARTICIPANT_TABLE, PHENOTYPE_TABLE, EXPERIMENT_TABLE, \ EXPERIMENT_LOOKUP_TABLE, FINDINGS_TABLE, FINDING_METADATA_COLUMNS, TRANSCRIPT_FIELDS, GENE_COLUMN, parse_population from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file, parse_file @@ -27,7 +28,7 @@ pm_or_data_manager_required, check_workspace_perm from seqr.views.utils.project_context_utils import add_project_tag_type_counts from seqr.views.utils.individual_utils import delete_individuals, add_or_update_individuals_and_families -from seqr.views.utils.variant_utils import bulk_create_tagged_variants +from seqr.views.utils.variant_utils import bulk_create_tagged_variants, get_saved_variant_annotations @login_and_policies_required @@ -831,8 +832,11 @@ def import_gregor_metadata(request, project_guid): variant_id = '-'.join([row[col] for col in ['chrom', 'pos', 'ref', 'alt']]) key = (individual['family_id'], variant_id) variant = {k: v for k, v in row.items() if v and v != 'NA'} + pos = int(variant['pos']) variant.update({ - 'pos': int(variant['pos']), + 'variantId': variant_id, + 'xpos': get_xpos(variant['chrom'], pos), + 'pos': pos, 'genomeVersion': variant['variant_reference_assembly'].replace('GRCh', ''), 'transcripts': {}, 'transcript': { @@ -865,7 +869,7 @@ def import_gregor_metadata(request, project_guid): new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( family_variant_data, tag_name=GREGOR_FINDING_TAG_TYPE, user=request.user, project=project, - get_metadata=lambda v: {k: v[k] for k in FINDING_METADATA_COLUMNS if k in v}, + get_metadata=lambda v: {k: v[k] for k in FINDING_METADATA_COLUMNS if k in v}, parse_new_saved_variants=_parse_new_aip_saved_variants, ) loaded_info = f'Loaded {len(new_keys)} new and {len(update_keys)} updated findings tags' if skipped_keys: @@ -878,6 +882,24 @@ def import_gregor_metadata(request, project_guid): return create_json_response(response_json) +def _parse_new_aip_saved_variants(new_variant_keys, family_variant_data): + family_id = next(family_id for family_id, _ in new_variant_keys) + genome_version = Family.objects.filter(id=family_id).values_list('project__genome_version', flat=True).first() + variants_by_id = get_saved_variant_annotations(new_variant_keys, genome_version) + new_variant_data = {} + for key in new_variant_keys: + variant = family_variant_data[key] + variant_id = key[1] + if variant_id in variants_by_id: + variant.update(variants_by_id[variant_id]) + else: + variant.update({'key': None, 'saved_variant_json': {k: v for k, v in variant.items() if k in { + 'chrom', 'pos', 'ref', 'alt', 'variantId', 'xpos', 'genomeVersion', 'genotypes', 'transcripts', 'mainTranscriptId', + }}}) + new_variant_data[key] = variant + return new_variant_data + + def _iter_metadata_table(file_path, table_name, user, filter_row): file_name = f'{file_path}/{table_name}.tsv' file_rows = parse_file(file_name, file_iter(file_name, user=user, no_project=True), iter_file=True) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 843f80b255..94cb1c80fa 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -364,13 +364,18 @@ def test_delete_individuals(self, mock_pm_group): })) self.assertEqual(response.status_code, 400) + response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ + 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] + })) + self.assertEqual(response.status_code, 400) + self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) + Sample.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000173_na21987']).update(is_active=False) + # send valid requests response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] })) - self._assert_expected_delete_individuals(response, mock_pm_group) - def _assert_expected_delete_individuals(self, response, mock_pm_group): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), {'individualsByGuid', 'familiesByGuid'}) @@ -613,7 +618,8 @@ def test_individuals_table_handler(self): self.assertEqual(response.status_code, 200) def _assert_expected_reload_calls(self, project_guid): - self.assertEqual(len(responses.calls), 0) + self.assertEqual(len(responses.calls), 1) + self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @mock.patch('seqr.views.utils.pedigree_info_utils.NO_VALIDATE_MANIFEST_PROJECT_CATEGORIES') @@ -1213,9 +1219,7 @@ def test_import_gregor_metadata(self): saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__name='GREGoR Finding' ).order_by('family_id', 'variant_id').distinct().values( - 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json__genomeVersion', - 'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId', - 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes', 'gene_ids', + 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1223,15 +1227,17 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': 'F000012_12', - 'saved_variant_json__genomeVersion': None, - 'saved_variant_json__transcripts': None, - 'saved_variant_json__genotypes': None, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, + 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, 'gene_ids': ['ENSG00000240361', 'ENSG00000135953'], + 'main_transcript': { + 'aminoAcids': None, 'biotype': 'protein_coding', 'canonical': 1, 'codons': 'Gtg/Atg', + 'consequenceTerms': ['intron_variant'], 'geneId': 'ENSG00000240361', 'hgvsc': 'ENST00000262738.3:c.3955G>A', + 'hgvsp': 'ENST00000505820.2:c.1586-17C>G', 'loftee': [None, []], 'majorConsequence': 'intron_variant', + 'transcriptId': 'ENST00000505820', 'transcriptRank': 0, + }, }) self.assertEqual(len(saved_variants[0]['genotypes']), 2) self.assertDictEqual(saved_variants[1], { @@ -1239,16 +1245,24 @@ def test_import_gregor_metadata(self): 'variant_id': '1-249045487-A-G', 'xpos': 1249045487, 'family__guid': 'F000012_12', - 'saved_variant_json__genomeVersion': '37', - 'saved_variant_json__transcripts': { - 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], + 'saved_variant_json': { + 'alt': 'G', + 'chrom': '1', + 'genomeVersion': '37', + 'genotypes': {created_individual_guid: {'numAlt': 1}}, + 'mainTranscriptId': None, + 'pos': 249045487, + 'ref': 'A', + 'variantId': '1-249045487-A-G', + 'xpos': 1249045487, + 'transcripts': { + 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], + }, }, - 'saved_variant_json__genotypes': {created_individual_guid: {'numAlt': 1}}, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, + 'main_transcript': {}, 'key': None, - 'dataset_type': None, - 'genotypes': {}, + 'dataset_type': 'SNV_INDEL', + 'genotypes': {created_individual_guid: {'numAlt': 1}}, 'gene_ids': ['ENSG00000240361'], }) new_family_genotypes = {new_family_individual_guid: {'numAlt': 2}} @@ -1257,15 +1271,12 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': new_family_guid, - 'saved_variant_json__genomeVersion': None, - 'saved_variant_json__transcripts': None, - 'saved_variant_json__genotypes': None, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, + 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, 'gene_ids': ['ENSG00000135953'], + 'main_transcript': {'hgvsc': 'c.3955G>A','hgvsp': 'c.1586-17C>G', 'transcriptId': 'ENST00000505820'}, }) variant_tags = VariantTag.objects.filter(variant_tag_type__name='GREGoR Finding') @@ -1454,11 +1465,3 @@ def _mock_subprocess(self, command, **kwargs): else: self.mock_subprocess.stdout.__iter__.return_value = self.gs_files[file_name] return self.mock_subprocess - - def _assert_expected_delete_individuals(self, response, mock_pm_group): - self.assertEqual(response.status_code, 400) - self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) - - def _assert_expected_reload_calls(self, project_guid): - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 2593c1cc56..b5793f8db7 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -257,7 +257,7 @@ def project_families(request, project_guid): **{key: family_id in data_families for key, data_families in has_data_families.items()}, }) - response = families_discovery_tags(families, genome_version=project.genome_version, project=project) + response = families_discovery_tags(families, project=project) return create_json_response(response) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index c8f7dea333..43b4fcf6ad 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -447,9 +447,6 @@ def test_project_families(self): empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) self._check_empty_project(empty_url, response_keys) - self._assert_expected_project_families(url, response_keys) - - def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False): response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -496,8 +493,7 @@ def _assert_expected_project_families(self, url, response_keys, no_discovery_tag self.assertListEqual(family_3['discoveryGeneIds'], []) self.assertListEqual(empty_family['discoveryGeneIds'], []) self.assertListEqual(family_1['discoveryGeneIds'], ['ENSG00000135953']) - family_2_tags = [] if no_discovery_tags else ['ENSG00000135953'] - self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryGeneIds'], family_2_tags) + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryGeneIds'], ['ENSG00000135953']) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ len(response_json['familiesByGuid'][family_guid]['discoveryGeneIds']) for family_guid in no_discovery_families @@ -1085,7 +1081,7 @@ def _not_found_error(file): # Tests for AnVIL access disabled class LocalProjectAPITest(AuthenticationTestCase, ProjectAPITest): - fixtures = ['users', '1kg_project', 'reference_data'] + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] PROJECT_COLLABORATORS = BASE_COLLABORATORS CREATE_PROJECT_JSON = BASE_CREATE_PROJECT_JSON PROJECT_COLLABORATOR_GROUPS = [{'name': 'analysts', 'hasViewPermissions': True, 'hasEditPermissions': True}] @@ -1195,20 +1191,6 @@ def test_project_collaborators(self): 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') self.assertEqual(self.mock_get_ws_access_level.call_count, 5) - def _assert_expected_project_families(self, *args, **kwargs): - super()._assert_expected_project_families(*args, **kwargs) - - # Test success when clickhouse is unavailable - self.reset_logs() - connections['clickhouse'].close() - super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True) - self.assert_json_logs(None, [ - ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }), - ]) - @staticmethod def _set_file_not_found(file_name, mock_subprocess, mock_does_file_exist, mock_open): mock_subprocess.return_value.stdout = [b'CommandException: One or more URLs matched no objects'] diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index cc8f19de42..7f1dc425f3 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -7,6 +7,7 @@ import re import requests +from clickhouse_search.search import get_variant_details_queryset from seqr.utils.file_utils import is_google_bucket_file_path, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -410,6 +411,7 @@ def _add_row(row, family_id, row_type): airtable_fields=[[PARTICIPANT_ID_FIELD, 'Recontactable'], [SMID_FIELD]], include_mondo=True, proband_only_variants=True, + get_variant_json=_get_variant_json, ) airtable_metadata_by_participant = _get_gregor_airtable_data(participant_rows, request.user, smids_by_airtable_record_id) @@ -1018,14 +1020,51 @@ def _add_row(row, family_id, row_type): individual_samples={i: None for i in individuals}, individual_data_types={i.individual_id: i.data_types for i in individuals}, add_row=_add_row, - include_clinvar=True, - include_variant_id=True, - variant_attr_fields=['tags'], mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__variant_id'), include_family_name_display=True, include_mondo=True, omit_airtable=True, proband_only_variants=True, + get_variant_json=_get_metadata_variant_json, ) return create_json_response({'rows': variant_rows}) + + +def _get_metadata_variant_json(*args): + return _get_variant_json( + *args, + include_clinvar=True, + variantId=F('variant_id'), + tags=ArrayAgg('varianttag__variant_tag_type__name', distinct=True), + ) + + +def _get_variant_json(saved_variants, fields, annotations, include_clinvar=False, **kwargs): + dataset_type_keys = saved_variants.exclude(dataset_type__startswith='SV').filter( + key__isnull=False, + ).values( + 'dataset_type', genome_version=F('family__project__genome_version'), + ).annotate(keys=ArrayAgg('key', distinct=True, filter=Q(key__isnull=False))) + clickhouse_metadata = {} + for o in dataset_type_keys: + clickhouse_metadata.update(_get_clickhouse_metadata(**o, include_clinvar=include_clinvar)) + + variants = saved_variants.values('dataset_type', 'key', *fields, **annotations, **kwargs) + for variant in variants: + metadata = clickhouse_metadata.get((variant.pop('dataset_type'), variant['genome_version'], variant.pop('key')), {}) + variant['ClinGen_allele_ID'] = metadata.get('caid') + if include_clinvar: + variant['clinvar'] = metadata.get('clinvar') + + return variants + +def _get_clickhouse_metadata(dataset_type, genome_version, keys, include_clinvar): + qs = get_variant_details_queryset(genome_version, dataset_type,keys) + fields = ['key'] + if include_clinvar: + fields.append('clinvar') + qs = qs.join_clinvar() + if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + fields.append('caid') + return {(dataset_type, genome_version, v.pop('key')): v for v in qs.values(*fields)} diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index a5efdf6ed6..e52abc78c1 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1321,7 +1321,7 @@ def test_variant_metadata(self): **BASE_VARIANT_METADATA_ROW, 'alt': 'G', 'chrom': '21', - 'clinvar': {'alleleId': None, 'clinicalSignificance': '', 'goldStars': None, 'variationId': None}, + 'clinvar': None, 'condition_id': 'OMIM:615120', 'condition_inheritance': 'Autosomal recessive|X-linked', 'displayName': '1', @@ -1506,7 +1506,7 @@ def test_variant_metadata(self): class LocalReportAPITest(AuthenticationTestCase, ReportAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] ADDITIONAL_FAMILIES = ['F000014_14'] ADDITIONAL_FINDINGS = ['NA21234_1_248367227', 'NA21234_MT_14783', 'NA21234_1_249045487_DEL'] HAS_PM_OVERRIDE = True diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 90a38d0e84..3a1b289226 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -3,11 +3,9 @@ from django.db.models import Q -from clickhouse_search.search import get_variant_main_transcripts_by_key from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project, Sample -from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import backend_specific_call, es_only +from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -15,7 +13,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import reset_cached_search_results, get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json logger = logging.getLogger(__name__) @@ -49,6 +47,56 @@ def saved_variant_data(request, project_guid, variant_guids=None): return create_json_response(response) +@login_and_policies_required +def create_manual_saved_variant_handler(request, family_guid): + family = Family.objects.get(guid=family_guid) + check_project_permissions(family.project, request.user) + + variant_json = json.loads(request.body) + tags = variant_json.pop('tags', []) + saved_variant_guids = {guid for guid, is_selected in variant_json.pop('variants', {}).items() if is_selected} + + genome_version = family.project.genome_version + try: + xpos = get_xpos(variant_json['chrom'], variant_json['pos']) + variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" + except (KeyError, ValueError) as e: + return create_json_response({'error': str(e)}, status=400) + + variant_json.update({ + 'genomeVersion': genome_version, + 'transcripts': {}, + 'variantId': variant_id, + 'xpos': xpos, + }) + + gene_id = variant_json.pop('geneId', None) + if gene_id: + variant_json['transcripts'][gene_id] = [] + if variant_json.get('mainTranscriptId'): + variant_json['transcripts'][gene_id].append({ + 'transcriptId': variant_json['mainTranscriptId'], + 'hgvsc': variant_json.pop('hgvsc', None), + 'hgvsp': variant_json.pop('hgvsp', None), + }) + + variant_json['saved_variant_json'] = {**variant_json} + variant_json.update({ + 'key': None, + 'dataset_type': Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, + }) + + model_json = parse_saved_variant_json(variant_json, family.id) + saved_variant = create_model_from_json(SavedVariant, model_json, request.user) + saved_variant_guids.add(saved_variant.guid) + + saved_variants = SavedVariant.objects.filter(family=family, guid__in=saved_variant_guids) + _update_tags(saved_variants, {'tags': tags}, request.user) + + response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) + return create_json_response(response) + + @login_and_policies_required def create_saved_variant_handler(request): variant_json = json.loads(request.body) @@ -63,19 +111,17 @@ def create_saved_variant_handler(request): saved_variant_guids = [] for single_variant_json in variants_json: - try: - create_json, update_json = parse_saved_variant_json(single_variant_json, family.id) - except ValueError as e: - return create_json_response({'error': str(e)}, status=400) + parsed_json = parse_saved_variant_json(single_variant_json, family.id) + create_json = {field: parsed_json[field] for field in ['key', 'xpos', 'variant_id', 'family_id']} saved_variant, _ = get_or_create_model_from_json( - SavedVariant, create_json=create_json, update_json=update_json, + SavedVariant, create_json=create_json, update_json=parsed_json, user=request.user, update_on_create_only=True) saved_variant_guids.append(saved_variant.guid) saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) response = {} if variant_json.get('note'): - _, response = _create_variant_note(saved_variants, variant_json, request.user, family.project.genome_version) + _, response = _create_variant_note(saved_variants, variant_json, request.user) elif variant_json.get('tags'): _update_tags(saved_variants, variant_json, request.user) @@ -102,7 +148,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response({'error': 'Note is required'}, status=400) # update saved_variants - note, response = _create_variant_note(saved_variants, request_json, request.user, family.project.genome_version) + note, response = _create_variant_note(saved_variants, request_json, request.user) note_json = get_json_for_variant_note(note) note_json['variantGuids'] = all_variant_guids response.update({ @@ -115,7 +161,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response(response) -def _create_variant_note(saved_variants, note_json, user, genome_version): +def _create_variant_note(saved_variants, note_json, user): note = create_model_from_json(VariantNote, { 'note': note_json.get('note'), 'report': note_json.get('report') or False, @@ -125,7 +171,9 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) + variant = saved_variants[0] + gene_id = variant.gene_ids[0] if len(variant.gene_ids) == 1 or not variant.selected_main_transcript_id \ + else variant.main_transcript.get('geneId') create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -134,22 +182,6 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): return note, response -def _variant_gene_id(variant, genome_version): - if variant.selected_main_transcript_id and len(variant.gene_ids) > 1: - return next( - gene_id for gene_id, transcripts in variant.saved_variant_json['transcripts'].items() - if any(t['transcriptId'] == variant.selected_main_transcript_id for t in transcripts)) - return variant.gene_ids[0] - - -def _clickhouse_variant_gene_id(variant, genome_version): - if not (variant.key and variant.selected_main_transcript_id) or len(variant.gene_ids) == 1: - return _variant_gene_id(variant, genome_version) - return get_variant_main_transcripts_by_key( - genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, {variant.key: [variant.selected_main_transcript_id]}, - )[variant.key]['main_transcripts'][variant.selected_main_transcript_id].get('geneId') - - @login_and_policies_required def update_variant_note_handler(request, variant_guids, note_guid): note = VariantNote.objects.get(guid=note_guid) @@ -309,25 +341,14 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari model.saved_variants.set(saved_variants) -@login_and_policies_required -@es_only -def update_saved_variant_json(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) - reset_cached_search_results(project) - try: - updated_saved_variant_guids = update_project_saved_variant_json(project.id, project.genome_version, user=request.user) - except Exception as e: - logger.error('Unable to reset saved variant json for {}: {}'.format(project_guid, e)) - updated_saved_variant_guids = [] - - return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids or []}) - - @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) check_project_permissions(saved_variant.family.project, request.user, can_edit=True) - update_model_from_json(saved_variant, {'selected_main_transcript_id': transcript_id}, request.user) + update_model_from_json(saved_variant, { + 'selected_main_transcript_id': transcript_id, + 'main_transcript': json.loads(request.body), + }, request.user) return create_json_response({'savedVariantsByGuid': {variant_guid: {'selectedMainTranscriptId': transcript_id}}}) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 399de79cce..80549e3dc1 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -5,9 +5,9 @@ from django.urls.base import reverse from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict -from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Sample, Project +from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ - update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, update_saved_variant_json, \ + update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, create_manual_saved_variant_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ @@ -30,68 +30,62 @@ 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', } SAVED_VARIANT_RESPONSE_KEYS = { - *VARIANT_TAG_RESPONSE_KEYS, 'familiesByGuid', 'omimIntervals', + *VARIANT_TAG_RESPONSE_KEYS, 'familiesByGuid', 'omimIntervals', 'totalSampleCounts', 'genesById', 'locusListsByGuid', 'rnaSeqData', 'mmeSubmissionsByGuid', 'transcriptsById', 'phenotypeGeneScores', } COMPOUND_HET_3_JSON = { + 'key': 113, 'alt': 'C', 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', - 'mainTranscriptId': None, - 'originalAltAlleles': ['C'], + 'mainTranscriptId': '', 'populations': {'callset': {'ac': 17, 'af': 0.607, 'an': 28}}, 'pos': 62456358, 'predictions': {'cadd': 12.34}, 'ref': 'A', 'transcripts': {}, 'xpos': 15062456358, - 'projectGuid': 'R0001_1kg', 'familyGuids': ['F000001_1'], 'variantId': '15-62456358-A-C', 'acmgClassification': None, + 'CAID': None, + 'rsid': None, + 'clinvar': None, + 'hgmd': None, + 'liftedOverGenomeVersion': '38', + 'liftedOverChrom': None, + 'liftedOverPos': None, } COMPOUND_HET_4_JSON = { + 'key': 114, 'alt': 'A', 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', - 'mainTranscriptId': None, - 'originalAltAlleles': ['A'], + 'mainTranscriptId': '', 'populations': {'callset': {'ac': 1, 'af': 0.033, 'an': 8686}}, 'pos': 62456406, 'predictions': {'cadd': 13.56}, 'ref': 'G', 'transcripts': {}, 'xpos': 15062456406, - 'projectGuid': 'R0001_1kg', 'familyGuids': ['F000001_1'], 'variantId': '15-62456406-G-A', 'acmgClassification': None, -} - -COMPOUND_HET_5_JSON = { - 'alt': 'C', - 'chrom': '16', - 'genotypes': {}, - 'genomeVersion': '37', - 'mainTranscriptId': None, - 'originalAltAlleles': ['C'], - 'populations': {'callset': {'ac': 18, 'af': 0.563, 'an': 32}}, - 'pos': 31096164, - 'predictions': {'cadd': 7.099}, - 'ref': 'G', - 'transcripts': {}, - 'xpos': 16031096164, - 'projectGuid': 'R0001_1kg', - 'familyGuids': ['F000001_1'], - 'variantId': '16-31096164-G-C', - 'acmgClassification': None, + 'CAID': None, + 'rsid': None, + 'clinvar': None, + 'hgmd': None, + 'liftedOverGenomeVersion': '38', + 'liftedOverChrom': None, + 'liftedOverPos': None, } CREATE_VARIANT_JSON = { + 'key': 123, 'alt': 'A', 'chrom': '2', 'genotypes': {}, @@ -105,10 +99,11 @@ 'topmed': {'ac': 0, 'af': 0.0, 'an': 0, 'het': 0, 'hom': 0}, }, 'pos': 61413835, + 'xpos': 2061413835, 'liftedOverGenomeVersion': '38', 'liftedOverChrom': None, 'liftedOverPos': None, - 'predictions': {'cadd': 21.9, + 'predictions': {'cadd': 21.9, 'eigen': None, 'fathmm': None, 'mpc': None, @@ -161,16 +156,15 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) variants = response_json['savedVariantsByGuid'] self.assertSetEqual(set(variants.keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID}) variant = variants[VARIANT_GUID] - self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), self.SAVED_VARIANT_DETAIL_FIELDS) - fields = {'mainTranscriptId', 'mmeSubmissions'} - fields.update(self.SAVED_VARIANT_DETAIL_FIELDS) - self.assertSetEqual(set(variant.keys()), fields) + fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} + self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), fields) + self.assertSetEqual(set(variant.keys()), {*fields, 'mmeSubmissions'}) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) self.assertSetEqual( @@ -221,7 +215,7 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() response_keys = {'projectsByGuid'} - response_keys.update(self.SAVED_VARIANT_RESPONSE_KEYS) + response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) project = response_json['projectsByGuid'][PROJECT_GUID] @@ -232,7 +226,7 @@ def test_saved_variant_data(self): response = self.client.get('{}?includeLocusLists=true'.format(url)) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) locus_list_fields.update(LOCUS_LIST_FIELDS) locus_list_fields.update(PA_LOCUS_LIST_FIELDS) @@ -247,7 +241,7 @@ def test_saved_variant_data(self): family_context_response_keys = { 'individualsByGuid', 'familyNotesByGuid', 'igvSamplesByGuid', 'projectsByGuid' } - family_context_response_keys.update(self.SAVED_VARIANT_RESPONSE_KEYS) + family_context_response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), family_context_response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2'}) @@ -266,7 +260,7 @@ def test_saved_variant_data(self): # get variants with no tags for whole project response = self.client.get('{}?includeNoteVariants=true'.format(url)) self.assertEqual(response.status_code, 200) - no_families_response_keys = {*self.SAVED_VARIANT_RESPONSE_KEYS} + no_families_response_keys = {*SAVED_VARIANT_RESPONSE_KEYS} no_families_response_keys.remove('familiesByGuid') no_families_response_keys.remove('transcriptsById') self.assertSetEqual(set(response.json().keys()), no_families_response_keys) @@ -322,7 +316,7 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) variants = response_json['savedVariantsByGuid'] self.assertSetEqual( set(variants.keys()), @@ -378,45 +372,37 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), { - 'discoveryTags', *self.SAVED_VARIANT_DETAIL_FIELDS, *self.SAVED_VARIANT_38_DETAIL_FIELDS, + *fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - create_variant_json = self._format_create_variant(CREATE_VARIANT_JSON) create_variant_request_body = { 'searchHash': 'd380ed0fd28c3127d07a64ea2ba907d7', 'familyGuid': 'F000001_1', 'tags': [{'name': 'Review', 'metadata': 'a note'}], 'note': '', 'functionalData': [], - 'variant': create_variant_json, + 'variant': CREATE_VARIANT_JSON, } - - invalide_create_variant_request_body = deepcopy(create_variant_request_body) - invalide_create_variant_request_body['variant']['chrom'] = '27' - response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - invalide_create_variant_request_body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'error': 'Invalid chromosome: 27'}) - - response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - create_variant_request_body)) + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(create_variant_request_body)) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) variant_guid = next(iter(response.json()['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - variant_json = {'xpos': 2061413835} - variant_json.update(create_variant_json) - self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258', 'ENSG00000177000']) + self._assert_created_variant( + saved_variant, CREATE_VARIANT_JSON, gene_ids=['ENSG00000277258', 'ENSG00000177000'], + main_transcript=CREATE_VARIANT_JSON['transcripts']['ENSG00000277258'][0], + ) - variant_json.update({ + variant_json = { + **CREATE_VARIANT_JSON, 'variantGuid': variant_guid, 'familyGuids': ['F000001_1'], 'acmgClassification': None, @@ -425,7 +411,7 @@ def test_create_saved_variant(self): 'functionalDataGuids': [], 'transcripts': mock.ANY, 'mainTranscriptId': mock.ANY, - }) + } response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][variant_guid] tag_guids = response_variant_json.pop('tagGuids') @@ -443,13 +429,15 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _format_create_variant(self, variant_json, **kwargs): - return variant_json - - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, **kwargs): - for field in ['xpos', 'ref', 'alt']: + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None, has_saved_variant_json=False): + for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) + self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) + self.assertEqual(dataset_type, saved_variant.dataset_type) + self.assertEqual(sv_type, saved_variant.sv_type) + self.assertDictEqual(main_transcript or {}, saved_variant.main_transcript) + self.assertDictEqual(variant_json if has_saved_variant_json else {}, saved_variant.saved_variant_json) def test_create_saved_sv_variant(self): # SVs are only supported on build 38 @@ -458,7 +446,8 @@ def test_create_saved_sv_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - variant_json = self._format_create_variant({ + variant_json = { + 'key': 123, 'chrom': '2', 'genotypes': {}, 'genomeVersion': '38', @@ -476,7 +465,8 @@ def test_create_saved_sv_variant(self): 'svType': 'DUP', 'variantId': 'batch_123_DUP', 'acmgClassification': None, - }) + 'xpos': 2061413835, + } request_body = { 'familyGuid': 'F000001_1', @@ -496,8 +486,7 @@ def test_create_saved_sv_variant(self): variant_guid = next(iter(response_json['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - variant_json.update({'xpos': 2061413835}) - self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', gene_ids=['ENSG00000240361']) + self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', sv_type='DUP', gene_ids=['ENSG00000240361']) self.assertEqual(saved_variant.xpos_end, 2061414175) variant_json.update({ @@ -541,23 +530,26 @@ def test_create_saved_compound_hets(self): saved_compound_het_3 = SavedVariant.objects.get(guid=new_compound_het_3_guid, family__guid='F000001_1') saved_compound_het_4 = SavedVariant.objects.get(guid=new_compound_het_4_guid, family__guid='F000001_1') - self.assertDictEqual(COMPOUND_HET_3_JSON, saved_compound_het_3.saved_variant_json) - self.assertDictEqual(COMPOUND_HET_4_JSON, saved_compound_het_4.saved_variant_json) + self._assert_created_variant(saved_compound_het_3, COMPOUND_HET_3_JSON) + self._assert_created_variant(saved_compound_het_4, COMPOUND_HET_4_JSON) - expected_compound_het_3_json = { - 'variantGuid': new_compound_het_3_guid, + expected_json = { 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], + 'predictions': mock.ANY, + 'populations': mock.ANY, + } + expected_compound_het_3_json = { + 'variantGuid': new_compound_het_3_guid, + **COMPOUND_HET_3_JSON, + **expected_json, } - expected_compound_het_3_json.update(self._format_create_variant(COMPOUND_HET_3_JSON, key=None)) expected_compound_het_4_json = { 'variantGuid': new_compound_het_4_guid, - 'selectedMainTranscriptId': None, - 'noteGuids': [], - 'functionalDataGuids': [], + **COMPOUND_HET_4_JSON, + **expected_json, } - expected_compound_het_4_json.update(self._format_create_variant(COMPOUND_HET_4_JSON, key=None)) response_json = response.json() response_compound_het_3_json = response_json['savedVariantsByGuid'][new_compound_het_3_guid] response_compound_het_4_json = response_json['savedVariantsByGuid'][new_compound_het_4_guid] @@ -573,6 +565,116 @@ def test_create_saved_compound_hets(self): self.assertListEqual(["Review"], [vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__contains=new_compound_het_4_guid)]) + def test_create_manual_variant(self): + create_saved_variant_url = reverse(create_manual_saved_variant_handler, args=['F000001_1']) + self.check_collaborator_login(create_saved_variant_url) + + manual_variant_request_body = { + 'tags': [{'name': 'Review'}], + 'alt': 'A', + 'ref': 'GAG', + 'chrom': '27', + 'pos': 53417465, + 'mainTranscriptId': 'ENST00000459627', + 'geneId': 'ENSG00000277258', + 'hgvsc': 'c.156GAG>A', + 'hgvsp': 'p.Leu52Phe', + 'genotypes': { + 'I000001_na19675': {'numAlt': 1}, + 'I000002_na19678': {'numAlt': 0}, + 'I000003_na19679': {'numAlt': 0}, + }, + } + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Invalid chromosome: 27'}) + + manual_variant_request_body['chrom'] = '7' + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 200) + + self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) + variant_guid = next(iter(response.json()['savedVariantsByGuid'])) + + saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') + variant_json = { + 'xpos': 7053417465, + 'variantId': '7-53417465-GAG-A', + 'genomeVersion': '37', + 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, + **{k: v for k, v in manual_variant_request_body.items() if k in {'alt', 'ref', 'chrom', 'pos', 'genotypes', 'mainTranscriptId'}}, + } + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True, main_transcript={ + 'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627', + }) + self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + + base_variant_json = { + 'variantGuid': variant_guid, + 'familyGuids': ['F000001_1'], + 'acmgClassification': None, + 'selectedMainTranscriptId': None, + 'key': None, + 'noteGuids': [], + 'functionalDataGuids': [], + } + variant_json.update(base_variant_json) + response_json = response.json() + response_variant_json = response_json['savedVariantsByGuid'][variant_guid] + tag_guids = response_variant_json.pop('tagGuids') + self.assertEqual(len(tag_guids), 1) + tag = response_json['variantTagsByGuid'][tag_guids[0]] + self.assertEqual('Review', tag['name']) + self.assertListEqual([variant_guid], tag['variantGuids']) + self.assertDictEqual(variant_json, response_variant_json) + + # Create manual SV + manual_variant_request_body = { + 'chrom': '7', + 'pos': 10417443, + 'end': 11719832, + 'svName': 'Deletion:chr7:(?_10417443)_(11719832_?)', + 'svType': 'DEL', + 'geneId': 'ENSG00000240361', + 'genotypes': { + 'I000001_na19675': {'cn': 1}, + 'I000002_na19678': {'cn': 2}, + 'I000003_na19679': {'cn': 2}, + }, + 'tags': manual_variant_request_body['tags'], + 'variants': {'SV0059956_11560662_f019313_1': True, 'SV0000002_1248367227_r0390_100': False}, + } + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 200) + + self.assertEqual(len(response.json()['savedVariantsByGuid']), 2) + self.assertTrue('SV0059956_11560662_f019313_1' in response.json()['savedVariantsByGuid']) + sv_variant_guid = next(guid for guid in response.json()['savedVariantsByGuid'] if guid != 'SV0059956_11560662_f019313_1') + + saved_sv_variant = SavedVariant.objects.get(guid=sv_variant_guid, family__guid='F000001_1') + sv_variant_json = { + 'xpos': 7010417443, + 'genomeVersion': '37', + 'variantId': 'Deletion:chr7:(?_10417443)_(11719832_?)', + 'transcripts': {'ENSG00000240361': []}, + **{k: v for k, v in manual_variant_request_body.items() if k in {'chrom', 'pos', 'end', 'svName', 'svType', 'genotypes'}}, + } + self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL', has_saved_variant_json=True) + + sv_variant_json.update(base_variant_json) + sv_variant_json.update({ + 'variantGuid': sv_variant_guid, + 'tagGuids': [mock.ANY], + 'ref': None, + 'alt': None, + }) + response_json = response.json() + response_variant_json = response_json['savedVariantsByGuid'][sv_variant_guid] + self.assertDictEqual(sv_variant_json, response_variant_json) + tag = response_json['variantTagsByGuid'][response_variant_json['tagGuids'][0]] + self.assertEqual('Review', tag['name']) + self.assertSetEqual({sv_variant_guid, 'SV0059956_11560662_f019313_1'}, set(tag['variantGuids'])) + def test_create_update_and_delete_variant_note(self): create_variant_note_url = reverse(create_variant_note_handler, args=[VARIANT_GUID]) self.check_collaborator_login(create_variant_note_url, request_data={'familyGuid': 'F000001_1'}) @@ -664,9 +766,9 @@ def test_create_partially_saved_compound_het_variant_note(self): self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) request_body = { - 'variant': [COMPOUND_HET_5_JSON, { + 'variant': [COMPOUND_HET_4_JSON, { 'variantId': '21-3343353-GAGA-G', 'xpos': 21003343353, 'ref': 'GAGA', 'alt': 'G', - 'variantGuid': 'SV0000001_2103343353_r0390_100', + 'variantGuid': 'SV0000001_2103343353_r0390_100', 'transcripts': {}, 'key': None, 'tagGuids': ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'], 'noteGuids': []}, ], 'note': 'one_saved_one_not_saved_compount_hets_note', @@ -682,15 +784,17 @@ def test_create_partially_saved_compound_het_variant_note(self): compound_het_5_guid = compound_het_guids[0] saved_compound_het_5 = SavedVariant.objects.get(guid=compound_het_5_guid, family__guid='F000001_1') - self.assertDictEqual(COMPOUND_HET_5_JSON, saved_compound_het_5.saved_variant_json) + self._assert_created_variant(saved_compound_het_5, COMPOUND_HET_4_JSON) expected_compound_het_5_json = { + **COMPOUND_HET_4_JSON, 'variantGuid': compound_het_5_guid, 'selectedMainTranscriptId': None, 'tagGuids': [], 'functionalDataGuids': [], + 'predictions': mock.ANY, + 'populations': mock.ANY, } - expected_compound_het_5_json.update(self._format_create_variant(COMPOUND_HET_5_JSON, key=None)) response_json = response.json() response_compound_het_5_json = response_json['savedVariantsByGuid'][compound_het_5_guid] note_guids = response_compound_het_5_json.pop('noteGuids') @@ -985,54 +1089,20 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) - @mock.patch('seqr.utils.search.elasticsearch.es_utils.MAX_VARIANTS_FETCH', 2) - @mock.patch('seqr.views.apis.saved_variant_api.logger') - @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') - def test_update_saved_variant_json(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ - [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples})} - for variant_id in variant_ids] - - url = reverse(update_saved_variant_json, args=['R0001_1kg']) - self.check_manager_login(url) - - response = self.client.post(url) - - self.assertEqual(response.status_code, 200) - - self.assertDictEqual( - response.json(), - {'SV0000002_1248367227_r0390_100': None, 'SV0000001_2103343353_r0390_100': None, - 'SV0059956_11560662_f019313_1': None} - ) - - mock_get_variants.assert_has_calls([ - mock.call(mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A'], self.manager_user), - mock.call(mock.ANY, '37', ['21-3343353-GAGA-G'], self.manager_user), - ]) - samples = set(Sample.objects.filter(individual__family_id__in=[1, 2], is_active=True).values_list('id', flat=True)) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), samples) - self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), samples) - mock_logger.error.assert_not_called() - - # Test handles update error - mock_get_variants.side_effect = Exception('Unable to fetch variants') - response = self.client.post(url) - self.assertEqual(response.status_code, 200) - mock_logger.error.assert_called_with('Unable to reset saved variant json for R0001_1kg: Unable to fetch variants') - def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) self.check_manager_login(update_main_transcript_url) - response = self.client.post(update_main_transcript_url) + transcript = {'transcriptId': transcript_id, 'majorConsequence': 'intron_variant'} + response = self.client.post(update_main_transcript_url, content_type='application/json', data=json.dumps(transcript)) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'savedVariantsByGuid': {VARIANT_GUID: {'selectedMainTranscriptId': transcript_id}}}) saved_variants = SavedVariant.objects.filter(guid=VARIANT_GUID) self.assertEqual(len(saved_variants), 1) self.assertEqual(saved_variants.first().selected_main_transcript_id, transcript_id) + self.assertDictEqual(saved_variants.first().main_transcript, transcript) self.assertEqual(get_json_for_saved_variants(saved_variants, add_details=True)[0]['selectedMainTranscriptId'], transcript_id) def test_update_variant_acmg_classification(self): @@ -1056,15 +1126,7 @@ def test_update_variant_acmg_classification(self): # Tests for AnVIL access disabled class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data'] - - SAVED_VARIANT_RESPONSE_KEYS = SAVED_VARIANT_RESPONSE_KEYS - SAVED_VARIANT_DETAIL_FIELDS = SAVED_VARIANT_DETAIL_FIELDS - SAVED_VARIANT_38_DETAIL_FIELDS = {} - - def _assert_created_variant(self, saved_variant, variant_json, **kwargs): - super()._assert_created_variant(saved_variant, variant_json, **kwargs) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] def assert_no_list_ws_has_al(self, acl_call_count): @@ -1079,10 +1141,6 @@ def assert_no_list_ws_has_al(self, acl_call_count): class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] - SAVED_VARIANT_RESPONSE_KEYS = {*SAVED_VARIANT_RESPONSE_KEYS, 'totalSampleCounts'} - SAVED_VARIANT_DETAIL_FIELDS = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} - SAVED_VARIANT_38_DETAIL_FIELDS = {'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences'} - @classmethod def setUpTestData(cls): super().setUpTestData() @@ -1104,7 +1162,7 @@ def test_saved_variant_data(self, *args): def test_create_saved_variant(self): super(AnvilSavedVariantAPITest, self).test_create_saved_variant() - assert_no_list_ws_has_al(self, 4) + assert_no_list_ws_has_al(self, 3) def test_create_saved_sv_variant(self): super(AnvilSavedVariantAPITest, self).test_create_saved_sv_variant() @@ -1142,13 +1200,6 @@ def test_update_compound_hets_variant_functional_data(self): super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_functional_data() assert_no_list_ws_has_al(self, 3) - def test_update_saved_variant_json(self, *args): - url = reverse(update_saved_variant_json, args=['R0001_1kg']) - self.check_require_login(url) - response = self.client.post(url) - self.assertEqual(response.status_code, 500) - self.assertDictEqual(response.json(), {'error': 'update_saved_variant_json is disabled without the elasticsearch backend'}) - def test_update_variant_main_transcript(self): super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() assert_no_list_ws_has_al(self, 2) @@ -1156,13 +1207,3 @@ def test_update_variant_main_transcript(self): def test_update_variant_acmg_classification(self): super(AnvilSavedVariantAPITest, self).test_update_variant_acmg_classification() assert_no_list_ws_has_al(self, 2) - - def _format_create_variant(self, variant_json, key=123, **kwargs): - return {**variant_json, 'key': key} - - def _assert_created_variant(self, saved_variant, variant_json, dataset_type='SNV_INDEL', **kwargs): - super()._assert_created_variant(saved_variant, variant_json, **kwargs) - self.assertEqual(variant_json['key'], saved_variant.key) - self.assertEqual(dataset_type, saved_variant.dataset_type) - self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) - self.assertDictEqual({}, saved_variant.saved_variant_json) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 48ef125034..7aa3f63934 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -1,20 +1,20 @@ from collections import defaultdict from datetime import datetime from django.core.exceptions import PermissionDenied -from django.core.mail.message import EmailMessage from django.contrib.auth.models import User from django.db.models import CharField, F, Value from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf import json +from clickhouse_search.search import get_clickhouse_genotypes from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology -from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy, Sample +from seqr.models import Project, Family, Individual, Sample, VariantTagType, SavedVariant, FamilyAnalysedBy from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file -from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream +from seqr.utils.communication_utils import safe_post_to_slack from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException from seqr.views.utils.json_utils import create_json_response @@ -25,8 +25,8 @@ from seqr.views.utils.permissions_utils import analyst_required, user_is_analyst, get_project_guids_user_can_view, \ login_and_policies_required, get_project_and_check_permissions, get_internal_projects from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, DISCOVERY_ROW_TYPE -from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, DISCOVERY_CATEGORY -from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL +from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, get_saved_variant_annotations, DISCOVERY_CATEGORY +from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL logger = SeqrLogger(__name__) @@ -215,8 +215,8 @@ def _load_aip_data(data: dict, user: User): today = datetime.now().strftime('%Y-%m-%d') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( - family_variant_data, tag_name=AIP_TAG_TYPE, user=user, load_new_variant_data=True, - get_metadata=lambda pred: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, + family_variant_data, tag_name=AIP_TAG_TYPE, user=user, parse_new_saved_variants=_search_new_saved_variants, + get_metadata=lambda pred: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, ) summary_message = f'Loaded {len(new_keys)} new and {len(update_keys)} updated AIP tags for {len(family_id_map)} families' @@ -232,6 +232,64 @@ def _load_aip_data(data: dict, user: User): }) +def _search_new_saved_variants(family_variant_ids, *args, **kwargs): + family_ids = set() + variant_families = defaultdict(list) + for family_id, variant_id in family_variant_ids: + family_ids.add(family_id) + variant_families[variant_id].append(family_id) + families_by_id = { + f['id']: f for f in Family.objects.filter(id__in=family_ids).values( + 'id', 'guid', 'family_id', 'project__guid', 'project__genome_version', + ) + } + + genome_version = next(family['project__genome_version'] for family in families_by_id.values()) + search_variants_by_id = { + v['variantId']: v for v in _get_clickhouse_variants(families_by_id, family_variant_ids, genome_version) + } + + new_variants = {} + missing = defaultdict(list) + for variant_id, family_ids in variant_families.items(): + variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} + for family_id in family_ids: + family = families_by_id[family_id] + if family['guid'] in variant['familyGuids']: + new_variants[(family_id, variant_id)] = variant + else: + missing[family['family_id']].append(variant_id) + + if missing: + missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] + raise ErrorsWarningsException([ + f"Unable to find the following family's variants in the search backend: {', '.join(missing_summary)}", + ]) + + return new_variants + + +def _get_clickhouse_variants(families_by_id: dict[int, dict], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: + variants_by_key = get_saved_variant_annotations(family_variant_ids, genome_version=genome_version, group_by_field='key') + families_by_project = defaultdict(list) + for family in families_by_id.values(): + families_by_project[family['project__guid']].append(family['guid']) + for project_guid, family_guids in families_by_project.items(): + genotype_keys = get_clickhouse_genotypes( + project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), + additional_fields=['xpos'] + ) + for key, entry_data in genotype_keys.items(): + variants_by_key[key]['xpos'] = entry_data['xpos'] + variants_by_key[key]['genotypes'] = {**entry_data['genotypes'], **variants_by_key[key].get('genotypes', {})} + variants_by_key[key]['familyGuids'] = sorted([ + *{g['familyGuid'] for g in entry_data['genotypes'].values()}, + *(variants_by_key[key].get('familyGuids', [])), + ]) + + return list(variants_by_key.values()) + + ALL_PROJECTS = 'all' GREGOR_CATEGORY = 'gregor' @@ -335,25 +393,3 @@ def _get_airtable_collaborator_names(user, collaborator_ids): collaborator_id: collaborator_map.get(collaborator_id, {}).get('CollaboratorID') for collaborator_id in collaborator_ids } - - -@login_and_policies_required -def send_vlm_email(request): - request_json = json.loads(request.body) - email_message = EmailMessage( - subject=request_json['subject'], - body=request_json['body'], - bcc=[s.strip() for s in request_json['to'].split(',')], - cc=[request.user.email], - reply_to=[request.user.email], - to=[VLM_SEND_EMAIL], - from_email=VLM_SEND_EMAIL, - ) - set_email_message_stream(email_message, 'vlm') - - try: - email_message.send() - except Exception as e: - logger.error(f'VLM Email Error: {e}', request.user, detail=request_json) - - return create_json_response({'success': True}) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index f8048a3966..1d82998deb 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -6,8 +6,8 @@ import responses from seqr.views.apis.summary_data_api import mme_details, success_story, saved_variants_page, hpo_summary_data, \ - bulk_update_family_external_analysis, individual_metadata, send_vlm_email -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, PARSED_VARIANTS, SAVED_VARIANT_FIELDS + bulk_update_family_external_analysis, individual_metadata +from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, SAVED_VARIANT_FIELDS from seqr.models import FamilyAnalysedBy, SavedVariant, VariantTag from settings import AIRTABLE_URL @@ -30,7 +30,7 @@ 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', } SAVED_VARIANT_RESPONSE_KEYS = { - *VARIANT_TAG_RESPONSE_KEYS, 'projectsByGuid', 'locusListsByGuid', 'genesById', + *VARIANT_TAG_RESPONSE_KEYS, 'projectsByGuid', 'locusListsByGuid', 'genesById', 'totalSampleCounts', 'individualsByGuid', 'familiesByGuid', 'familyNotesByGuid', 'mmeSubmissionsByGuid', 'transcriptsById', } @@ -116,8 +116,6 @@ 'condition_id': 'OMIM:616126', 'condition_inheritance': 'Autosomal recessive', 'known_condition_name': 'Immunodeficiency 38', - 'ClinGen_allele_ID-1': 'CA1501729', - 'ClinGen_allele_ID-2': None, } EXPECTED_SAMPLE_METADATA_ROW = { "dbgap_submission": "No", @@ -180,14 +178,13 @@ 'validated_name-1': None, 'transcript-1': None, 'analysis_groups': '', - 'ClinGen_allele_ID-1': 'CA1501729', 'alt-3': None, 'chrom-3': '1', 'gene_known_for_phenotype-3': 'Known', 'phenotype_contribution-3': 'Full', 'partial_contribution_explained-3': '', 'pos-3': 249045487, - 'chrom_end-3': '1', + 'chrom_end-3': None, 'pos_end-3': 249045898, 'ref-3': None, 'copy_number-3': 1, @@ -205,7 +202,6 @@ 'validated_name-3': None, 'variant_inheritance-3': 'unknown', 'transcript-3': None, - 'ClinGen_allele_ID-3': None, 'alt-2': 'C', 'chrom-2': 'MT', 'gene_known_for_phenotype-2': 'Candidate', @@ -230,7 +226,6 @@ 'validated_name-2': None, 'variant_inheritance-2': 'unknown', 'transcript-2': 'ENST00000361789.2', - 'ClinGen_allele_ID-2': None, } AIRTABLE_SAMPLE_RECORDS = { @@ -394,7 +389,7 @@ def test_saved_variants_page(self): response = self.client.get(gene_url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) expected_variant_guids = { 'SV0000001_2103343353_r0390_100', 'SV0000007_prefix_19107_DEL_r00', 'SV0000006_1248367227_r0003_tes', } @@ -419,7 +414,7 @@ def test_saved_variants_page(self): response = self.client.get(gene_url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json['savedVariantsByGuid'].keys()), expected_variant_guids) multi_tag_url = reverse(saved_variants_page, args=['Review;Tier 1 - Novel gene and phenotype']) @@ -629,6 +624,28 @@ def _assert_expected_new_saved_variant(self, new_saved_variant): self.assertEqual(new_saved_variant.ref, 'GAGA') self.assertEqual(new_saved_variant.alt, 'G') self.assertListEqual(new_saved_variant.gene_ids, ['ENSG00000135953', 'ENSG00000228198']) + self.assertEqual(new_saved_variant.key, 101) + self.assertEqual(new_saved_variant.dataset_type, 'SNV_INDEL') + self.assertDictEqual(new_saved_variant.genotypes, { + 'I000004_hg00731': { + 'ab': 0, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000004_hg00731', 'sampleType': 'WES', + }, + 'I000005_hg00732': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000005_hg00732', 'sampleType': 'WES', + }, + 'I000006_hg00733': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', + }, + }) + self.assertDictEqual(new_saved_variant.main_transcript, { + 'aminoAcids': None, 'biotype': 'protein_coding', 'canonical': 1, 'codons': None, + 'consequenceTerms': ['inframe_deletion'], 'geneId': 'ENSG00000135953', 'hgvsc': 'ENST00000262738.3:c.3955G>A', + 'hgvsp': 'ENST00000505820.2:c.1586-17C>G', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, + 'majorConsequence': 'inframe_deletion', 'transcriptId': 'ENST00000505820', 'transcriptRank': 0, + }) def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False): self.assertEqual(response.status_code, 200) @@ -688,7 +705,6 @@ def test_sample_metadata_export(self): rows = response.json()['rows'] self.assertEqual(len(rows), 3) test_row = next(r for r in rows if r['participant_id'] == 'NA21234') - self.maxDiff = None self.assertDictEqual(test_row, EXPECTED_NO_GENE_SAMPLE_METADATA_ROW) # Test analyst access @@ -781,58 +797,13 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assert_expected_airtable_call( -1, "OR(RECORD_ID()='reca4hcBnbA2cnZf9')", ['CollaboratorID']) - @mock.patch('seqr.views.apis.summary_data_api.EmailMessage') - def test_send_vlm_email(self, mock_email): - url = reverse(send_vlm_email) - self.check_require_login(url) - - self.reset_logs() - body = { - 'to': 'test@test.com , other_test@gmail.com', - 'body': 'some email content', - 'subject': 'some email subject' - } - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_vlm_email(response, mock_email) - - self.reset_logs() - mock_email.return_value.send.side_effect = Exception('Send failed') - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_vlm_email(response, mock_email, additional_logs=[ - ('VLM Email Error: Send failed', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - 'detail': body, - }), - ]) - - def _assert_expected_vlm_email(self, response, mock_email, additional_logs=None): - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), {'success': True}) - - mock_email.assert_called_with( - subject='some email subject', - body='some email content', - bcc=['test@test.com', 'other_test@gmail.com'], - cc=['test_user_no_access@test.com'], - reply_to=['test_user_no_access@test.com'], - to=['vlm-noreply@broadinstitute.org'], - from_email='vlm-noreply@broadinstitute.org') - self.assertDictEqual(mock_email.return_value.esp_extra, {'MessageStream': 'vlm'}) - mock_email.return_value.send.assert_called() - - self.assert_json_logs(self.no_access_user, (additional_logs or []) + [ - (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}) - ]) - # Tests for AnVIL access disabled class LocalSummaryDataAPITest(AuthenticationTestCase, SummaryDataAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = ['NA21234', 'NA21987', 'NA21654'] HAS_AIRTABLE = False - SAVED_VARIANT_RESPONSE_KEYS = SAVED_VARIANT_RESPONSE_KEYS def _test_metadata_airtable_responses(self, include_airtable_url, expected_individuals): # Returns successfully without airtable data when disabled @@ -840,14 +811,6 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assertEqual(response.status_code, 200) self._has_expected_metadata_response(response, expected_individuals) - @mock.patch('seqr.views.utils.variant_utils.get_es_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) - def test_bulk_update_family_external_analysis(self, *args, **kwargs): - super().test_bulk_update_family_external_analysis(*args, **kwargs) - - def _assert_expected_new_saved_variant(self, new_saved_variant): - super()._assert_expected_new_saved_variant(new_saved_variant) - self.assertDictEqual(new_saved_variant.saved_variant_json, PARSED_VARIANTS[1]) - def assert_has_expected_calls(self, users, skip_group_call_idxs=None): calls = [mock.call(user) for user in users] @@ -864,7 +827,6 @@ class AnvilSummaryDataAPITest(AnvilAuthenticationTestCase, SummaryDataAPITest): NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = [] HAS_AIRTABLE = True - SAVED_VARIANT_RESPONSE_KEYS = {*SAVED_VARIANT_RESPONSE_KEYS, 'totalSampleCounts'} def test_mme_details(self, *args): super(AnvilSummaryDataAPITest, self).test_mme_details(*args) @@ -878,22 +840,3 @@ def test_saved_variants_page(self): ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') - - def _assert_expected_new_saved_variant(self, new_saved_variant): - super()._assert_expected_new_saved_variant(new_saved_variant) - self.assertEqual(new_saved_variant.key, 101) - self.assertEqual(new_saved_variant.dataset_type, 'SNV_INDEL') - self.assertDictEqual(new_saved_variant.genotypes, { - 'I000004_hg00731': { - 'ab': 0, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000004_hg00731', 'sampleType': 'WES', - }, - 'I000005_hg00732': { - 'ab': 0, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000005_hg00732', 'sampleType': 'WES', - }, - 'I000006_hg00733': { - 'ab': 0, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', - }, - }) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ed322793fb..e0c62e77d8 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,27 +12,27 @@ from math import ceil import re -from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.gene_utils import get_gene -from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - variant_lookup, parse_variant_id -from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY +from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ + variant_lookup, parse_variant_id, export_variants from seqr.utils.search.utils import InvalidSearchException -from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display +from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ get_json_for_saved_searches, add_individual_hpo_details, FAMILY_ADDITIONAL_VALUES from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ - user_is_analyst, login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission + login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission from seqr.views.utils.project_context_utils import get_projects_child_entities -from seqr.views.utils.variant_utils import get_variant_key, get_variants_response +from seqr.views.utils.variant_utils import get_variants_response from seqr.views.utils.vlm_utils import vlm_lookup +logger = SeqrLogger(__name__) + GENOTYPE_AC_LOOKUP = { 'ref_ref': [0, 0], @@ -51,9 +51,6 @@ def query_variants_handler(request, search_hash): """ page = int(request.GET.get('page') or 1) per_page = int(request.GET.get('per_page') or 100) - sort = request.GET.get('sort') or XPOS_SORT_KEY - if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(request.user): - sort = PATHOGENICTY_HGMD_SORT_KEY search_context = json.loads(request.body or '{}') try: @@ -62,10 +59,8 @@ def query_variants_handler(request, search_hash): return create_json_response({'error': str(e)}, status=400, reason=str(e)) _check_results_permission(results_model, request.user) - skip_genotype_filter = bool(_all_project_family_search_genome(search_context)) - variants, total_results = query_variants(results_model, sort=sort, page=page, num_results=per_page, - skip_genotype_filter=skip_genotype_filter, user=request.user) + variants, total_results = query_variants(results_model, sort=request.GET.get('sort'), page=page, num_results=per_page, user=request.user) response = _process_variants(variants or [], results_model.families.all(), request, genome_version=results_model.variant_search.search.get('no_access_project_genome_version')) @@ -143,7 +138,7 @@ def query_single_variant_handler(request, variant_id): family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(family, variant_id, user=request.user) + variant = get_single_variant(family, variant_id, request.user) response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) @@ -155,7 +150,8 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo return {'searchedVariants': variants} flat_variants = _flatten_variants(variants) - saved_variants, variants_by_id = _get_saved_variant_models(flat_variants, families) + variants_by_id = {v['variantId']: v for v in flat_variants} + saved_variants = _get_saved_variant_models(flat_variants) response_json = get_variants_response( request, saved_variants, response_variants=flat_variants, add_all_context=add_all_context, @@ -164,7 +160,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo for saved_variant in response_json['savedVariantsByGuid'].values(): family_guids = saved_variant['familyGuids'] - searched_variant = variants_by_id.get(get_variant_key(**saved_variant)) + searched_variant = variants_by_id.get(saved_variant['variantId']) if not searched_variant: # This can occur when an hg38 family has a saved variant that did not successfully lift from hg37 continue @@ -176,19 +172,6 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo return response_json -PREDICTION_MAP = { - 'D': 'damaging', - 'T': 'tolerated', -} - - -POLYPHEN_MAP = { - 'D': 'probably_damaging', - 'P': 'possibly_damaging', - 'B': 'benign', -} - - MUTTASTR_MAP = { 'A': 'disease_causing', 'D': 'disease_causing', @@ -197,28 +180,15 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo } -def _get_prediction_val(prediction): - try: - return float(prediction or '') - except ValueError: - return PREDICTION_MAP.get(prediction[0]) if prediction else None - - -def _get_variant_main_transcript_field_val(parsed_variant): - return next( - (t for t in parsed_variant['transcripts'] if t['transcriptId'] == parsed_variant['mainTranscriptId']), {} - ).get('value') - - VARIANT_EXPORT_DATA = [ {'header': 'chrom'}, {'header': 'pos'}, {'header': 'ref'}, {'header': 'alt'}, - {'header': 'gene', 'value_path': '{transcripts: transcripts.*[].{value: geneSymbol || geneId, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'worst_consequence', 'value_path': '{transcripts: transcripts.*[].{value: majorConsequence, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'callset_freq', 'value_path': 'populations.callset.af || populations.seqr.ac'}, - {'header': 'exac_freq', 'value_path': 'populations.exac.af'}, + {'header': 'gene', 'value_path': 'mainTranscript.geneId'}, + {'header': 'gene_symbol', 'value_path': 'geneSymbol'}, + {'header': 'worst_consequence', 'value_path': 'mainTranscript.majorConsequence'}, + {'header': 'seqr_ac', 'value_path': 'populations.seqr.ac'}, {'header': 'gnomad_genomes_freq', 'value_path': 'populations.gnomad_genomes.af'}, {'header': 'gnomad_exomes_freq', 'value_path': 'populations.gnomad_exomes.af'}, {'header': 'topmed_freq', 'value_path': 'populations.topmed.af'}, @@ -226,14 +196,14 @@ def _get_variant_main_transcript_field_val(parsed_variant): {'header': 'revel', 'value_path': 'predictions.revel'}, {'header': 'eigen', 'value_path': 'predictions.eigen'}, {'header': 'splice_ai', 'value_path': 'predictions.splice_ai'}, - {'header': 'polyphen', 'value_path': 'predictions.polyphen', 'process': _get_prediction_val}, - {'header': 'sift', 'value_path': 'predictions.sift', 'process': _get_prediction_val}, - {'header': 'muttaster', 'value_path': 'predictions.mut_taster', 'process': _get_prediction_val}, - {'header': 'fathmm', 'value_path': 'predictions.fathmm', 'process': _get_prediction_val}, + {'header': 'polyphen', 'value_path': 'predictions.polyphen'}, + {'header': 'sift', 'value_path': 'predictions.sift'}, + {'header': 'muttaster', 'value_path': 'predictions.mut_taster', 'process': MUTTASTR_MAP.get}, + {'header': 'fathmm', 'value_path': 'predictions.fathmm'}, {'header': 'rsid', 'value_path': 'rsid'}, - {'header': 'hgvsc', 'value_path': '{transcripts: transcripts.*[].{value: hgvsc, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'hgvsp', 'value_path': '{transcripts: transcripts.*[].{value: hgvsp, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'clinvar_clinical_significance', 'value_path': 'clinvar.clinicalSignificance || clinvar.pathogenicity'}, + {'header': 'hgvsc', 'value_path': 'mainTranscript.hgvsc'}, + {'header': 'hgvsp', 'value_path': 'mainTranscript.hgvsp'}, + {'header': 'clinvar_clinical_significance', 'value_path': 'clinvar.pathogenicity'}, {'header': 'clinvar_gold_stars', 'value_path': 'clinvar.goldStars'}, ] @@ -252,7 +222,7 @@ def _get_variant_main_transcript_field_val(parsed_variant): VARIANT_SAMPLE_DATA = [ {'header': 'sample', 'value_path': 'sampleId'}, {'header': 'num_alt_alleles', 'value_path': 'numAlt'}, - {'header': 'filters', 'process': lambda val: ';'.join(val or []), 'variant_value_path': '[genotypeFilters][*]'}, + {'header': 'filters', 'process': lambda val: ';'.join(val or [])}, {'header': 'gq'}, {'header': 'ab'}, ] @@ -282,20 +252,14 @@ def export_variants_handler(request, search_hash): families = results_model.families.all() family_ids_by_guid = {family.guid: family.family_id for family in families} - variants, _ = query_variants(results_model, page=1, load_all=True, user=request.user) - variants = _flatten_variants(variants) + variants = export_variants(results_model, request.user) - saved_variants, variants_by_id = _get_saved_variant_models(variants, families) - json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=families[0].project.genome_version) + saved_variants = _get_saved_variant_models(variants) + json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants) saved_variants_by_variant_family = {} for saved_variant in json_saved_variants['savedVariantsByGuid'].values(): - variant_key = get_variant_key(**saved_variant) - searched_variant = variants_by_id.get(variant_key) - if searched_variant: - # Handles lifted over variant matching - variant_key = get_variant_key(**searched_variant) - saved_variants_by_variant_family[variant_key] = { + saved_variants_by_variant_family[saved_variant['variantId']] = { family_guid: saved_variant['variantGuid'] for family_guid in saved_variant['familyGuids'] } @@ -308,7 +272,7 @@ def export_variants_handler(request, search_hash): num_split = ceil(len(variant['familyGuids']) / MAX_FAMILIES_PER_ROW) gens_per_row = ceil(len(variant['genotypes']) / num_split) - gen_keys = list(variant['genotypes'].keys()) + gen_keys = sorted(variant['genotypes'].keys()) for i in range(num_split): split_var = deepcopy(variant) split_var['familyGuids'] = variant['familyGuids'][i*MAX_FAMILIES_PER_ROW:(i+1)*MAX_FAMILIES_PER_ROW] @@ -325,7 +289,7 @@ def export_variants_handler(request, search_hash): for variant in variants: row = [_get_field_value(variant, config) for config in VARIANT_EXPORT_DATA] - family_saved_variants = saved_variants_by_variant_family.get(get_variant_key(**variant), {}) + family_saved_variants = saved_variants_by_variant_family.get(variant['variantId'], {}) for family_guid in variant['familyGuids']: variant_guid = family_saved_variants.get(family_guid, '') family_tags = { @@ -336,9 +300,9 @@ def export_variants_handler(request, search_hash): row += [_get_field_value(family_tags, config) for config in VARIANT_FAMILY_EXPORT_DATA] row += ['' for i in range(len(VARIANT_FAMILY_EXPORT_DATA) * (max_families_per_variant - len(variant['familyGuids'])))] - genotypes = list(variant['genotypes'].values()) + genotypes = [genotype for _, genotype in sorted(variant['genotypes'].items())] for genotype in genotypes: - row += [_get_field_value(genotype, config, variant=variant) for config in VARIANT_SAMPLE_DATA] + row += [_get_field_value(genotype, config) for config in VARIANT_SAMPLE_DATA] row += ['' for i in range(len(VARIANT_SAMPLE_DATA) * (max_samples_per_variant - len(genotypes)))] rows.append(row) @@ -353,10 +317,8 @@ def export_variants_handler(request, search_hash): return export_table('search_results_{}'.format(search_hash), header, rows, file_format, titlecase_header=False) -def _get_field_value(value, config, variant=None): +def _get_field_value(value, config): field_value = jmespath.search(config.get('value_path', config['header']), value) - if config.get('variant_value_path') and not field_value: - field_value = jmespath.search(config['variant_value_path'], variant) if config.get('process'): field_value = config['process'](field_value) return field_value @@ -402,7 +364,7 @@ def search_context_handler(request): **FAMILY_ADDITIONAL_VALUES, )} - family_sample_types = get_search_samples(projects).values('individual__family__guid').annotate( + family_sample_types = Sample.objects.filter(individual__family__project__in=projects, is_active=True).values('individual__family__guid').annotate( samples=ArrayAgg(JSONObject(sampleType='sample_type', datasetType='dataset_type', isActive=Value(True)), distinct=True)) project_dataset_types = defaultdict(set) for agg in family_sample_types: @@ -520,24 +482,12 @@ def _get_saved_searches(user): return {'savedSearchesByGuid': {search['savedSearchGuid']: search for search in saved_searches}} -def _get_saved_variant_models(variants, families): - hg37_family_guids = families.filter(project__genome_version=GENOME_VERSION_GRCh37).values_list('guid', flat=True) if families else [] - +def _get_saved_variant_models(variants): variant_qs = [] - variants_by_id = {} variant_ids_by_family = defaultdict(set) for variant in variants: - variants_by_id[get_variant_key(**variant)] = variant for family_guid in variant.get('familyGuids', []): variant_ids_by_family[family_guid].add(variant['variantId']) - if variant.get('liftedOverGenomeVersion') == GENOME_VERSION_GRCh37 and hg37_family_guids: - variant_hg37_families = [family_guid for family_guid in variant['familyGuids'] if family_guid in hg37_family_guids] - if variant_hg37_families: - lifted_xpos = get_xpos(variant['liftedOverChrom'], variant['liftedOverPos']) - variant_qs.append(Q(xpos=lifted_xpos, ref=variant['ref'], alt=variant['alt'], family__guid__in=variant_hg37_families)) - variants_by_id[get_variant_key( - xpos=lifted_xpos, ref=variant['ref'], alt=variant['alt'], genomeVersion=variant['liftedOverGenomeVersion'] - )] = variant for family_guid, variant_ids in variant_ids_by_family.items(): variant_qs.append(Q(variant_id__in=variant_ids, family__guid=family_guid)) @@ -550,7 +500,7 @@ def _get_saved_variant_models(variants, families): else: saved_variants = SavedVariant.objects.none() - return saved_variants, variants_by_id + return saved_variants def _flatten_variants(variants): flattened_variants = [] @@ -581,7 +531,7 @@ def variant_lookup_handler(request): for variant in variants: variant['familyGuids'] = list(families.values_list('guid', flat=True)) - saved_variants, _ = _get_saved_variant_models(variants, None) if families else (None, None) + saved_variants = _get_saved_variant_models(variants) if families else None response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, @@ -592,12 +542,12 @@ def variant_lookup_handler(request): (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() } for variant in variants: - _update_lookup_variant(variant, response, individual_guid_map) + _update_lookup_variant(variant, response, individual_guid_map, request.user) return create_json_response(response) -def _update_lookup_variant(variant, response, individual_guid_map): +def _update_lookup_variant(variant, response, individual_guid_map, user): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) @@ -613,13 +563,19 @@ def _update_lookup_variant(variant, response, individual_guid_map): variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): - individual_guid = individual_guid_map[(family_guid, genotype['sampleId'])] + individual_guid = individual_guid_map.get((family_guid, genotype['sampleId'])) + if not individual_guid: + logger.error( + f'Unable to map sample {genotype["sampleId"]} in family {family_guid} to an individual for variant {variant["variantId"]}', + user, + ) + continue genotype = {**genotype, 'individualGuid': individual_guid} if individual_guid in variant['genotypes']: genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype - for i, (unmapped_family_guid, genotypes) in enumerate(variant.pop('familyGenotypes').items()): + for i, (unmapped_family_guid, genotypes) in enumerate(sorted(variant.pop('familyGenotypes').items())): family_guid = f'F{i}_{variant["variantId"]}' variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index ca1ec7f853..119b662664 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -5,20 +5,12 @@ from django.db import transaction from django.urls.base import reverse -from elasticsearch.exceptions import ConnectionTimeout, TransportError -from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, \ - SV_LOOKUP_VARIANT, SV_VARIANT4, GCNV_VARIANT4 -from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch -from seqr.utils.search.utils import InvalidSearchException -from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException -from seqr.views.apis.variant_search_api import query_variants_handler, query_single_variant_handler, vlm_lookup_handler, \ - export_variants_handler, search_context_handler, get_saved_search_handler, create_saved_search_handler, \ - update_saved_search_handler, delete_saved_search_handler, get_variant_gene_breakdown, variant_lookup_handler -from seqr.views.utils.test_utils import AuthenticationTestCase, VARIANTS, AnvilAuthenticationTestCase,\ - GENE_VARIANT_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS, FAMILY_FIELDS, \ - PA_LOCUS_LIST_FIELDS, INDIVIDUAL_FIELDS, FUNCTIONAL_FIELDS, IGV_SAMPLE_FIELDS, FAMILY_NOTE_FIELDS, ANALYSIS_GROUP_FIELDS, \ - VARIANT_NOTE_FIELDS, TAG_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_DETAIL_FIELDS, DYNAMIC_ANALYSIS_GROUP_FIELDS +from seqr.models import LocusList, Project, VariantSearch +from seqr.views.apis.variant_search_api import vlm_lookup_handler, search_context_handler, get_saved_search_handler, \ + create_saved_search_handler, update_saved_search_handler, delete_saved_search_handler +from seqr.views.utils.test_utils import AuthenticationTestCase, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, \ + ANALYSIS_GROUP_FIELDS, DYNAMIC_ANALYSIS_GROUP_FIELDS LOCUS_LIST_GUID = 'LL00049_pid_genes_autosomal_do' PROJECT_GUID = 'R0001_1kg' @@ -26,113 +18,8 @@ SEARCH = {'filters': {}, 'inheritance': None} PROJECT_FAMILIES = [{'projectGuid': PROJECT_GUID, 'familyGuids': ['F000001_1', 'F000002_2']}] -ALL_VARIANTS = VARIANTS + [VARIANT2, VARIANT3] - -VARIANTS_WITH_DISCOVERY_TAGS = deepcopy(ALL_VARIANTS) -DISCOVERY_TAGS = [{ - 'savedVariant': { - 'variantGuid': 'SV0000006_1248367227_r0003_tes', - 'familyGuid': 'F000012_12', - 'projectGuid': 'R0003_test', - }, - 'tagGuid': 'VT1726961_2103343353_r0003_tes', - 'name': 'Tier 1 - Novel gene and phenotype', - 'category': 'CMG Discovery Tags', - 'color': '#03441E', - 'searchHash': None, - 'metadata': None, - 'lastModifiedDate': '2018-05-29T16:32:51.449Z', - 'createdBy': None, -}] -VARIANTS_WITH_DISCOVERY_TAGS[2]['discoveryTags'] = DISCOVERY_TAGS - -SINGLE_FAMILY_VARIANT = deepcopy(VARIANTS[0]) -SINGLE_FAMILY_VARIANT['familyGuids'] = ['F000001_1'] - PROJECT_CONTEXT_FIELDS = {'locusListGuids', 'datasetTypes', 'analysisGroupsLoaded', 'projectGuid', 'name'} -PROJECT_TAG_TYPE_FIELDS = {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'} - -EXPECTED_TAG = {k: mock.ANY for k in TAG_FIELDS} -expected_functional_tag = {k: mock.ANY for k in FUNCTIONAL_FIELDS} -expected_aip_tag = { - 'structuredMetadata': { - '4': {'date': '2023-11-15', 'name': 'de Novo'}, - 'support': {'date': '2023-11-15', 'name': 'High in Silico Scores'}, - }, - **EXPECTED_TAG, -} -del expected_aip_tag['metadata'] -EXPECTED_GENE = {k: mock.ANY for k in GENE_VARIANT_FIELDS} -EXPECTED_GENE['locusListGuids'] = [] -expected_pa_gene = deepcopy(EXPECTED_GENE) -expected_pa_gene['locusListGuids'] = ['LL00049_pid_genes_autosomal_do'] -expected_pa_gene['panelAppDetail'] = mock.ANY -EXPECTED_SAVED_VARIANT = {k: mock.ANY for k in SAVED_VARIANT_DETAIL_FIELDS} -expected_detail_saved_variant = deepcopy(EXPECTED_SAVED_VARIANT) -expected_detail_saved_variant['mmeSubmissions'] = [ - {'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}, -] -expected_detail_saved_variant['mainTranscriptId'] = mock.ANY - -EXPECTED_EXOMISER_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, - 'scores': {'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, - 'variant_score': 0.999200702}}, - {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, - 'scores': {'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, - 'variant_score': 1}}, -] - -EXPECTED_LIRICAL_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, - 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, -] - -EXPECTED_SEARCH_RESPONSE = { - 'searchedVariants': ALL_VARIANTS, - 'savedVariantsByGuid': { - 'SV0000001_2103343353_r0390_100': expected_detail_saved_variant, - 'SV0000002_1248367227_r0390_100': EXPECTED_SAVED_VARIANT, - }, - 'genesById': { - 'ENSG00000227232': expected_pa_gene, 'ENSG00000268903': EXPECTED_GENE, 'ENSG00000233653': EXPECTED_GENE, - 'ENSG00000177000': mock.ANY, 'ENSG00000097046': mock.ANY, - }, - 'search': { - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': PROJECT_GUID, 'familyGuids': mock.ANY}], - 'totalResults': 5, - }, - 'variantTagsByGuid': { - 'VT1708633_2103343353_r0390_100': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726961_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726985_2103343353_r0390_100': expected_aip_tag, - }, - 'variantNotesByGuid': { - 'VN0714935_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - 'VN0714937_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - }, - 'variantFunctionalDataByGuid': { - 'VFD0000023_1248367227_r0390_10': expected_functional_tag, 'VFD0000024_1248367227_r0390_10': expected_functional_tag, - 'VFD0000025_1248367227_r0390_10': expected_functional_tag, 'VFD0000026_1248367227_r0390_10': expected_functional_tag, - }, - 'locusListsByGuid': {LOCUS_LIST_GUID: {'intervals': mock.ANY}}, - 'rnaSeqData': { - 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, - }, - 'phenotypeGeneScores': { - 'I000001_na19675': {'ENSG00000268903': {'exomiser': EXPECTED_EXOMISER_DATA}}, - 'I000002_na19678': {'ENSG00000268903': {'lirical': EXPECTED_LIRICAL_DATA}}, - }, - 'mmeSubmissionsByGuid': {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}, - 'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}, -} - -EXPECTED_TRANSCRIPTS_RESPONSE = { - 'transcriptsById': {'ENST00000624735': {'isManeSelect': False, 'refseqId': None, 'transcriptId': 'ENST00000624735'}}, -} - EXPECTED_SEARCH_CONTEXT_RESPONSE = { 'savedSearchesByGuid': { 'VS0079516_': mock.ANY, 'VS0079525_': mock.ANY, 'VS0079517_': mock.ANY, 'VS0145435_': mock.ANY, @@ -143,14 +30,6 @@ 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY, 'LL00005_retina_proteome': mock.ANY}, } -EXPECTED_SEARCH_FAMILY_CONTEXT = { - 'familiesByGuid': {'F000001_1': mock.ANY, 'F000002_2': mock.ANY}, - 'individualsByGuid': mock.ANY, - 'igvSamplesByGuid': mock.ANY, - 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY}, - 'familyNotesByGuid': mock.ANY, -} - MOCK_TOKEN = 'mock_token' # nosec MOCK_CLIENT_ID = 'mock_client_id' VLM_CLIENTS_RESPONSE = [ @@ -261,24 +140,10 @@ } } -def _get_es_variants(results_model, **kwargs): - results_model.save() - return deepcopy(ALL_VARIANTS), len(ALL_VARIANTS) - - -def _get_empty_es_variants(results_model, **kwargs): - results_model.save() - return [], 0 - - -COMP_HET_VARAINTS = [[VARIANTS[2], VARIANTS[1]]] -def _get_compound_het_es_variants(results_model, **kwargs): - results_model.save() - return deepcopy(COMP_HET_VARAINTS), 1 - @mock.patch('seqr.views.utils.permissions_utils.safe_redis_get_json', lambda *args: None) -class VariantSearchAPITest(object): +class VariantSearchAPITest(AuthenticationTestCase): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches', 'clickhouse_saved_variants'] def _assert_expected_search_context(self, response_json): self.assertSetEqual(set(response_json), set(EXPECTED_SEARCH_CONTEXT_RESPONSE)) @@ -305,505 +170,6 @@ def _assert_expected_search_context(self, response_json): 'analysedBy': [{'createdBy': 'Test No Access User', 'dataType': 'SNP', 'lastModifiedDate': '2022-07-22T19:27:08.563+00:00'}], }) - def _assert_expected_rnaseq_response(self, response_json): - self.assertDictEqual( - response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], - {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, - 'tissueType': 'M', 'zScore': 7.08} - ) - self.assertListEqual( - sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), - [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, - 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, - {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, - 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, - {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, - 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] - ) - - def _assert_expected_results_family_context(self, response_json, locus_list_detail=False): - self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) - - family_fields = {'individualGuids'} - family_fields.update(FAMILY_FIELDS) - if len(response_json['familiesByGuid']) > 1: - self.assertSetEqual(set(response_json['familiesByGuid']['F000002_2'].keys()), family_fields) - - family_fields.add('tpmGenes') - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) - - self.assertEqual(len(response_json['individualsByGuid']), len(response_json['familiesByGuid'])*3) - individual_fields = {'igvSampleGuids'} - individual_fields.update(INDIVIDUAL_FIELDS) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675'].keys()), individual_fields) - - self.assertEqual(len(response_json['igvSamplesByGuid']), 1) - self.assertSetEqual(set(response_json['igvSamplesByGuid']['S000145_na19675'].keys()), IGV_SAMPLE_FIELDS) - - self.assertEqual(len(response_json['familyNotesByGuid']), 3) - self.assertSetEqual(set(response_json['familyNotesByGuid']['FAN000001_1'].keys()), FAMILY_NOTE_FIELDS) - - self._assert_expected_rnaseq_response(response_json) - - def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True): - gene_fields = {'locusListGuids'} - gene_fields.update(GENE_VARIANT_FIELDS) - basic_gene_id = next(gene_id for gene_id in ['ENSG00000268903', 'ENSG00000233653'] if gene_id in response_json['genesById']) - self.assertSetEqual(set(response_json['genesById'][basic_gene_id].keys()), gene_fields) - if has_pa_detail: - gene_fields.add('panelAppDetail') - self.assertSetEqual(set(response_json['genesById']['ENSG00000227232'].keys()), gene_fields) - self.assertListEqual( - response_json['genesById']['ENSG00000227232']['locusListGuids'], [LOCUS_LIST_GUID] - ) - self.assertDictEqual( - response_json['genesById']['ENSG00000227232']['panelAppDetail'], {LOCUS_LIST_GUID: {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}} - ) - - locus_list_fields = {'intervals'} - if locus_list_detail: - locus_list_fields.update(LOCUS_LIST_FIELDS) - if has_pa_detail: - locus_list_fields.update({'paLocusList'}) - self.assertSetEqual(set(response_json['locusListsByGuid'][LOCUS_LIST_GUID].keys()), locus_list_fields) - intervals = response_json['locusListsByGuid'][LOCUS_LIST_GUID]['intervals'] - self.assertEqual(len(intervals), 2) - self.assertSetEqual( - set(intervals[0].keys()), - {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} - ) - - self.assertSetEqual(set(next(iter(response_json['variantTagsByGuid'].values())).keys()), TAG_FIELDS) - if response_json['variantNotesByGuid']: - self.assertSetEqual(set(next(iter(response_json['variantNotesByGuid'].values())).keys()), VARIANT_NOTE_FIELDS) - if response_json['variantFunctionalDataByGuid']: - self.assertSetEqual(set(next(iter(response_json['variantFunctionalDataByGuid'].values())).keys()), FUNCTIONAL_FIELDS) - - if rnaseq: - self._assert_expected_rnaseq_response(response_json) - - @mock.patch('seqr.utils.middleware.logger.error') - @mock.patch('seqr.views.apis.variant_search_api.get_variant_query_gene_counts') - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_error_logger): - url = reverse(query_variants_handler, args=['abc']) - self.check_collaborator_login(url, request_data={'projectFamilies': PROJECT_FAMILIES}) - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - - # Test invalid inputs - response = self.client.get(url) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.reason_phrase, 'Invalid search hash: {}'.format(SEARCH_HASH)) - mock_error_logger.assert_not_called() - - response = self.client.post(url, content_type='application/json', data=json.dumps({'search': SEARCH})) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.reason_phrase, 'Invalid search: no projects/ families specified') - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = InvalidIndexException('Invalid index') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Invalid index') - self.assertFalse('traceback' in response.json()) - mock_error_logger.assert_called_with( - 'Invalid index', self.collaborator_user, http_request_json=mock.ANY, traceback=mock.ANY, request_body=mock.ANY, detail=None) - - mock_get_variants.side_effect = InvalidSearchException('Invalid search') - mock_error_logger.reset_mock() - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Invalid search') - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = ConnectionTimeout('', '', ValueError('Timeout')) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 504) - self.assertEqual(response.json()['error'], 'ConnectionTimeout caused by - ValueError(Timeout)') - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = TransportError('N/A', 'search_phase_execution_exception', {'error': 'Invalid'}) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], "TransportError: N/A - 'search_phase_execution_exception' - 'Invalid'") - self.assertEqual(response.json()['detail'], {'error': 'Invalid'}) - mock_error_logger.assert_not_called() - - error_info_json = {'error': {'root_cause': [{'type': 'response_handler_failure_transport_exception'}]}} - mock_get_variants.side_effect = TransportError('401', 'search_phase_execution_exception', error_info_json) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 401) - self.assertEqual( - response.json()['error'], - "TransportError: 401 - 'search_phase_execution_exception' - response_handler_failure_transport_exception") - self.assertEqual(response.json()['detail'], error_info_json) - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = _get_es_variants - - # Test new search - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(self.EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), {'F000001_1', 'F000002_2'}) - self._assert_expected_results_context(response_json) - - results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) - mock_error_logger.assert_not_called() - - # include project context info - response = self.client.get('{}?loadProjectTagTypes=true'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_response = {'projectsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['projectsByGuid']} - expected_search_response.update(self.EXPECTED_SEARCH_RESPONSE) - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_context(response_json) - self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) - - # include family context info - response = self.client.get('{}?loadFamilyContext=true'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_response = { - **self.EXPECTED_SEARCH_RESPONSE, - **EXPECTED_SEARCH_FAMILY_CONTEXT, - } - self.assertSetEqual(set(response_json.keys()), set(expected_response.keys())) - self.assertDictEqual(response_json, expected_response) - self._assert_expected_results_family_context(response_json) - - # Test pagination - response = self.client.get('{}?page=3'.format(url)) - self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=3, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) - mock_error_logger.assert_not_called() - - # Test sort - response = self.client.get('{}?sort=pathogenicity'.format(url)) - self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='pathogenicity', page=1, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) - mock_error_logger.assert_not_called() - - # Test export - export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) - response = self.client.get(export_url) - self.assertEqual(response.status_code, 200) - expected_content = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', - 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', - 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', - 'tags_1', 'notes_1', 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', - 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', - '', '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '2', '', '', 'NA19675', '1', 'VQSRTrancheSNP99.95to100.00', '46.0', - '0.702127659574', 'NA19679', '0', 'VQSRTrancheSNP99.95to100.00', '99.0', '0.0', '', '', '', '', ''], - ['3', '835', 'AAAG', 'A', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '1', '', '', '', '', '', 'NA19679', '0', 'artifact_prone_site', '99.0', '0.0', '', '', '', '', '', '', '', '', '', ''], - ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', - '', '', '', '', '', '', '', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', - '0.00288997', '0.246152', '25.9', '0.197', - '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', - 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1.0', - 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0.0', - '0.362681', '28.754', '', '1.378', '0.01', '', '', '', - '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741'], - ] - self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) - - # test export with max families - with mock.patch('seqr.views.apis.variant_search_api.MAX_FAMILIES_PER_ROW', 1): - response = self.client.get(export_url) - self.assertEqual(response.status_code, 200) - expected_content = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', - 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', - 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', - 'tags_1', 'notes_1', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', - 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', '', - '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', 'NA19675', '1', 'VQSRTrancheSNP99.95to100.00', '46.0', '0.702127659574', - '', '', '', '', '', '', '', '', '', '',], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', '', - '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '2', '', '', - 'NA19679', '0', 'VQSRTrancheSNP99.95to100.00', '99.0', '0.0', '', '', '', '', '', '', '', '', '', '',], - ['3', '835', 'AAAG', 'A', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '1', '', '', 'NA19679', '0', 'artifact_prone_site', '99.0', '0.0', '', '', '', '', '', '', '', '', '', '',], - ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', - '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', - '0.00288997', '0.246152', '25.9', '0.197', - '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', - 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', 'HG00731', '2', '', '99', '1.0', - 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0.0', - '0.362681', '28.754', '', '1.378', '0.01', '', '', - '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', - '0.40741'], - ] - self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) - - mock_get_variants.assert_called_with(results_model, page=1, load_all=True, user=self.collaborator_user) - mock_error_logger.assert_not_called() - - # Test gene breakdown - gene_counts = { - 'ENSG00000227232': {'total': 2, 'families': {'F000001_1': 2, 'F000002_2': 1}}, - 'ENSG00000268903': {'total': 1, 'families': {'F000002_2': 1}} - } - mock_get_gene_counts.return_value = gene_counts - - gene_breakdown_url = reverse(get_variant_gene_breakdown, args=[SEARCH_HASH]) - response = self.client.get(gene_breakdown_url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'searchGeneBreakdown', 'genesById'}) - self.assertDictEqual(response_json['searchGeneBreakdown'], {SEARCH_HASH: gene_counts}) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000227232', 'ENSG00000268903'}) - self.assertSetEqual(set(response_json['genesById']['ENSG00000227232'].keys()), GENE_VARIANT_DISPLAY_FIELDS) - - # Test compound hets - mock_get_variants.side_effect = _get_compound_het_es_variants - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_response = deepcopy(self.EXPECTED_SEARCH_RESPONSE) - expected_search_response.update({ - 'searchedVariants': COMP_HET_VARAINTS, - 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, - 'genesById': {'ENSG00000233653': EXPECTED_GENE}, - 'variantTagsByGuid': { - 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726985_2103343353_r0390_100': expected_aip_tag, - }, - 'variantFunctionalDataByGuid': {}, - 'phenotypeGeneScores': {}, - 'rnaSeqData': {}, - 'mmeSubmissionsByGuid': {}, - }) - expected_search_response['search']['totalResults'] = 1 - del expected_search_response['familiesByGuid'] - expected_search_response.pop('transcriptsById', None) - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_context(response_json, has_pa_detail=False, rnaseq=False) - mock_error_logger.assert_not_called() - - # Test cross-project discovery for analyst users - self.login_analyst_user() - mock_get_variants.side_effect = _get_es_variants - response = self.client.get('{}?sort=pathogenicity'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_results = deepcopy(self.EXPECTED_SEARCH_RESPONSE) - expected_search_results['searchedVariants'] = VARIANTS_WITH_DISCOVERY_TAGS - expected_search_results['savedVariantsByGuid']['SV0000002_1248367227_r0390_100']['discoveryTags'] = DISCOVERY_TAGS - expected_search_results['familiesByGuid'].update({'F000012_12': mock.ANY}) - self.assertSetEqual(set(response_json.keys()), set(expected_search_results.keys())) - self.assertDictEqual(response_json, expected_search_results) - self._assert_expected_results_context(response_json) - - mock_get_variants.assert_called_with(results_model, sort='pathogenicity_hgmd', page=1, num_results=100, skip_genotype_filter=False, user=self.analyst_user) - mock_error_logger.assert_not_called() - - # Test no results - VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete() - mock_get_variants.side_effect = _get_empty_es_variants - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': {**SEARCH, 'exclude': {'previousSearch': True}}, 'previousSearchHash': 'abc1234', - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json, { - 'searchedVariants': [], - 'search': { - 'search': {**SEARCH, 'exclude': { - 'previousSearch': True, - 'previousSearchHash': 'abc1234', - }}, - 'projectFamilies': PROJECT_FAMILIES, - 'totalResults': 0, - } - }) - mock_error_logger.assert_not_called() - - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_all_projects_variants(self, mock_get_variants): - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - self.check_require_login(url) - - expected_searched_families = set() - def _get_variants(results_model, **kwargs): - results_model.save() - self.assertSetEqual(expected_searched_families, {f.guid for f in results_model.families.all()}) - matched_variants = [ - deepcopy(variant) for variant in ALL_VARIANTS - if any(family_guid in expected_searched_families for family_guid in variant['familyGuids']) - ] - return matched_variants, len(matched_variants) - - mock_get_variants.side_effect = _get_variants - - body = {'allGenomeProjectFamilies': '37', 'search': SEARCH} - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - empty_search_response = { - 'searchedVariants': [], 'search': { - 'search': SEARCH, - 'projectFamilies': [], - 'totalResults': 0, - }} - self.assertDictEqual(response_json, empty_search_response) - results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=True, user=self.no_access_user) - - VariantSearchResults.objects.filter(search_hash=SEARCH_HASH).delete() - self.login_data_manager_user() - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), empty_search_response) - - VariantSearchResults.objects.filter(search_hash=SEARCH_HASH).delete() - self.login_collaborator() - expected_searched_families = { - 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', - 'F000009_9', 'F000010_10', 'F000013_13'} - body['projectFamilies'] = [{'projectGuid': PROJECT_GUID, 'familyGuids': ['F000001_1']}] - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - - result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) - mock_get_variants.assert_called_with(result_model, sort='xpos', page=1, num_results=100, - skip_genotype_filter=True, user=self.collaborator_user) - - result_model.delete() - body['includeNoAccessProjects'] = True - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_no_access_resonse = deepcopy(self.EXPECTED_SEARCH_RESPONSE) - expected_no_access_resonse['search']['search']['no_access_project_genome_version'] = '37' - self.assertDictEqual(response_json, expected_no_access_resonse) - self._assert_expected_results_context(response_json) - self.assertSetEqual(set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) - mock_get_variants.assert_called_with( - result_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=True, user=self.collaborator_user, - ) - - result_model.delete() - expected_searched_families.remove('F000007_7') - expected_searched_families.remove('F000010_10') - body['unsolvedFamiliesOnly'] = True - del body['includeNoAccessProjects'] - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(self.EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - - VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete() - body['trioFamiliesOnly'] = True - expected_searched_families = {'F000002_2'} - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_variants = self.EXPECTED_SEARCH_RESPONSE['searchedVariants'][:1] + self.EXPECTED_SEARCH_RESPONSE['searchedVariants'][2:] - expected_response = { - **self.EXPECTED_SEARCH_RESPONSE, - 'searchedVariants': expected_variants, - 'search': {**self.EXPECTED_SEARCH_RESPONSE['search'], 'totalResults': len(expected_variants)}, - 'mmeSubmissionsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'savedVariantsByGuid': { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() - if k != 'SV0000001_2103343353_r0390_100' - }, - 'variantTagsByGuid': { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'].items() - if k not in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'] - }, - } - self.assertSetEqual(set(response_json.keys()), set(expected_response.keys())) - self.assertDictEqual(response_json, expected_response) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_all_project_families_variants(self, mock_get_variants): - url = reverse(query_variants_handler, args=['abc']) - self.check_collaborator_login(url, request_data={'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}]}) - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - - mock_get_variants.side_effect = _get_es_variants - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}], - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json['search'], { - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}], - 'totalResults': 5, - }) - - mock_get_variants.assert_called_with( - VariantSearchResults.objects.get(search_hash=SEARCH_HASH), sort='xpos', page=1, num_results=100, - skip_genotype_filter=False, user=self.collaborator_user) - - # Test export disabled in demo projects - export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) - response = self.client.get(export_url) - self.assertEqual(response.status_code, 403) - def test_search_context(self): search_context_url = reverse(search_context_handler) self.check_collaborator_login(search_context_url, request_data={'familyGuid': 'F000001_1'}) @@ -882,333 +248,6 @@ def test_search_context(self): self._assert_expected_search_context(response_json) - @mock.patch('seqr.views.apis.variant_search_api.get_single_variant') - def test_query_single_variant(self, mock_get_variant): - mock_get_variant.return_value = SINGLE_FAMILY_VARIANT - - url = '{}?familyGuid=F000001_1'.format(reverse(query_single_variant_handler, args=['21-3343353-GAGA-G'])) - self.check_collaborator_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self._assert_expected_single_variant_results_context(response.json(), searchedVariants=[SINGLE_FAMILY_VARIANT]) - - mock_get_variant.assert_called_with(mock.ANY, '21-3343353-GAGA-G', user=self.collaborator_user) - searched_family = mock_get_variant.call_args.args[0] - self.assertEqual(searched_family.guid, 'F000001_1') - - mock_get_variant.side_effect = InvalidSearchException('Variant not found') - response = self.client.get(url) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Variant not found') - - def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response): - omit_fields = {'search', *(omit_fields or [])} - - expected_search_response = deepcopy({**self.EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) - expected_search_response.update(expected_response) - expected_search_response.update({ - k: EXPECTED_SEARCH_CONTEXT_RESPONSE[k] for k in ['projectsByGuid', 'familiesByGuid', 'locusListsByGuid'] - }) - for k in omit_fields: - expected_search_response.pop(k) - expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') - expected_search_response['variantTagsByGuid'] = { - k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] - for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} - } - expected_search_response['variantNotesByGuid'] = {} - expected_search_response['genesById'] = { - k: v for k, v in expected_search_response['genesById'].items() if k in {'ENSG00000227232', 'ENSG00000268903'} - } - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_family_context(response_json, locus_list_detail=True) - self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) - self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1'}) - - @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') - def test_variant_lookup(self, mock_variant_lookup): - response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(response_variant)] - - url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38&affectedOnly=true' - self.check_require_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A', 'F2_1-10439-AC-A'], - 'liftedFamilyGuids': ['F2_1-10439-AC-A'], - 'genotypes': { - 'I0_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I1_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I2_F0_1-10439-AC-A': [ - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I0_F1_1-10439-AC-A': [ - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, - }, - } - del expected_variant['familyGenotypes'] - expected_individuals = { - 'I0_F0_1-10439-AC-A': { - 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I0_F1_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F1_1-10439-AC-A', 'individualGuid': 'I0_F1_1-10439-AC-A', 'sex': 'M', - 'features': [{'category': 'HP:0001626', 'label': '1 terms'}, {'category': 'Other', 'label': '1 terms'}], - 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', - }, - 'I0_F2_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I1_F0_1-10439-AC-A': { - 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I2_F0_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', - 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, - {'category': 'HP:0001626', 'label': '1 terms'}], - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - } - expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None, affected_only=True, hom_only=False) - - response_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_gene_body = { - 'genesById': {'ENSG00000227232': EXPECTED_GENE, 'ENSG00000268903': EXPECTED_GENE}, - } - expected_body.update(expected_gene_body) - if 'transcriptsById' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['transcriptsById'] = self.EXPECTED_SEARCH_RESPONSE['transcriptsById'] - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), expected_body) - - response_variant['variantId'] = '1-248367227-TC-T' - response_variant['genomeVersion'] = '37' - self.login_manager() - response = self.client.get(url.replace("38", "37") + '&homOnly=true') - self.assertEqual(response.status_code, 200) - - individual_guid_map = [ - ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), - ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), - ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), - ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), - ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), - ] - expected_variant.update({ - 'lookupFamilyGuids': ['F000002_2', 'F000011_11', 'F000014_14'], - 'liftedFamilyGuids': ['F000014_14'], - 'genotypes': { - individual_guid: [ - {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] - ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { - **expected_variant['genotypes'][anon_individual_guid], **genotype, 'individualGuid': individual_guid, - } for individual_guid, anon_individual_guid, genotype in individual_guid_map - }, - 'genomeVersion': '37', - 'variantId': '1-248367227-TC-T', - }) - expected_individuals = { - individual_guid: { - **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, - **{k: v for k, v in expected_individuals[anon_individual_guid].items() - if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, - } for individual_guid, anon_individual_guid, _ in individual_guid_map - } - expected_individuals.update({individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) - expected_body = self._expected_lookup_body( - expected_individuals, [expected_variant], include_context=True, - project_guids=[PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], - ) - expected_body.update({ - k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} - }) - expected_body['savedVariantsByGuid']= { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] - } - expected_body.update(expected_gene_body) - expected_body['genesById']['ENSG00000227232'] = expected_pa_gene - expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop('MS000001_na19675') - expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY - expected_body['variantTagsByGuid']['VT1726970_2103343353_r0004_tes'] = EXPECTED_TAG - expected_body['variantTagsByGuid']['VT1726961_2103343353_r0005_tes'] = EXPECTED_TAG - for k in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100']: - del expected_body['variantTagsByGuid'][k] - expected_body['rnaSeqData']['I000019_na21987'] = {'outliers': {}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}} - - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.manager_user, '1-10439-AC-A', '37', sample_type=None, affected_only=True, hom_only=True, - ) - - def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): - exclude_keys = {'searchedVariants', 'search', 'transcriptsById'} - if not include_context: - exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) - expected_body = { - **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, - **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, - 'projectsByGuid': { - p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids or [] - }, - 'familiesByGuid': { - f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids or [] - }, - 'individualsByGuid': individuals_by_guid, - 'variants': variants, - } - if is_build_38: - expected_body['omimIntervals'] = {} - if include_context: - expected_body.update({ - **EXPECTED_TRANSCRIPTS_RESPONSE, - 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], - }) - if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] - return expected_body - - @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') - def test_sv_variant_lookup(self, mock_variant_lookup): - lookup_variants = [ - {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}}, - SV_LOOKUP_VARIANT, - ] - mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(v) for v in lookup_variants] - - url = f'{reverse(variant_lookup_handler)}?variantId=phase2_DEL_chr14_4640&genomeVersion=37&sampleType=WGS' - self.check_require_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant = { - **GCNV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], - 'genotypes': { - 'I0_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_sv_variant = { - **SV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], - 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_individuals = { - 'I0_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I0_F0_suffix_140608_DUP': { - 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', - 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, {'category': 'HP:0001626', 'label': '1 terms'}], - 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I1_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I1_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', - 'sex': 'M','features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I2_F0_phase2_DEL_chr14_4640': { - 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I2_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', - 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - } - expected_body = self._expected_lookup_body(expected_individuals,[expected_gcnv_variant, expected_sv_variant]) - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False, - ) - - self.login_manager() - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant.update({ - 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], - 'genotypes': GCNV_VARIANT4['genotypes'], - }) - expected_sv_variant.update({ - 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], - 'genotypes': SV_VARIANT4['genotypes'], - }) - expected_individuals = { - i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in [ - 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654', - ] - } - expected_body = self._expected_lookup_body( - expected_individuals, - [expected_gcnv_variant, expected_sv_variant], - project_guids=[PROJECT_GUID, 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000014_14'], - include_context=True, - ) - del expected_body['transcriptsById'] - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False,) - @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_SECRET', 'abc123') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_ID', MOCK_CLIENT_ID) @mock.patch('seqr.utils.redis_utils.redis.StrictRedis') @@ -1386,91 +425,3 @@ def test_search_results_redirect(self): response = self.client.get('/report/custom_search/6ebb895dfca0f63c34be1ca59d950205?page=2&sort=cadd') self.assertEqual(response.status_code, 301) self.assertEqual(response.url, '/variant_search/results/6ebb895dfca0f63c34be1ca59d950205?page=2&sort=cadd') - - -# Tests for AnVIL access disabled -class LocalVariantSearchAPITest(AuthenticationTestCase, VariantSearchAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'variant_searches'] - - EXPECTED_SEARCH_RESPONSE = { - **EXPECTED_SEARCH_RESPONSE, - **EXPECTED_TRANSCRIPTS_RESPONSE, - } - - -def assert_no_list_ws_has_al(self, acl_call_count, group_call_count, workspace_name=None): - self.mock_list_workspaces.assert_not_called() - assert_ws_has_al(self, acl_call_count, group_call_count, workspace_name) - - -def assert_has_list_ws(self, has_data_manager=False): - calls = [ - mock.call(self.no_access_user), - mock.call(self.collaborator_user), - ] - if has_data_manager: - calls.insert(1, mock.call(self.data_manager_user)) - self.mock_list_workspaces.assert_has_calls(calls) - - -def assert_no_al_has_list_ws(self, group_count=1, has_data_manager=False): - assert_has_list_ws(self, has_data_manager) - self.mock_get_ws_access_level.assert_not_called() - assert_workspace_calls(self, group_count) - - -def assert_ws_has_al(self, acl_call_count, group_call_count, workspace_name=None, user=None): - if not workspace_name: - workspace_name = 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de' - self.mock_get_ws_access_level.assert_called_with(self.collaborator_user, 'my-seqr-billing', workspace_name) - self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) - assert_workspace_calls(self, group_call_count, user) - - -def assert_workspace_calls(self, group_call_count, user=None): - self.assertEqual(self.mock_get_groups.call_count, group_call_count) - self.mock_get_groups.assert_called_with(user or self.collaborator_user) - - self.mock_get_ws_acl.assert_not_called() - self.mock_get_group_members.assert_not_called() - - -# Test for permissions from AnVIL only -class AnvilVariantSearchAPITest(AnvilAuthenticationTestCase, VariantSearchAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches', 'clickhouse_saved_variants'] - - EXPECTED_SEARCH_RESPONSE = { - **EXPECTED_SEARCH_RESPONSE, - 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, - } - EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'] = { - k: {**v, 'key': mock.ANY, 'mainTranscriptId': mock.ANY} for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() - } - - def test_query_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_variants(*args) - assert_ws_has_al(self, 1, 9, user=self.analyst_user) - assert_has_list_ws(self) - - def test_query_all_projects_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_all_projects_variants(*args) - assert_no_al_has_list_ws(self, group_count=4, has_data_manager=True) - - def test_query_all_project_families_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_all_project_families_variants(*args) - assert_no_al_has_list_ws(self) - - def test_search_context(self): - super(AnvilVariantSearchAPITest, self).test_search_context() - assert_no_al_has_list_ws(self, 12) - - def test_query_single_variant(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_single_variant(*args) - assert_no_list_ws_has_al(self, 4, 1) - - def test_saved_search(self): - super(AnvilVariantSearchAPITest, self).test_saved_search() - assert_workspace_calls(self, 3, user=self.no_access_user) - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_not_called() - diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 8d754b8c57..0b40591a39 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -15,7 +15,6 @@ VLM_CLIENT_ID, ) from seqr.models import WarningMessage -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import get_json_for_user, get_json_for_current_user from seqr.views.utils.permissions_utils import login_active_required @@ -55,7 +54,6 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'version': '{}-{}'.format(SEQR_VERSION, ui_version), 'hijakEnabled': DEBUG or False, 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, - 'elasticsearchEnabled': backend_specific_call(True, False), 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], }} diff --git a/seqr/views/react_app_tests.py b/seqr/views/react_app_tests.py index 10e2d48975..f363edcb09 100644 --- a/seqr/views/react_app_tests.py +++ b/seqr/views/react_app_tests.py @@ -24,7 +24,6 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, 'version': mock.ANY, 'hijakEnabled': False, 'oauthLoginProvider': self.OAUTH_PROVIDER, - 'elasticsearchEnabled': bool(self.ES_HOSTNAME), 'vlmEnabled': vlm_enabled, 'warningMessages': [{'id': 1, 'header': 'Warning!', 'message': 'A sample warning'}], }) diff --git a/seqr/views/status.py b/seqr/views/status.py index d0e6502a1e..36a1244bf6 100644 --- a/seqr/views/status.py +++ b/seqr/views/status.py @@ -3,7 +3,6 @@ import redis from settings import SEQR_VERSION, REDIS_SERVICE_HOSTNAME, REDIS_SERVICE_PORT, DATABASES -from seqr.utils.search.utils import ping_search_backend, ping_search_backend_admin from seqr.views.utils.json_utils import create_json_response logger = logging.getLogger(__name__) @@ -30,21 +29,6 @@ def status_view(request): secondary_services_ok = False logger.error('Redis connection error: {}'.format(str(e))) - # Test search backend connection - try: - ping_search_backend() - except Exception as e: - dependent_services_ok = False - logger.error('Search backend connection error: {}'.format(str(e))) - - # Test search admin view connection - try: - ping_search_backend_admin() - except Exception as e: - secondary_services_ok = False - logger.error('Search Admin connection error: {}'.format(str(e))) - - return create_json_response( {'version': SEQR_VERSION, 'dependent_services_ok': dependent_services_ok, 'secondary_services_ok': secondary_services_ok}, status= 200 if dependent_services_ok else 400 diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index f2cbc40e18..7ef6af7c3f 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -5,23 +5,13 @@ from requests import HTTPError from seqr.views.status import status_view -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses @mock.patch('seqr.views.status.redis.StrictRedis') @mock.patch('seqr.views.status.logger') -class ElasticsearchStatusTest(TestCase): +class StatusTest(TestCase): databases = '__all__' - SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' - HAS_KIBANA = True - ES_HOSTNAME = 'testhost' - - def setUp(self): - patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) - patcher.start() - self.addCleanup(patcher.stop) - def _post_teardown(self): for conn in connections.all(): conn.connection = None @@ -30,13 +20,11 @@ def _post_teardown(self): conn.connection.pool.closed = False conn.connection.is_closed = False - @urllib3_responses.activate def test_status_error(self, mock_logger, mock_redis): for conn in connections.all(): conn.ensure_connection() conn.connection.close() mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) url = reverse(status_view) response = self.client.get(url) @@ -50,49 +38,23 @@ def test_status_error(self, mock_logger, mock_redis): mock.call('Database "clickhouse" connection error: connection already closed'), mock.call('Redis connection error: Bad connection'), ] - if self.SEARCH_BACKEND_ERROR: - calls.append(mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}')) - if self.HAS_KIBANA: - calls.append(mock.call('Search Admin connection error: Kibana Error 400: Bad Request')) mock_logger.error.assert_has_calls(calls) - @urllib3_responses.activate def test_status(self, mock_logger, mock_redis): url = reverse(status_view) - mock_redis.return_value.ping.side_effect = None - urllib3_responses.add(urllib3_responses.HEAD, '/', status=200) - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=500 if self.HAS_KIBANA else 200) - + mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') response = self.client.get(url) self.assertEqual(response.status_code, 200) - if self.HAS_KIBANA: - self.assertDictEqual( - response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': False}) - mock_logger.error.assert_has_calls([ - mock.call('Search Admin connection error: Kibana Error 500: Internal Server Error'), - ]) - - mock_logger.reset_mock() - urllib3_responses.replace_json('/status', {'success': True}, method=urllib3_responses.HEAD, status=200) + self.assertDictEqual( + response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': False}) + mock_logger.error.assert_called_with('Redis connection error: Bad connection') - response = self.client.get(url) + mock_logger.reset_mock() + mock_redis.return_value.ping.side_effect = None + response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertDictEqual( response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': True}) mock_logger.error.assert_not_called() - self._assert_expected_requests() - - def _assert_expected_requests(self): - self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) - - -class ClickhouseStatusTest(ElasticsearchStatusTest): - - SEARCH_BACKEND_ERROR = None - HAS_KIBANA = False - ES_HOSTNAME = '' - - def _assert_expected_requests(self): - self.assertEqual(len(urllib3_responses.calls), 0) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 3f341af4e5..5d9d860ab0 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,22 +1,19 @@ from collections import defaultdict from datetime import datetime -from django.db.models import F, Q, Value, CharField, Aggregate +from django.db.models import F, Q, Value, Case, When, CharField, Aggregate from django.db.models.functions import Replace from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg import requests from typing import Callable, Iterable -from clickhouse_search.backend.functions import ArrayFilter -from clickhouse_search.search import get_variants_queryset, get_variant_main_transcripts_by_key from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples, backend_specific_call from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY @@ -159,6 +156,10 @@ def _get_family_metadata(family_filter, family_fields, include_family_name_displ return family_data_by_id +def _get_variant_json(saved_variants, fields, annotations): + return saved_variants.values(*fields, **annotations) + + def parse_anvil_metadata( projects: Iterable[Project], user: User, add_row: Callable[[dict, str, str], None], max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s, @@ -166,7 +167,7 @@ def parse_anvil_metadata( get_additional_individual_fields: Callable[[Individual, dict], dict] = None, individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, - include_variant_id: bool = False, include_clinvar: bool = False, variant_attr_fields: Iterable[str] = None, post_process_variant: Callable[[dict, list[dict]], dict] = None, + get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict] = _get_variant_json, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False, proband_only_variants: bool = False): @@ -189,7 +190,7 @@ def parse_anvil_metadata( sample_ids.add(sample.sample_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( - list(family_data_by_id.keys()), bool(mme_value), include_variant_id, include_clinvar, variant_attr_fields, + list(family_data_by_id.keys()), bool(mme_value), get_variant_json, ) condition_map = _get_condition_map(family_data_by_id.values()) @@ -284,7 +285,7 @@ def _get_all_project_individual_samples(projects): def _get_sorted_search_samples(projects): - return get_search_samples(projects, active_only=False).order_by('-loaded_date') + return Sample.objects.filter(individual__family__project__in=projects).order_by('-loaded_date') HET = 'Heterozygous' @@ -323,70 +324,69 @@ def _get_discovery_notes(variant, gene_variants, omit_parent_mnvs): ] def _get_parsed_saved_discovery_variants_by_family( - families: Iterable[Family], include_metadata: bool, include_variant_id: bool, include_clinvar: bool, - variant_attr_fields: list[str], + families: Iterable[Family], include_metadata: bool, get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict], ): tag_types = VariantTagType.objects.filter(project__isnull=True, category=DISCOVERY_CATEGORY) annotations = dict( genome_version=F('family__project__genome_version'), - tags=ArrayAgg('varianttag__variant_tag_type__name', distinct=True), + known_gene_tags=ArrayAgg('varianttag', distinct=True, filter=Q(varianttag__variant_tag_type__name='Known gene for phenotype')), notes=ArrayAgg('variantnote__note', distinct=True, filter=Q(variantnote__report=True)), partial_hpo_terms=ArrayAgg('variantfunctionaldata__metadata', distinct=True, filter=Q(variantfunctionaldata__functional_data_tag='Partial Phenotype Contribution')), validated_name=ArrayAgg('variantfunctionaldata__metadata', distinct=True, filter=Q(variantfunctionaldata__functional_data_tag='Validated Name')), + sv_name=Case(When(sv_type__isnull=False, key__isnull=True, then=F('variant_id')), default=Value(None)), ) project_saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__in=tag_types, family__id__in=families, - ).order_by('created_date').distinct().annotate(**annotations) + ).order_by('created_date').distinct() - variant_json_fields = ['genotypes'] - if include_clinvar: - variant_json_fields.append('clinvar') variants = [] gene_ids = set() - variant_json_by_guid = backend_specific_call( - _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, - )(project_saved_variants, include_clinvar) - for variant in project_saved_variants: - chrom, pos = get_chrom_pos(variant.xpos) - - variant_json = variant_json_by_guid[variant.guid] - main_transcript = variant_json['main_transcript'] - gene_ids.add(variant_json['gene_id']) - sv_type = variant_json.get('svType') + json_variants = get_variant_json( + project_saved_variants, + ['family_id', 'ref', 'alt', 'xpos', 'xpos_end', 'genotypes', 'gene_ids', 'main_transcript', 'sv_type'], + annotations, + ) + for variant in json_variants: + chrom, pos = get_chrom_pos(variant.pop('xpos')) + chrom_end = end = None + xpos_end = variant.pop('xpos_end') + sv_type = variant.get('sv_type') + if sv_type: + chrom_end, end = get_chrom_pos(xpos_end) + + main_transcript = variant.pop('main_transcript') + gene_id = main_transcript.get('geneId') + if not gene_id and len(variant['gene_ids'] or []) == 1: + gene_id = variant['gene_ids'][0] + gene_ids.add(gene_id) variant_type = next( - (variant_type for variant_type, has_type in VARIANT_TYPES if has_type(variant.ref, variant.alt)), + (variant_type for variant_type, has_type in VARIANT_TYPES if has_type(variant['ref'], variant['alt'])), 'INDEL') - partial_hpo_terms = variant.partial_hpo_terms[0] if variant.partial_hpo_terms else '' + partial_hpo_terms = (variant.pop('partial_hpo_terms') or [''])[0] phenotype_contribution = 'Partial' if partial_hpo_terms else 'Full' - if partial_hpo_terms == 'Uncertain': + if partial_hpo_terms and partial_hpo_terms == 'Uncertain': phenotype_contribution = 'Uncertain' partial_hpo_terms = '' parsed_variant = { 'chrom': 'MT' if chrom == 'M' else chrom, 'pos': pos, - 'variant_reference_assembly': GENOME_VERSION_LOOKUP[variant.genome_version], - 'gene_id': variant_json['gene_id'], - 'gene_ids': variant_json['gene_ids'], - 'gene_known_for_phenotype': 'Known' if 'Known gene for phenotype' in variant.tags else 'Candidate', + 'variant_reference_assembly': GENOME_VERSION_LOOKUP[variant.pop('genome_version')], + 'gene_id': gene_id, + 'gene_known_for_phenotype': 'Known' if variant.pop('known_gene_tags') else 'Candidate', 'phenotype_contribution': phenotype_contribution, 'partial_contribution_explained': partial_hpo_terms.replace(', ', '|'), - 'notes': variant.notes, - 'sv_type': sv_type, - 'sv_name': (variant_json.get('svName') or f'{sv_type}:chr{chrom}:{pos}-{variant_json["end"]}') if sv_type else None, + **variant, + 'sv_name': (variant['sv_name'] or f'{sv_type}:chr{chrom}:{pos}-{end}') if sv_type else None, 'variant_type': variant_type, - 'validated_name': variant.validated_name[0] if variant.validated_name else None, + 'validated_name': variant['validated_name'][0] if variant['validated_name'] else None, **{k: _get_transcript_field(k, config, main_transcript) for k, config in TRANSCRIPT_FIELDS.items()}, - **{k: variant_json.get(k) for k in variant_json_fields}, - **{k: variant_json.get(field) if sv_type else None for k, field in [('chrom_end', 'endChrom'), ('pos_end', 'end')]}, - 'ClinGen_allele_ID': variant_json.get('CAID'), - **{k: getattr(variant, k) for k in ['family_id', 'ref', 'alt'] + (variant_attr_fields or [])}, + 'chrom_end': chrom_end if chrom_end and chrom_end != chrom else None, + 'pos_end': end, } - if include_variant_id: - parsed_variant['variantId'] = variant.variant_id if include_metadata: parsed_variant.update({ 'seqr_chosen_consequence': main_transcript.get('majorConsequence'), @@ -405,94 +405,6 @@ def _get_parsed_saved_discovery_variants_by_family( return saved_variants_by_family -def _get_variant_json_by_guid(saved_variants, *args, **kwargs): - variant_json_by_guid = {} - for v in saved_variants: - main_transcript = _get_variant_main_transcript(v) - gene_id = main_transcript.get('geneId') - gene_ids = [gene_id] if gene_id else v.gene_ids - variant_json_by_guid[v.guid] = { - **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, - } - - return variant_json_by_guid - - -def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar): - variant_json_by_guid = {} - variant_keys_by_search_type = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - no_key_variants = [] - for v in saved_variants: - if v.key: - end_chrom, end = get_chrom_pos(v.xpos_end) - variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids} - variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id)) - else: - no_key_variants.append(v) - variant_json_by_guid.update(_get_variant_json_by_guid(no_key_variants)) - - for genome_version, keys_by_dataset_type in variant_keys_by_search_type.items(): - for dataset_type, key_map in keys_by_dataset_type.items(): - set_json_func = _set_clickhouse_sv_json if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) else _set_clickhouse_snv_indel_json - set_json_func(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar) - - return variant_json_by_guid - - -def _set_clickhouse_sv_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): - annotations = get_variants_queryset(genome_version, dataset_type, key_map.keys()).values( - 'key', svType=F('sv_type'), - ) - defaults = {'CAID': None, 'gene_id': None, 'main_transcript': {}} - if include_clinvar: - defaults['clinvar'] = None - for anns in annotations: - for guid, _ in key_map[anns['key']]: - variant_json_by_guid[guid].update({**anns, **defaults}) - - -def _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): - annotation_values = {field: Value(None, output_field=CharField()) for field in ['CAID', 'svType']} - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - annotation_values['CAID'] = F('caid') - - selected_transcripts_by_key = { - key: [transcript_id for (_, transcript_id) in variants] for key, variants in key_map.items() - } - annotations_by_key = get_variant_main_transcripts_by_key( - genome_version, dataset_type, selected_transcripts_by_key, include_clinvar=include_clinvar, additional_values=annotation_values, - ) - for key, anns in annotations_by_key.items(): - for guid, selected_main_transcript_id in key_map[key]: - main_transcript = anns['main_transcripts'][selected_main_transcript_id] - gene_id = main_transcript.get('geneId') - variant_json_by_guid[guid].update({ - **anns, - 'main_transcript': main_transcript, - 'gene_id': gene_id, - }) - if gene_id: - variant_json_by_guid[guid]['gene_ids'] = [gene_id] - - -def _get_variant_main_transcript(variant_model): - variant = variant_model.saved_variant_json - main_transcript_id = variant_model.selected_main_transcript_id or variant.get('mainTranscriptId') - if main_transcript_id: - for gene_id, transcripts in variant.get('transcripts', {}).items(): - main_transcript = next((t for t in transcripts if t['transcriptId'] == main_transcript_id), None) - if main_transcript: - if 'geneId' not in main_transcript: - main_transcript['geneId'] = gene_id - return main_transcript - elif len(variant.get('transcripts', {})) == 1: - gene_id = next(k for k in variant['transcripts'].keys()) - # Handle manually created SNPs - if variant['transcripts'][gene_id] == []: - return {'geneId': gene_id} - return {} - - def _get_transcript_field(field, config, transcript): value = transcript.get(config.get('seqr_field', field)) if config.get('format'): diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 88f62eb1a8..f0a5571cf2 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -2,13 +2,12 @@ from datetime import datetime from django.contrib.postgres.aggregates import ArrayAgg from django.db.models import F, Q -from django.utils import timezone import gzip import json import os from tqdm import tqdm -from seqr.models import Sample, Individual, Family, Project, RnaSample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier +from seqr.models import Individual, Project, RnaSample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier from seqr.utils.file_utils import file_iter, is_google_bucket_file_path, run_gsutil_with_wait from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -22,102 +21,6 @@ logger = SeqrLogger(__name__) -def load_mapping_file(mapping_file_path, user): - file_content = parse_file(mapping_file_path, file_iter(mapping_file_path, user=user)) - return load_mapping_file_content(file_content) - - -def load_mapping_file_content(file_content): - id_mapping = {} - for line in file_content: - if len(line) != 2: - raise ValueError("Must contain 2 columns: " + ', '.join(line)) - id_mapping[line[0]] = line[1] - return id_mapping - - -def _find_or_create_samples( - sample_project_tuples, - projects, - user, - sample_type, - dataset_type, - sample_id_to_individual_id_mapping, - raise_no_match_error=False, - raise_unmatched_error_template=None, - sample_data=None, -): - sample_params = {'sample_type': sample_type, 'dataset_type': dataset_type} - sample_params.update(sample_data or {}) - - samples_by_key = { - (s.pop('sample_id'), s.pop('individual__family__project__name')): s - for s in Sample.objects.filter( - individual__family__project__in=projects, - sample_id__in={sample_id for sample_id, _ in sample_project_tuples}, - **sample_params - ).values('guid', 'individual_id', 'sample_id', 'individual__family__project__name') - } - existing_samples = { - key: s for key, s in samples_by_key.items() if key in sample_project_tuples - } - remaining_sample_keys = set(sample_project_tuples) - set(existing_samples.keys()) - - matched_individual_ids = {sample['individual_id'] for sample in existing_samples.values()} - loaded_date = timezone.now() - samples_guids = [sample['guid'] for sample in existing_samples.values()] - individual_ids = {sample['individual_id'] for sample in existing_samples.values()} - if len(remaining_sample_keys) > 0: - remaining_individuals_dict = _get_individuals_by_key(projects, matched_individual_ids) - - # find Individual records with exactly-matching individual_ids - sample_id_to_individual_record = {} - for sample_key in remaining_sample_keys: - individual_key = _get_individual_key(sample_key, sample_id_to_individual_id_mapping) - if individual_key not in remaining_individuals_dict: - continue - sample_id_to_individual_record[sample_key] = remaining_individuals_dict[individual_key] - del remaining_individuals_dict[individual_key] - - logger.debug(str(len(sample_id_to_individual_record)) + " matched individual ids", user) - - remaining_sample_keys -= set(sample_id_to_individual_record.keys()) - if raise_no_match_error and len(remaining_sample_keys) == len(sample_project_tuples): - raise ValueError( - 'None of the individuals or samples in the project matched the {} expected sample id(s)'.format(len(sample_project_tuples))) - if raise_unmatched_error_template and remaining_sample_keys: - raise ValueError(raise_unmatched_error_template.format( - sample_ids=(', '.join(sorted([sample_id for sample_id, _ in remaining_sample_keys]))))) - - # create new Sample records for Individual records that matches - new_sample_args = { - sample_key: { - 'individual_id': individual['id'], - 'sample_id': sample_key[0], - } for sample_key, individual in sample_id_to_individual_record.items() - } - individual_ids.update({sample['individual_id'] for sample in new_sample_args.values()}) - new_sample_models = _create_samples( - new_sample_args.values(), - user, - loaded_date=loaded_date, - **sample_params, - ) - samples_guids += [s.guid for s in new_sample_models] - - return samples_guids, individual_ids, loaded_date - - -def _create_samples(sample_data, user, loaded_date=timezone.now(), **kwargs): - new_samples = [ - Sample( - loaded_date=loaded_date, - **created_sample_data, - **kwargs, - ) for created_sample_data in sample_data] - return Sample.bulk_create(user, new_samples) - - def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): new_samples = [ RnaSample(individual_id=individual_id, tissue_type=tissue_type, **kwargs) @@ -138,103 +41,6 @@ def _get_rna_sample_data_by_key(values=None, **kwargs): } -def _get_individuals_by_key(projects, matched_individual_ids=None): - individuals = Individual.objects.filter(family__project__in=projects) - if matched_individual_ids: - individuals = individuals.exclude(id__in=matched_individual_ids) - return { - (i['individual_id'], i.pop('family__project__name')): i - for i in individuals.values('id', 'individual_id', 'family__project__name') - } - - -def _get_individual_key(sample_key, sample_id_to_individual_id_mapping): - return ((sample_id_to_individual_id_mapping or {}).get(sample_key[0], sample_key[0]), sample_key[1]) - - -def _validate_samples_families(samples_guids, included_family_guids, sample_type, dataset_type, expected_families=None): - missing_individuals = Individual.objects.filter( - family__guid__in=included_family_guids, - sample__is_active=True, - sample__dataset_type=dataset_type, - sample__sample_type=sample_type, - ).exclude(sample__guid__in=samples_guids).select_related('family') - missing_family_individuals = defaultdict(list) - for individual in missing_individuals: - missing_family_individuals[individual.family].append(individual) - - if missing_family_individuals: - raise ValueError( - 'The following families are included in the callset but are missing some family members: {}.'.format( - ', '.join(sorted( - ['{} ({})'.format(family.family_id, ', '.join(sorted([i.individual_id for i in missing_indivs]))) - for family, missing_indivs in missing_family_individuals.items()] - )))) - - if expected_families: - missing_families = [f.family_id for f in expected_families if f.guid not in included_family_guids] - if missing_families: - raise ValueError( - 'The following families have saved variants but are missing from the callset: {}.'.format( - ', '.join(missing_families) - )) - - -def _update_variant_samples(samples_guids, individual_ids, user, dataset_type, sample_type, sample_data): - - activated_sample_guids = Sample.bulk_update(user, { - 'is_active': True, - **sample_data, - }, guid__in=samples_guids, is_active=False) - - inactivate_samples = Sample.objects.filter( - individual_id__in=individual_ids, - is_active=True, - dataset_type=dataset_type, - sample_type=sample_type, - ).exclude(guid__in=samples_guids) - - inactivate_sample_guids = Sample.bulk_update(user, {'is_active': False}, queryset=inactivate_samples) - - return activated_sample_guids, inactivate_sample_guids - - -def match_and_update_search_samples( - projects, sample_project_tuples, sample_type, dataset_type, sample_data, user, expected_families=None, - sample_id_to_individual_id_mapping=None, raise_unmatched_error_template='Matches not found for sample ids: {sample_ids}', -): - samples_guids, individual_ids, loaded_date = _find_or_create_samples( - sample_project_tuples=sample_project_tuples, - projects=projects, - user=user, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_no_match_error=not raise_unmatched_error_template, - raise_unmatched_error_template=raise_unmatched_error_template, - sample_type=sample_type, - dataset_type=dataset_type, - sample_data=sample_data, - ) - - included_families = dict(Family.objects.filter(individual__id__in=individual_ids).values_list('guid', 'analysis_status')) - _validate_samples_families(samples_guids, included_families.keys(), sample_type, dataset_type, expected_families=expected_families) - - activated_sample_guids, inactivated_sample_guids = _update_variant_samples( - samples_guids, individual_ids, user, dataset_type, sample_type, sample_data={'loaded_date': loaded_date, **sample_data}) - updated_samples = Sample.objects.filter(guid__in=activated_sample_guids) - - family_guids_to_update = [ - family_guid for family_guid, analysis_status in included_families.items() - if analysis_status in {Family.ANALYSIS_STATUS_WAITING_FOR_DATA, Family.ANALYSIS_STATUS_LOADING_FAILED} - ] - Family.bulk_update( - user, {'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, guid__in=family_guids_to_update) - - previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - new_samples = updated_samples.exclude(individual_id__in=previous_loaded_individuals) - - return new_samples, updated_samples, inactivated_sample_guids, family_guids_to_update - - def _parse_tsv_row(row): return [s.strip().strip('"') for s in row.rstrip('\n').split('\t')] @@ -465,9 +271,10 @@ def _process_rna_errors(gene_ids, missing_required_fields, unmatched_samples, ig ] misconfigured = defaultdict(set) for sample_id in sorted(unmatched_samples.intersection(set((misconfigured_samples or {}).keys()))): - misconfigured[misconfigured_samples[sample_id]].add(sample_id) + for error in misconfigured_samples[sample_id]: + misconfigured[error].add(sample_id) unmatched += [ - (samples, f'that are improperly configured in Airtable with {error}') for error, samples in misconfigured.items() + (samples, f'that are improperly configured in Airtable with {error}') for error, samples in sorted(misconfigured.items()) ] for samples, _ in unmatched: unmatched_samples -= samples diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b5e2a68629..b9e46d1a21 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -8,7 +8,6 @@ from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.search.add_data_utils import trigger_rebuild_gt_stats -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \ update_family_from_json from seqr.views.utils.orm_to_json_utils import _get_json_for_individuals, _get_json_for_families, get_json_for_family_notes @@ -88,9 +87,9 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected and not skip_gt_stats_rebuild: - backend_specific_call(lambda *args: True, trigger_rebuild_gt_stats)(project, user) + trigger_rebuild_gt_stats(project, user) if updated_sex: - backend_specific_call(lambda *args: True, SexDict.reload)() + SexDict.reload() pedigree_json = None if get_update_json: @@ -215,7 +214,7 @@ def check_project_individuals_deletable(project, individual_guids=None): if individual_guids is not None: individuals_to_delete = individuals_to_delete.filter(guid__in=individual_guids) - errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples)(individuals_to_delete) + errors = _validate_no_sumissions_no_search_samples(individuals_to_delete) return errors, individuals_to_delete @@ -227,15 +226,11 @@ def _validate_delete_individuals(individuals_to_delete, error_type, query): return errors -def _validate_no_submissions(individuals_to_delete): +def _validate_no_sumissions_no_search_samples(individuals_to_delete): return _validate_delete_individuals( individuals_to_delete, 'MME submission', dict(matchmakersubmission__isnull=False, matchmakersubmission__deleted_date__isnull=True) - ) - - -def _validate_no_sumissions_no_search_samples(individuals_to_delete): - return _validate_no_submissions(individuals_to_delete) + _validate_delete_individuals( + ) + _validate_delete_individuals( individuals_to_delete, 'search sample', dict(sample__is_active=True) ) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 09d825efd3..44a0d4c803 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -235,14 +235,12 @@ def _get_case_review_fields(model_cls, has_case_review_perm): def _get_json_for_families(families, user=None, add_individual_guids_field=False, project_guid=None, is_analyst=None, - has_case_review_perm=False, additional_values=None): + has_case_review_perm=False): family_additional_values = { **FAMILY_ADDITIONAL_VALUES, 'pedigreeImage': NullIf(Concat(Value(MEDIA_URL), 'pedigree_image', output_field=CharField()), Value(MEDIA_URL)), } - if additional_values: - family_additional_values.update(additional_values) if add_individual_guids_field: family_additional_values.update(INDIVIDUAL_GUIDS_VALUES) @@ -267,28 +265,13 @@ def get_json_for_family_note(note): INDIVIDUAL_DISPLAY_NAME_EXPR = Coalesce(NullIf('display_name', Value('')), 'individual_id', output_field=CharField()) -def _get_json_for_individuals(individuals, user=None, project_guid=None, family_guid=None, add_sample_guids_field=False, - family_fields=None, add_hpo_details=False, is_analyst=None, has_case_review_perm=False): - """Returns a JSON representation for the given list of Individuals. - - Args: - individuals (array): array of django models for the individual. - user (object): Django User object for determining whether to include restricted/internal-only fields - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - family_guid (boolean): An optional field to use as the familyGuid instead of querying the DB - add_sample_guids_field (boolean): A flag to indicate weather sample ids should be added - Returns: - array: array of json objects - """ - +def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False, + add_hpo_details=False, is_analyst=None, has_case_review_perm=False): additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm) nested_fields = [ - {'fields': ('family', 'guid'), 'value': family_guid}, + {'fields': ('family', 'guid')}, {'fields': ('family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, ] - if family_fields: - for field in family_fields: - nested_fields.append({'fields': ('family', field), 'key': _to_camel_case(field)}) if add_hpo_details: additional_model_fields += [ @@ -335,16 +318,13 @@ def add_individual_hpo_details(parsed_individuals): feature.update({'category': hpo.category_id, 'label': hpo.name}) -def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid=None, skip_nested=False, **kwargs): - if project_guid or not skip_nested: - additional_kwargs = {'nested_fields': [ - {'fields': ('individual', 'guid'), 'value': individual_guid}, - {'fields': ('individual', 'family', 'guid'), 'key': 'familyGuid', 'value': family_guid}, - {'fields': ('individual', 'family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, - ]} - else: - additional_kwargs = {'additional_model_fields': ['individual_id']} - return {'guid_key': 'sampleGuid', **additional_kwargs, **kwargs} +def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid=None, **kwargs): + nested_fields = [ + {'fields': ('individual', 'guid'), 'value': individual_guid}, + {'fields': ('individual', 'family', 'guid'), 'key': 'familyGuid', 'value': family_guid}, + {'fields': ('individual', 'family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, + ] + return {'guid_key': 'sampleGuid', 'nested_fields': nested_fields, **kwargs} def get_json_for_samples(samples, **kwargs): @@ -422,11 +402,9 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo additional_fields = [] additional_fields += additional_model_fields or [] if add_details: - from seqr.utils.search.utils import backend_specific_call - additional_fields += ['saved_variant_json'] + backend_specific_call( - lambda x: [], - lambda gv: ['key', 'genotypes', 'dataset_type'] + ([] if gv else ['family__project__genome_version']), - )(genome_version) + additional_fields += ['saved_variant_json', 'key', 'genotypes', 'dataset_type'] + ( + [] if genome_version else ['family__project__genome_version'] + ) results = get_json_for_queryset( saved_variants, guid_key='variantGuid', additional_values=sv_additional_values, @@ -434,8 +412,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo ) if add_details: - from seqr.utils.search.utils import backend_specific_call - backend_specific_call(lambda *args: None, _add_clickhouse_annotations)(results, genome_version) + _add_clickhouse_annotations(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index 1b553d3f30..f5a68c23a0 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -142,6 +142,10 @@ def test_json_for_saved_variants_with_tags(self): self.assertListEqual(json['variantTagsByGuid'][tag_guid]['variantGuids'], [variant_guid_1]) for tag_guid in v2_tag_guids: self.assertListEqual(json['variantTagsByGuid'][tag_guid]['variantGuids'], [variant_guid_2]) + self.assertDictEqual( + json['variantTagsByGuid']['VT1726985_2103343353_r0390_100']['structuredMetadata'], + {'4': {'date': '2023-11-15', 'name': 'de Novo'}, 'support': {'date': '2023-11-15', 'name': 'High in Silico Scores'}}, + ) self.assertSetEqual(set(json['variantNotesByGuid'].keys()), set(v2_note_guids)) self.assertSetEqual(set(json['variantNotesByGuid'][v2_note_guids[0]].keys()), VARIANT_NOTE_FIELDS) diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index fd122b4fb2..f29f8a2cc1 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -13,10 +13,9 @@ from seqr.utils.communication_utils import send_html_email from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples from seqr.views.utils.json_utils import _to_snake_case, _to_title_case from seqr.views.utils.permissions_utils import user_is_pm, get_pm_user_emails -from seqr.models import Individual +from seqr.models import Individual, Sample logger = SeqrLogger(__name__) @@ -48,8 +47,6 @@ def parse_pedigree_table(parsed_file, filename, user, project): if is_merged_pedigree_sample_manifest: if not user_is_pm(user): raise ValueError('Unsupported file format') - if not project: - raise ValueError('Project argument required for parsing sample manifest') header, rows = _parse_merged_pedigree_sample_manifest_rows(parsed_file[1:]) else: header = None @@ -364,7 +361,7 @@ def get_validated_related_individuals(project, records_by_id, errors, related_gu affected = records_by_id[individual_id].get(JsonConstants.AFFECTED_COLUMN, Individual.AFFECTED_STATUS_UNKNOWN) affected_status_by_family[family_id].append(affected) - search_samples = get_search_samples([project]) + search_samples = Sample.objects.filter(individual__family__project=project, is_active=True) sample_type = None if search_dataset_type: diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index 9617f8734b..5ec013e8f9 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -159,11 +159,6 @@ def _map_anvil_seqr_permission(anvil_permission): return CAN_VIEW if access_level == 'READER' else None -def anvil_has_perm(user, permission_level, project): - if not project_has_anvil(project): - return False - return has_workspace_perm(user, permission_level, project.workspace_namespace, project.workspace_name) - def has_workspace_perm(user, permission_level, namespace, name, can_share=False, meta_fields=None): kwargs = {'meta_fields': meta_fields } if meta_fields else {} @@ -211,7 +206,7 @@ def has_project_permissions(project, user, can_edit=False): def _user_project_permission(user, permission_level, project): if anvil_enabled(): - return anvil_has_perm(user, permission_level, project) + return has_workspace_perm(user, permission_level, project.workspace_namespace, project.workspace_name) return user.has_perm(permission_level, project) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 6aa26b55ae..4319549b77 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,14 +1,11 @@ from collections import defaultdict + from django.db.models import Count, Q, F, prefetch_related_objects -from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models.functions import JSONObject -from clickhouse_search.search import get_variant_main_transcripts_by_key from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ - VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample + VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ get_json_for_analysis_groups, get_json_for_samples, get_json_for_locus_lists, \ get_json_for_family_notes @@ -117,28 +114,16 @@ def add_child_ids(response): family['individualGuids'] = individual_guids_by_family[family['familyGuid']] -def families_discovery_tags(families, genome_version, project=None): +def families_discovery_tags(families, project=None): families_by_guid = {f['familyGuid']: dict(discoveryGeneIds=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} - discovery_variants = SavedVariant.objects.filter( + family_discovery_genes = list(SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ).annotate(family_guid=F('family__guid')) - - family_discovery_genes = list(discovery_variants.filter( - Q(gene_ids__len=1) | (Q(selected_main_transcript_id__isnull=True) & ~Q(dataset_type__startswith='SV')) - ).values_list('family_guid', F('gene_ids__0'))) - try: - backend_specific_call(_get_no_key_selected_transcript_gene_id, _get_clickhouse_selected_transcript_gene_id)( - family_discovery_genes, - discovery_variants.filter(selected_main_transcript_id__isnull=False, gene_ids__len__gt=1), - genome_version=genome_version, - ) - except Exception as e: - logger.error(f'Error loading discovery genes from clickhouse: {e}', None) - + ).values_list('family__guid', 'main_transcript__geneId', 'gene_ids__0')) gene_ids = set() - for family_guid, gene_id in family_discovery_genes: + for family_guid, selected_gene_id, main_gene_id in family_discovery_genes: + gene_id = selected_gene_id or main_gene_id gene_ids.add(gene_id) families_by_guid[family_guid]['discoveryGeneIds'].append(gene_id) @@ -148,40 +133,6 @@ def families_discovery_tags(families, genome_version, project=None): } -def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): - return next(( - gene_id for gene_id, gene_transcripts in (transcripts or {}).items() - if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts) - ), None) - - -def _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants, **kwargs): - family_discovery_genes += [ - (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts)) - for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values_list( - 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts', - ) - ] - - -def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): - _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants.filter(key__isnull=True)) - - tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( - keys=ArrayAgg('key', distinct=True), - variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')), - ) - - for dataset_type, keys, variants in tags_by_dataset_type.values_list('dataset_type', 'keys', 'variants'): - selected_transcripts_by_key = defaultdict(list) - for v in variants: - selected_transcripts_by_key[v['key']].append(v['selected_main_transcript_id']) - transcripts_by_key = get_variant_main_transcripts_by_key(genome_version, dataset_type, selected_transcripts_by_key) - for v in variants: - gene_id = transcripts_by_key[v['key']]['main_transcripts'][v['selected_main_transcript_id']].get('geneId') - family_discovery_genes.append((v['family_guid'], gene_id)) - - MME_TAG_NAME = 'MME Submission' diff --git a/seqr/views/utils/terra_api_utils_tests.py b/seqr/views/utils/terra_api_utils_tests.py index 2e95ebb74b..8268bd06ac 100644 --- a/seqr/views/utils/terra_api_utils_tests.py +++ b/seqr/views/utils/terra_api_utils_tests.py @@ -97,7 +97,7 @@ def _check_exceptions(self, path, func, args, kwargs=None, responses_body=None): with self.assertRaises(exception) as ec: _ = func(*args, **kwargs) self.assertEqual(str(ec.exception), message.format(path=path)) - if isinstance(exception, TerraAPIException): + if status == 401: self.assertEqual(ec.exception.status_code, status) self.assert_no_logs() diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index f3868942d2..1be9e8a56a 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -19,6 +19,36 @@ WINDOW_REGEX_TEMPLATE = 'window\.{key}=(?P[^)<]+)' +class DifferentDbTransactionSupportMixin(object): + + @classmethod + def _databases_support_transactions(cls): + return True + + @classmethod + def _rollback_atomics(cls, atomics): + """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" + for db_name in reversed(cls._databases_names()): + if connections[db_name].features.supports_transactions: + transaction.set_rollback(True, using=db_name) + atomics[db_name].__exit__(None, None, None) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + for db_name in cls._databases_names(): + if not connections[db_name].features.supports_transactions: + call_command( + "flush", + verbosity=0, + interactive=False, + database=db_name, + reset_sequences=False, + allow_cascade=False, + inhibit_post_migrate=False, + ) + + class AuthenticationTestMixin(object): SUPERUSER = 'superuser' ANALYST = 'analyst' @@ -29,7 +59,6 @@ class AuthenticationTestMixin(object): AUTHENTICATED_USER = 'authenticated' NO_POLICY_USER = 'no_policy' - ES_HOSTNAME = 'testhost' MOCK_AIRTABLE_KEY = '' CLICKHOUSE_HOSTNAME = '' @@ -44,9 +73,6 @@ class AuthenticationTestMixin(object): no_policy_user = None def set_up_test(self): - patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) - patcher.start() - self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_API_KEY', self.MOCK_AIRTABLE_KEY) patcher.start() self.addCleanup(patcher.stop) @@ -271,8 +297,8 @@ def assert_json_logs(self, user, expected, offset=0): def assert_no_logs(self): self.assertEqual(self._log_stream.getvalue(), '') -class AuthenticationTestCase(AuthenticationTestMixin, TestCase): - databases = ['default', 'reference_data'] +class AuthenticationTestCase(DifferentDbTransactionSupportMixin, AuthenticationTestMixin, TestCase): + databases = '__all__' def setUp(self): self.set_up_test() @@ -520,39 +546,8 @@ def get_group_members_side_effect(user, group, use_sa_credentials=False): return {} -class DifferentDbTransactionSupportMixin(object): - - @classmethod - def _databases_support_transactions(cls): - return True - - @classmethod - def _rollback_atomics(cls, atomics): - """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" - for db_name in reversed(cls._databases_names()): - if connections[db_name].features.supports_transactions: - transaction.set_rollback(True, using=db_name) - atomics[db_name].__exit__(None, None, None) - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - for db_name in cls._databases_names(): - if not connections[db_name].features.supports_transactions: - call_command( - "flush", - verbosity=0, - interactive=False, - database=db_name, - reset_sequences=False, - allow_cascade=False, - inhibit_post_migrate=False, - ) - - class AnvilAuthenticationTestMixin(AuthenticationTestMixin): - ES_HOSTNAME = '' CLICKHOUSE_HOSTNAME = 'testhost' MOCK_AIRTABLE_KEY = 'airflow_access' SKIP_RESET_VARIANT_JSON = False @@ -598,8 +593,6 @@ def set_up_test(self): self.mock_get_group_members.side_effect = get_group_members_side_effect self.addCleanup(patcher.stop) super().set_up_test() - if self.CLICKHOUSE_HOSTNAME and not self.SKIP_RESET_VARIANT_JSON: - SavedVariant.objects.filter(key__isnull=False).update(saved_variant_json={}) @classmethod def add_additional_user_groups(cls): @@ -771,677 +764,6 @@ def _get_list_param(call, param): GENE_DETAIL_FIELDS = {'notes', 'mgiMarkerId'} GENE_DETAIL_FIELDS.update(GENE_VARIANT_FIELDS) -VARIANTS = [ - { - 'alt': 'G', - 'ref': 'GAGA', - 'chrom': '21', - 'pos': 3343400, - 'xpos': 21003343400, - 'genomeVersion': '38', - 'liftedOverChrom': '21', - 'liftedOverPos': 3343353, - 'liftedOverGenomeVersion': '37', - 'variantId': '21-3343400-GAGA-G', - 'mainTranscriptId': 'ENST00000623083', - 'transcripts': { - 'ENSG00000227232': [ - { - 'aminoAcids': 'G/S', - 'geneSymbol': 'WASH7P', - 'biotype': 'protein_coding', - 'category': 'missense', - 'cdnaEnd': 1075, - 'cdnaStart': 1075, - 'codons': 'Ggt/Agt', - 'consequenceTerms': ['missense_variant'], - 'hgvs': 'ENSP00000485442.1:p.Gly359Ser', - 'hgvsc': 'ENST00000623083.3:c.1075G>A', - 'hgvsp': 'ENSP00000485442.1:p.Gly359Ser', - 'majorConsequence': 'missense_variant', - 'majorConsequenceRank': 11, - 'proteinId': 'ENSP00000485442', - 'transcriptId': 'ENST00000623083', - 'transcriptRank': 0 - } - ], - 'ENSG00000268903': [ - { - 'aminoAcids': 'G/S', - 'biotype': 'protein_coding', - 'category': 'missense', - 'cdnaEnd': 1338, - 'cdnaStart': 1338, - 'codons': 'Ggt/Agt', - 'consequenceTerms': ['missense_variant'], - 'geneId': 'ENSG00000268903', - 'hgvs': 'ENSP00000485351.1:p.Gly368Ser', - 'hgvsc': 'ENST00000624735.1:c.1102G>A', - 'hgvsp': 'ENSP00000485351.1:p.Gly368Ser', - 'majorConsequence': 'missense_variant', - 'majorConsequenceRank': 11, - 'proteinId': 'ENSP00000485351', - 'transcriptId': 'ENST00000624735', - 'transcriptRank': 1 - } - ] - }, - 'familyGuids': ['F000001_1', 'F000002_2'], - 'populations': { - 'callset': {'af': 0.13, 'ac': 4192, 'an': '32588'}, - 'gnomad_genomes': {'af': 0.007}, - }, - 'genotypes': { - 'NA19675': { - 'sampleId': 'NA19675', - 'ab': 0.702127659574, - 'gq': 46.0, - 'numAlt': 1, - 'dp': '50', - 'ad': '14,33', - 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'NA19679': { - 'sampleId': 'NA19679', - 'ab': 0.0, - 'gq': 99.0, - 'numAlt': 0, - 'dp': '45', - 'ad': '45,0', - 'filters': ['VQSRTrancheSNP99.95to100.00'], - } - } - }, - { - 'alt': 'A', - 'ref': 'AAAG', - 'chrom': '3', - 'pos': 835, - 'xpos': 3000000835, - 'genomeVersion': '37', - 'liftedOverGenomeVersion': '', - 'variantId': '3-835-AAAG-A', - 'transcripts': {}, - 'familyGuids': ['F000001_1'], - 'genotypes': { - 'NA19679': { - 'filters': ['artifact_prone_site'], - 'sampleId': 'NA19679', - 'ab': 0.0, - 'gq': 99.0, - 'numAlt': 0, - 'dp': '45', - 'ad': '45,0' - } - } - }, - { - 'alt': 'T', - 'ref': 'TC', - 'chrom': '12', - 'pos': 48367227, - 'xpos': 1248367227, - 'genomeVersion': '37', - 'liftedOverGenomeVersion': '', - 'variantId': '1-248367227-TC-T', - 'transcripts': {'ENSG00000233653': {}}, - 'familyGuids': ['F000002_2'], - 'genotypes': {} - } -] - -SINGLE_VARIANT = { - 'alt': 'A', - 'ref': 'G', - 'chrom': '1', - 'pos': 46394160, - 'xpos': 1046394160, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'variantId': '1-46394160-G-A', - 'transcripts': {'ENSG00000233653': {}}, - 'familyGuids': ['F000002_2'], - 'genotypes': {} -} - -TRANSCRIPT_1 = { - 'aminoAcids': 'LL/L', - 'biotype': 'protein_coding', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 10, - 'codons': 'ctTCTc/ctc', - 'geneSymbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'PROSITE_profiles:PS50850', - ], - 'canonical': 1, - 'transcriptRank': 0, - 'cdnaEnd': 421, - 'lofFilter': None, - 'hgvs': 'ENSP00000258436.5:p.Leu126del', - 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', - 'cdnaStart': 419, - 'transcriptId': 'ENST00000258436', - 'proteinId': 'ENSP00000258436', - 'category': 'missense', - 'geneId': 'ENSG00000135953', - 'hgvsp': 'ENSP00000258436.5:p.Leu126del', - 'majorConsequence': 'inframe_deletion', - 'consequenceTerms': [ - 'inframe_deletion' - ] -} -TRANSCRIPT_2 = { - 'aminoAcids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 4, - 'codons': 'Ccc/cc', - 'geneSymbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcriptRank': 0, - 'cdnaEnd': 897, - 'lofFilter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdnaStart': 897, - 'transcriptId': 'ENST00000456743', - 'proteinId': 'ENSP00000389625', - 'category': 'lof', - 'geneId': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'majorConsequence': 'frameshift_variant', - 'consequenceTerms': [ - 'frameshift_variant' - ] -} -TRANSCRIPT_3 = { - 'aminoAcids': 'LL/L', - 'biotype': 'nonsense_mediated_decay', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 10, - 'codons': 'ctTCTc/ctc', - 'geneSymbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Gene3D:1', - ], - 'canonical': None, - 'transcriptRank': 1, - 'cdnaEnd': 143, - 'lofFilter': None, - 'hgvs': 'ENSP00000413641.1:p.Leu48del', - 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', - 'cdnaStart': 141, - 'transcriptId': 'ENST00000428085', - 'proteinId': 'ENSP00000413641', - 'category': 'missense', - 'geneId': 'ENSG00000135953', - 'hgvsp': 'ENSP00000413641.1:p.Leu48del', - 'majorConsequence': 'frameshift_variant', - 'consequenceTerms': [ - 'frameshift_variant', - 'inframe_deletion', - 'NMD_transcript_variant' - ] -} - -PARSED_VARIANTS = [ - { - 'alt': 'T', - 'chrom': '1', - 'bothsidesSupport': None, - 'clinvar': {'clinicalSignificance': 'Pathogenic/Likely_pathogenic', 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': '2023-03-05'}, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'familyGuids': ['F000003_3'], - 'cpxIntervals': None, - 'algorithms': None, - 'genotypes': { - 'I000007_na20870': { - 'ab': 1, 'ad': None, 'gq': 99, 'sampleId': 'NA20870', 'numAlt': 2, 'dp': 74, 'pl': None, - 'sampleType': 'WES', - } - }, - 'genomeVersion': '37', - 'genotypeFilters': '', - 'hgmd': {'accession': None, 'class': 'DM'}, - 'mainTranscriptId': TRANSCRIPT_3['transcriptId'], - 'selectedMainTranscriptId': None, - 'originalAltAlleles': ['T'], - 'populations': { - 'callset': {'an': 32, 'ac': 2, 'hom': 3, 'af': 0.063, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': 30946, 'ac': 4, 'hom': 0, 'af': 0.00012925741614425127, 'hemi': 0, - 'filter_af': 0.0004590314436538903, 'het': 0, 'id': None, 'max_hl': None}, - 'exac': {'an': 121308, 'ac': 8, 'hom': 0, 'af': 0.00006589, 'hemi': 0, 'filter_af': 0.0006726888333653661, - 'het': 0, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': 245930, 'ac': 16, 'hom': 0, 'af': 0.00006505916317651364, 'hemi': 0, - 'filter_af': 0.0009151523074911753, 'het': 0, 'id': None, 'max_hl': None}, - 'topmed': {'an': 125568, 'ac': 21, 'hom': 0, 'af': 0.00016724, 'hemi': 0, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'hom': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, 'hom': None, - 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': 32, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 248367227, - 'predictions': {'splice_ai': 0.75, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': 'D', - 'vest': '0.335', 'mut_pred': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': '25.9', 'primate_ai': None, - 'mpc': None, 'strvctvre': None, 'splice_ai_consequence': None, 'gnomad_noncoding': 1.01272,}, - 'ref': 'TC', - 'rsid': None, - 'screenRegionType': 'dELS', - 'transcripts': { - 'ENSG00000135953': [TRANSCRIPT_3], - 'ENSG00000228198': [TRANSCRIPT_2], - }, - 'variantId': '1-248367227-TC-T', - 'xpos': 1248367227, - 'end': None, - 'svType': None, - 'svTypeDetail': None, - 'numExon': None, - 'rg37LocusEnd': None, - '_sort': [1248367227], - }, - { - 'alt': 'G', - 'chrom': '2', - 'bothsidesSupport': None, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': '2023-03-05'}, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'familyGuids': ['F000002_2', 'F000003_3'], - 'cpxIntervals': None, - 'algorithms': None, - 'genotypes': { - 'I000004_hg00731': { - 'ab': 0, 'ad': None, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'pl': None, - 'sampleType': 'WES', - }, - 'I000005_hg00732': { - 'ab': 0, 'ad': None, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'pl': None, - 'sampleType': 'WES', - }, - 'I000006_hg00733': { - 'ab': 0, 'ad': None, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'pl': None, - 'sampleType': 'WES', - }, - 'I000007_na20870': { - 'ab': 0.70212764, 'ad': None, 'gq': 46, 'sampleId': 'NA20870', 'numAlt': 1, 'dp': 50, 'pl': None, - 'sampleType': 'WES', - } - }, - 'genotypeFilters': '', - 'genomeVersion': '37', - 'hgmd': {'accession': None, 'class': None}, - 'mainTranscriptId': TRANSCRIPT_1['transcriptId'], - 'selectedMainTranscriptId': TRANSCRIPT_2['transcriptId'], - 'originalAltAlleles': ['G'], - 'populations': { - 'callset': {'an': 32, 'ac': 1, 'hom': None, 'af': 0.031, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': 0, 'ac': 0, 'hom': 0, 'af': 0.0, 'hemi': 0, 'filter_af': None, 'het': 0, - 'id': None, 'max_hl': None}, - 'exac': {'an': 121336, 'ac': 6, 'hom': 0, 'af': 0.00004942, 'hemi': 0, 'filter_af': 0.000242306760358614, - 'het': 0, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': 245714, 'ac': 6, 'hom': 0, 'af': 0.000024418633044922146, 'hemi': 0, - 'filter_af': 0.00016269686320447742, 'het': 0, 'id': None, 'max_hl': None}, - 'topmed': {'an': 0, 'ac': 0, 'hom': 0, 'af': 0.0, 'hemi': 0, 'filter_af': None, 'het': None, 'id': None, - 'max_hl': None}, - 'sv_callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'hom': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': 32, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 103343353, - 'predictions': { - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, 'gnomad_noncoding': None, - 'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, 'polyphen': None, - 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': 1, 'vest': None, 'mut_pred': None, - 'mpc': None, 'strvctvre': None, 'splice_ai_consequence': None, - }, - 'ref': 'GAGA', - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000135953': [TRANSCRIPT_1], - 'ENSG00000228198': [TRANSCRIPT_2], - }, - 'variantId': '2-103343353-GAGA-G', - 'xpos': 2103343353, - 'end': None, - 'svType': None, - 'svTypeDetail': None, - 'numExon': None, - 'rg37LocusEnd': None, - '_sort': [2103343353], - }, -] - -PARSED_SV_VARIANT = { - 'alt': None, - 'chrom': '1', - 'bothsidesSupport': True, - 'familyGuids': ['F000002_2'], - 'cpxIntervals': None, - 'algorithms': None, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'numAlt': -1, 'geneIds': ['ENSG00000228198'], - 'cn': 1, 'end': None, 'start': None, 'numExon': None, 'defragged': False, 'qs': 33, 'gq': None, - 'prevCall': False, 'prevOverlap': False, 'newCall': True, 'prevNumAlt': None, - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'numAlt': -1, 'sampleType': None, 'geneIds': None, 'gq': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'isRef': True, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'numAlt': -1, 'geneIds': None, 'gq': None, - 'cn': 2, 'end': 49045890, 'start': 49045987, 'numExon': 1, 'defragged': False, 'qs': 80, - 'prevCall': False, 'prevOverlap': True, 'newCall': False, 'prevNumAlt': None, - }, - }, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': None}, - 'hgmd': {'accession': None, 'class': None}, - 'genomeVersion': '37', - 'genotypeFilters': '', - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'originalAltAlleles': [], - 'populations': { - 'callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'exac': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'topmed': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': 10088, 'ac': 7, 'hom': None, 'af': 0.000693825, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': None, 'hemi': 0, 'hom': 0, 'het': 0, 'id': None, - 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 49045487, - 'predictions': {'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, 'gnomad_noncoding': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': None, - 'vest': None, 'mut_pred': None, 'mpc': None, 'strvctvre': 0.374, 'splice_ai_consequence': None}, - 'ref': None, - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000228198': [ - { - 'geneId': 'ENSG00000228198' - }, - ], - 'ENSG00000135953': [ - { - 'geneId': 'ENSG00000135953' - }, - ], - }, - 'variantId': 'prefix_19107_DEL', - 'xpos': 1049045487, - 'end': 49045899, - 'svType': 'INS', - 'svTypeDetail': None, - 'svSourceDetail': {'chrom': '9'}, - 'numExon': 2, - 'rg37LocusEnd': None, - '_sort': [1049045387], -} - -PARSED_SV_WGS_VARIANT = { - 'alt': None, - 'chrom': '2', - 'bothsidesSupport': None, - 'familyGuids': ['F000014_14'], - 'cpxIntervals': [{'chrom': '2', 'end': 3000, 'start': 1000, 'type': 'DUP'}, - {'chrom': '20', 'end': 13000, 'start': 11000, 'type': 'INV'}], - 'algorithms': 'wham, manta', - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'genotypes': { - 'I000018_na21234': { - 'gq': 33, 'sampleId': 'NA21234', 'numAlt': 1, 'geneIds': None, - 'cn': -1, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': 'WGS', - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': 2, - }, - 'I000019_na21987': { - 'gq': None, 'sampleId': 'NA21987', 'numAlt': -1, 'isRef': True, 'geneIds': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': None, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - 'I000021_na21654': { - 'gq': None, 'sampleId': 'NA21654', 'numAlt': -1, 'isRef': True, 'geneIds': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': None, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - }, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': None}, - 'hgmd': {'accession': None, 'class': None}, - 'genomeVersion': '38', - 'genotypeFilters': '', - 'liftedOverChrom': '2', - 'liftedOverGenomeVersion': '37', - 'liftedOverPos': 49272526, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'originalAltAlleles': [], - 'populations': { - 'callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'exac': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'topmed': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': 10088, 'ac': 7, 'hom': None, 'af': 0.000693825, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': 22, 'af': 0.00679, 'an': 3240, 'filter_af': None, 'hemi': 0, 'hom': 0, 'het': 0, - 'id': 'gnomAD-SV_v2.1_BND_1_1', 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 49045387, - 'predictions': {'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, - 'vest': None, 'mut_pred': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': None, - 'mpc': None, 'strvctvre': None, 'gnomad_noncoding': None, 'splice_ai_consequence': None}, - 'ref': None, - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000228198': [ - { - 'geneSymbol': 'OR4F5', - 'majorConsequence': 'DUP_PARTIAL', - 'geneId': 'ENSG00000228198' - }, - ], - 'ENSG00000228199': [ - { - 'geneId': 'ENSG00000228199', - 'geneSymbol': 'FBXO28', - 'majorConsequence': 'MSV_EXON_OVERLAP' - } - ], - 'ENSG00000228201': [ - { - 'geneId': 'ENSG00000228201', - 'geneSymbol': 'FAM131C', - 'majorConsequence': 'INTRAGENIC_EXON_DUP' - } - ] - }, - 'variantId': 'prefix_19107_CPX', - 'xpos': 2049045387, - 'end': 12345678, - 'endChrom': '20', - 'svType': 'CPX', - 'svTypeDetail': 'dupINV', - 'numExon': None, - 'rg37LocusEnd': {'contig': '20', 'position': 12326326}, - '_sort': [2049045387], -} - -PARSED_MITO_VARIANT = { - '_sort': [25000010195], - 'algorithms': None, - 'alt': 'A', - 'bothsidesSupport': None, - 'chrom': 'M', - 'clinvar': {'alleleId': None, 'clinicalSignificance': 'Likely_pathogenic', 'goldStars': None, 'variationId': None, 'version': None}, - 'commonLowHeteroplasmy': False, - 'cpxIntervals': None, - 'end': 10195, - 'familyGuids': ['F000002_2'], - 'genomeVersion': '37', - 'genotypeFilters': '', - 'genotypes': - {'I000004_hg00731': - {'contamination': 0.0, 'dp': 5139.0, 'gq': 60.0, 'hl': 1.0, 'mitoCn': 319.03225806451616, 'numAlt': 2, - 'sampleId': 'HG00731', 'sampleType': 'WES'}}, - 'hgmd': {'accession': None, 'class': None}, - 'highConstraintRegion': True, - 'mainTranscriptId': 'ENST00000361227', - 'mitomapPathogenic': True, - 'numExon': None, - 'originalAltAlleles': [], - 'populations': - {'callset': {'ac': 0, 'af': 0.0, 'an': 2520, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'exac': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': 1368, 'af': 0.024246292, 'an': 56421, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None,'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': 1312, 'af': 0.0033268193, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'sv_callset': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'topmed': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': 1, 'af': 0.0003968253968253968, 'an': 2520, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': 3, 'af': 5.317169e-05, 'an': 56421, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': 1.0}, - 'helix_heteroplasmy': {'ac': 5, 'af': 4.081987e-05, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': 0.90441}, - }, - 'pos': 10195, - 'predictions': {'hmtvar': 0.71, 'apogee': 0.42, 'cadd': None, 'dann': None, 'eigen': None, 'fathmm': 'T', - 'haplogroup_defining': None, 'mitotip': None, 'mpc': None, 'mut_taster': 'N', 'polyphen': None, - 'primate_ai': None, 'revel': None, 'sift': 'D', 'splice_ai': None, 'splice_ai_consequence': None, - 'vest': None, 'mut_pred': None, 'strvctvre': None, 'gnomad_noncoding': None,}, - 'ref': 'C', - 'rg37LocusEnd': None, - 'rsid': None, - 'screenRegionType': None, - 'selectedMainTranscriptId': None, - 'svType': None, - 'svTypeDetail': None, - 'transcripts': {'ENSG00000198840': [ - {'aminoAcids': 'P/H', 'biotype': 'protein_coding', 'canonical': 1, 'category': 'missense', 'cdnaEnd': 137, - 'cdnaStart': 137, 'codons': 'cCc/cAc', 'consequenceTerms': [ - 'missense_variant'], - 'domains': ['Gene3D:1', 'ENSP_mappings:5xtc', 'ENSP_mappings:5xtd', 'Pfam:PF00507', 'PANTHER:PTHR11058', - 'PANTHER:PTHR11058'], 'geneId': 'ENSG00000198840', 'geneSymbol': 'MT-ND3', 'hgvs': 'p.Pro46His', - 'hgvsc': 'ENST00000361227.2:c.137C>A', 'hgvsp': 'ENSP00000355206.2:p.Pro46His', - 'majorConsequence': 'missense_variant', 'majorConsequenceRank': 11, - 'polyphenPrediction': 'probably_damaging', 'proteinId': 'ENSP00000355206', 'proteinStart': 46, - 'siftPrediction': 'deleterious_low_confidence', 'transcriptId': 'ENST00000361227', 'transcriptRank': 0}]}, - 'variantId': 'M-10195-C-A', - 'xpos': 25000010195 -} - -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT = deepcopy(PARSED_VARIANTS) -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT[1].update({ - 'familyGuids': ['F000003_3'], - 'mainTranscriptId': TRANSCRIPT_2['transcriptId'], - 'selectedMainTranscriptId': None, -}) -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT[1]['transcripts']['ENSG00000135953'][0]['majorConsequence'] = 'frameshift_variant' -for variant in PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT: - variant['_sort'][0] += 100 - variant['familyGuids'].append('F000011_11') - variant['genotypes'].update({ - 'I000015_na20885': { - 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, - 'sampleType': 'WES', - }, - }) - GOOGLE_API_TOKEN_URL = 'https://oauth2.googleapis.com/token' # nosec GOOGLE_ACCESS_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' # nosec diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a9d7dd1955..7a3aed8dc1 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -1,25 +1,20 @@ -from collections import defaultdict +from collections import defaultdict, abc from clickhouse_backend.models import ArrayField, StringField from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import F, Q, Count, prefetch_related_objects +from django.db.models import F, Q, Count, Value import json import logging import redis -from tqdm import tqdm -import traceback from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap -from clickhouse_search.search import get_variants_queryset, get_clickhouse_genotypes +from clickhouse_search.search import get_variants_queryset from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.utils import backend_specific_call, variant_dataset_type +from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants -from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ @@ -35,91 +30,46 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, update_function, **kwargs): - success = {} - skipped = {} - error = {} - logger.info(f'Reloading saved variants in {len(projects)} projects') - for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): - try: - updated_saved_variants = update_function( - project_id, genome_version, family_guids=family_guids, project_guid=project_guid, **kwargs) - if updated_saved_variants is None: - skipped[project_name] = True - else: - success[project_name] = len(updated_saved_variants) - family_summary = f' in {len(family_guids)} families' if family_guids else '' - logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - except Exception as e: - traceback_message = traceback.format_exc() - logger.error(traceback_message) - logger.error(f'Error reloading variants in {project_name}: {e}') - error[project_name] = e - - logger.info('Reload Summary: ') - for k, v in success.items(): - if v > 0: - logger.info(f' {k}: Updated {v} variants') - if skipped: - logger.info(f'Skipped the following {len(skipped)} project with no saved variants: {", ".join(skipped)}') - if len(error): - logger.info(f'{len(error)} failed projects') - for k, v in error.items(): - logger.info(f' {k}: {v}') - - -def get_saved_variants(genome_version, project_id=None, family_guids=None, clickhouse_dataset_type=None): - saved_variants = SavedVariant.objects.filter( - Q(saved_variant_json__genomeVersion__isnull=True) | - Q(saved_variant_json__genomeVersion=genome_version.replace('GRCh', '')) - ) - if project_id: - saved_variants = saved_variants.filter(family__project_id=project_id) - if family_guids: - saved_variants = saved_variants.filter(family__guid__in=family_guids) - if clickhouse_dataset_type: - saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) - return saved_variants - - -def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_type=None): - if 'xpos' not in variant_json: - variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos']) +def parse_saved_variant_json(variant_json, family_id): xpos = variant_json['xpos'] ref = variant_json.get('ref') - alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - variant_id = variant_json.get('variantId', variant_id) - if variant_json.get('key'): - update_json = { - 'key': variant_json['key'], - 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': dataset_type or variant_dataset_type({'variantId': variant_id, **variant_json}), - } - else: - update_json = {'saved_variant_json': variant_json} if 'transcripts' in variant_json: - update_json['gene_ids'] = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) - elif 'gene_ids' in variant_json: - update_json['gene_ids'] = variant_json['gene_ids'] + main_transcript_id = variant_json.get('mainTranscriptId') + gene_ids = sorted( + variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json, main_transcript_id) + ) + main_gene_transcripts = variant_json['transcripts'][gene_ids[0]] if gene_ids else [] + main_transcript = next( + t for t in main_gene_transcripts if t.get('transcriptId') == main_transcript_id + ) if main_transcript_id else {} + else: + gene_ids = variant_json['gene_ids'] + main_transcript = variant_json['main_transcript'] return { + 'key': variant_json['key'], 'xpos': xpos, 'xpos_end': xpos + var_length, 'ref': ref, - 'alt': alt, + 'alt': variant_json.get('alt'), + 'sv_type': variant_json.get('svType'), 'family_id': family_id, - 'variant_id': variant_id, - }, update_json + 'variant_id': variant_json['variantId'], + 'genotypes': variant_json.get('genotypes', {}), + 'dataset_type': variant_json.get('dataset_type') or variant_dataset_type(variant_json), + 'gene_ids': gene_ids, + 'main_transcript': main_transcript, + 'saved_variant_json': variant_json.get('saved_variant_json', {}), + } -def _transcript_sort(gene_id, saved_variant_json): +def _transcript_sort(gene_id, saved_variant_json, main_transcript_id): gene_transcripts = saved_variant_json['transcripts'][gene_id] - main_transcript_id = saved_variant_json.get('mainTranscriptId') is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) -def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, get_comp_het_metadata=None, load_new_variant_data=False, remove_missing_metadata=True, primary_id_field='variant_id', dataset_type=None, **kwargs): +def bulk_create_tagged_variants(family_variant_data, tag_name, parse_new_saved_variants, get_metadata, user, project=None, get_comp_het_metadata=None, remove_missing_metadata=True, primary_id_field='variant_id'): all_family_ids = {family_id for family_id, _ in family_variant_data.keys()} all_variant_ids = {variant_id for _, variant_id in family_variant_data.keys()} @@ -130,20 +80,11 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: - if load_new_variant_data: - genome_version = Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() - new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) - else: - new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} - new_variant_data = backend_specific_call( - lambda o, *args, **kwargs: o, _get_clickhouse_variant_annotations, - )(new_variant_data, primary_id_field, project=project, dataset_type=dataset_type, **kwargs) - - new_variant_models = [] - for (family_id, variant_id), variant in new_variant_data.items(): - create_json, update_json = parse_saved_variant_json(variant, family_id, variant_id=variant_id, dataset_type=dataset_type) - new_variant_models.append(SavedVariant(**create_json, **update_json)) - + new_variant_data = parse_new_saved_variants(new_variant_keys, family_variant_data) + new_variant_models = [ + SavedVariant(**parse_saved_variant_json(variant, family_id)) + for (family_id, _), variant in new_variant_data.items() + ] saved_variant_map.update({ (v.family_id, getattr(v, primary_id_field)): v for v in SavedVariant.bulk_create(user, new_variant_models) }) @@ -229,81 +170,16 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ return updated_tag -def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: User, genome_version: str) -> dict[tuple[int, str], dict]: - family_ids = set() - variant_families = defaultdict(list) - for family_id, variant_id in family_variant_ids: - family_ids.add(family_id) - variant_families[variant_id].append(family_id) - families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} - - samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - search_variants_by_id = { - v['variantId']: v for v in backend_specific_call( - _get_es_variants, _get_clickhouse_variants, - )(samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), family_variant_ids=family_variant_ids, user=user, genome_version=genome_version) - } - - new_variants = {} - missing = defaultdict(list) - for variant_id, family_ids in variant_families.items(): - variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} - for family_id in family_ids: - family = families_by_id[family_id] - if family.guid in variant['familyGuids']: - new_variants[(family_id, variant_id)] = variant - else: - missing[family.family_id].append(variant_id) - - if missing: - missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] - raise ErrorsWarningsException([ - f"Unable to find the following family's variants in the search backend: {', '.join(missing_summary)}", - ]) - - return new_variants - - -def _get_es_variants(samples: Sample.objects, families_by_id: dict[int, Family], *args, **kwargs): - return get_es_variants_for_variant_ids(samples.filter(individual__family_id__in=families_by_id.keys()), *args, **kwargs) - - -def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str = None, **kwargs) -> list[dict]: - variant_data = _get_clickhouse_variant_annotations( - {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, - ) - variants_by_key = {variant['key']: variant for variant in variant_data.values() if variant.get('key')} - - families = list(families_by_id.values()) - prefetch_related_objects(families, 'project') - families_by_project = defaultdict(list) - for family in families: - families_by_project[family.project.guid].append(family.guid) - for project_guid, family_guids in families_by_project.items(): - genotype_keys = get_clickhouse_genotypes( - project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), - samples, - ) - for key, genotypes in genotype_keys.items(): - variants_by_key[key]['genotypes'].update(genotypes) - variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in genotypes.values()}) - - return list(variants_by_key.values()) - - -def _get_clickhouse_variant_annotations(variant_data: dict[tuple[int, str], dict], primary_id_field: str = 'variant_id', genome_version: str = None, project: Project = None, dataset_type: str = None) -> dict[tuple[int, str], dict]: +def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], genome_version: str, primary_id_field: str = 'variant_id', group_by_field=None, dataset_type: str = None) -> dict[str, dict]: dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS - variant_ids = { - variant_id for (_, variant_id), variant in variant_data.items() - if not (variant.get('key') and variant.get('variantId')) - } + variant_ids = {variant_id for _, variant_id in variant_keys} keys = None if primary_id_field == 'key': keys = variant_ids variant_ids = None qs = get_variants_queryset( - genome_version or project.genome_version, dataset_type, keys=keys, variant_ids=variant_ids, - ) + genome_version, dataset_type, keys=keys, variant_ids=variant_ids, + ).join_variant_id() key_field = 'variantId' if primary_id_field == 'variant_id' else primary_id_field variant_fields = ['key'] variant_values = { @@ -314,14 +190,18 @@ def _get_clickhouse_variant_annotations(variant_data: dict[tuple[int, str], dict ), } if dataset_type.startswith('SV'): - variant_fields += ['chrom', 'pos', 'end'] + variant_fields += ['chrom', 'pos', 'end', 'sv_type'] + variant_values.update({ + 'svType': F('sv_type'), + 'main_transcript': Value(None, output_field=StringField()), + }) else: variant_values.update(qs.split_variant_id_annotations()) - variants_by_id = {v[key_field]: v for v in qs.join_variant_id().values(*variant_fields, **variant_values)} - for (_, variant_id), variant in variant_data.items(): - if variant_id in variants_by_id: - variant.update(variants_by_id[variant_id]) - return variant_data + if qs.variant_detail_field: + variant_values['main_transcript'] = F(f'{qs.variant_detail_field}__transcripts__0') + else: + variant_values['main_transcript'] = F('sorted_transcript_consequences__0') + return {v[group_by_field or key_field]: v for v in qs.values(*variant_fields, **variant_values)} def reset_cached_search_results(project, reset_index_metadata=False): @@ -368,7 +248,7 @@ def _get_variants_reference_data_response(variants, genome_versions, get_family_ for var in variant: for gene_id, transcripts in var.get('transcripts', {}).items(): gene_ids.add(gene_id) - if backend_specific_call(lambda v: True, _requires_transcript_metadata)(variant): + if _requires_transcript_metadata(variant): transcript_ids.update([t['transcriptId'] for t in transcripts if t.get('transcriptId')]) if get_family_genes: for family_guid in var.get('familyGuids', []): @@ -393,7 +273,7 @@ def _get_variants_reference_data_response(variants, genome_versions, get_family_ if any(genome_version == OMIM_GENOME_VERSION for genome_version in genome_versions): response['omimIntervals'] = _get_omim_intervals(variants) - backend_specific_call(lambda *args: None, _add_sample_count_stats)(response, genome_versions) + _add_sample_count_stats(response, genome_versions) return response diff --git a/settings.py b/settings.py index f0b0166431..94fc51a8b3 100644 --- a/settings.py +++ b/settings.py @@ -358,29 +358,6 @@ IS_ANVIL_LOADING_DELAY = bool(os.environ.get('IS_ANVIL_LOADING_DELAY')) # External service settings -ELASTICSEARCH_SERVICE_HOSTNAME = os.environ.get('ELASTICSEARCH_SERVICE_HOSTNAME') -ELASTICSEARCH_SERVICE_PORT = os.environ.get('ELASTICSEARCH_SERVICE_PORT', '9200') -ELASTICSEARCH_SERVER = '{host}:{port}'.format( - host=ELASTICSEARCH_SERVICE_HOSTNAME, port=ELASTICSEARCH_SERVICE_PORT) - -SEQR_ELASTICSEARCH_PASSWORD = os.environ.get('SEQR_ES_PASSWORD') -ELASTICSEARCH_USER = os.environ.get('ELASTICSEARCH_USER', 'seqr') -ELASTICSEARCH_CREDENTIALS = (ELASTICSEARCH_USER, SEQR_ELASTICSEARCH_PASSWORD) if SEQR_ELASTICSEARCH_PASSWORD else None -ELASTICSEARCH_PROTOCOL = os.environ.get('ELASTICSEARCH_PROTOCOL', 'http') -ELASTICSEARCH_CA_PATH = os.environ.get('ELASTICSEARCH_CA_PATH') -# if we have a custom CA certificate for elasticsearch, add it to the verification path for connections -if ELASTICSEARCH_CA_PATH: - ES_SSL_CONTEXT = create_default_context(cafile=ELASTICSEARCH_CA_PATH) -else: - ES_SSL_CONTEXT = None - -KIBANA_SERVER = '{host}:{port}'.format( - host=os.environ.get('KIBANA_SERVICE_HOSTNAME', 'localhost'), - port=os.environ.get('KIBANA_SERVICE_PORT', 5601) -) -KIBANA_ELASTICSEARCH_USER = os.environ.get('KIBANA_ELASTICSEARCH_USER', 'kibana') -KIBANA_ELASTICSEARCH_PASSWORD = os.environ.get('KIBANA_ES_PASSWORD') - REDIS_SERVICE_HOSTNAME = os.environ.get('REDIS_SERVICE_HOSTNAME', 'localhost') REDIS_SERVICE_PORT = int(os.environ.get('REDIS_SERVICE_PORT', '6379')) diff --git a/ui/app.jsx b/ui/app.jsx index 619f7e0a63..533fd4c5b7 100644 --- a/ui/app.jsx +++ b/ui/app.jsx @@ -10,6 +10,7 @@ import Dashboard from 'pages/Dashboard/Dashboard' import Project from 'pages/Project/Project' import LoadWorkspaceData, { WorkspaceAccessError } from 'pages/AnvilWorkspace/LoadWorkspaceData' import VariantSearch from 'pages/Search/VariantSearch' +import VariantLookup from 'pages/Search/VariantLookup' import DataManagement from 'pages/DataManagement/DataManagement' import Report from 'pages/Report/Report' import SummaryData from 'pages/SummaryData/SummaryData' @@ -50,6 +51,7 @@ ReactDOM.render( + diff --git a/ui/package-lock.json b/ui/package-lock.json index 6f072b7d8d..022be7b520 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -13764,9 +13764,9 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "engines": { "node": ">=8.6" @@ -30044,9 +30044,9 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true }, "pify": { diff --git a/ui/pages/Project/components/CreateVariantButton.jsx b/ui/pages/Project/components/CreateVariantButton.jsx index 9f031fa592..87836734b0 100644 --- a/ui/pages/Project/components/CreateVariantButton.jsx +++ b/ui/pages/Project/components/CreateVariantButton.jsx @@ -3,16 +3,16 @@ import PropTypes from 'prop-types' import { connect } from 'react-redux' import { Grid, Divider, Accordion } from 'semantic-ui-react' -import { updateVariantTags } from 'redux/rootReducer' import { getUser, getSortedIndividualsByFamily } from 'redux/selectors' import UpdateButton from 'shared/components/buttons/UpdateButton' import { Select, IntegerInput, LargeMultiselect } from 'shared/components/form/Inputs' import { validators, configuredField } from 'shared/components/form/FormHelpers' import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' -import { GENOME_VERSION_FIELD, SV_TYPES } from 'shared/utils/constants' +import { SV_TYPES } from 'shared/utils/constants' -import { TAG_FORM_FIELD, TAG_FIELD_NAME } from '../constants' +import { TAG_FORM_FIELD } from '../constants' +import { createManualVariant } from '../reducers' import { getTaggedVariantsByFamilyType, getProjectTagTypeOptions, getCurrentProject } from '../selectors' import SelectSavedVariantsTable, { VARIANT_POS_COLUMN, TAG_COLUMN, GENES_COLUMN } from './SelectSavedVariantsTable' @@ -25,9 +25,6 @@ const TRANSCRIPT_ID_FIELD_NAME = 'mainTranscriptId' const HGVSC_FIELD_NAME = 'hgvsc' const HGVSP_FIELD_NAME = 'hgvsp' const VARIANTS_FIELD_NAME = 'variants' -const FORMAT_RESPONSE_FIELDS = [ - GENE_ID_FIELD_NAME, HGVSC_FIELD_NAME, HGVSP_FIELD_NAME, TAG_FIELD_NAME, VARIANTS_FIELD_NAME, -] const ZygosityInput = React.memo(({ individuals, name, title, individualField, error }) => (
@@ -88,8 +85,6 @@ const SAVED_VARIANT_COLUMNS = [ ] const SAVED_SV_COLUMNS = [{ name: 'svType', content: 'SV Type', width: 2 }, ...SAVED_VARIANT_COLUMNS] -const GENOME_FIELD = { ...GENOME_VERSION_FIELD, inline: false, width: null } - const ZYGOSITY_FIELD = { name: 'genotypes', component: connect(mapZygosityInputStateToProps)(ZygosityInput), @@ -127,7 +122,6 @@ const GENE_FIELD = { const SAVED_VARIANT_FIELD = { name: VARIANTS_FIELD_NAME, idField: 'variantGuid', - includeSelectedRowData: true, control: SavedVariantField, } @@ -156,7 +150,6 @@ const SNV_FIELDS = [ { name: TRANSCRIPT_ID_FIELD_NAME, label: 'Transcript ID', width: 6 }, { name: HGVSC_FIELD_NAME, label: 'HGVSC', width: 5, validate: validateHasTranscriptId }, { name: HGVSP_FIELD_NAME, label: 'HGVSP', width: 5, validate: validateHasTranscriptId }, - GENOME_FIELD, TAG_FIELD, { accordionLabel: 'Multinucleotide Variant', columns: SAVED_VARIANT_COLUMNS, ...SAVED_VARIANT_FIELD }, { @@ -176,7 +169,6 @@ const SV_FIELDS = [ START_FIELD, END_FIELD, GENE_FIELD, - GENOME_FIELD, TAG_FIELD, { name: SV_FIELD_NAME, validate: validators.required, label: 'SV Name', width: 8 }, { @@ -206,7 +198,6 @@ const BaseCreateVariantButton = React.memo(({ variantType, family, user, project (project.isAnalystProject ? user.isAnalyst : project.canEdit) ? ( ({ }) const mapDispatchToProps = (dispatch, ownProps) => ({ - onSubmit: (values) => { - const variant = ownProps.formFields.map(({ name }) => name).filter( - name => !FORMAT_RESPONSE_FIELDS.includes(name), - ).reduce( - (acc, name) => ({ ...acc, [name]: values[name] }), {}, - ) - - if (variant.svName) { - variant.variantId = values.svName - if (values[GENE_ID_FIELD_NAME]) { - variant.transcripts = { [values[GENE_ID_FIELD_NAME]]: [] } - } - } else { - variant.variantId = `${values.chrom}-${values.pos}-${values.ref}-${values.alt}` - variant.transcripts = { - [values[GENE_ID_FIELD_NAME]]: values[TRANSCRIPT_ID_FIELD_NAME] ? [{ - transcriptId: values[TRANSCRIPT_ID_FIELD_NAME], - [HGVSC_FIELD_NAME]: values[HGVSC_FIELD_NAME], - [HGVSP_FIELD_NAME]: values[HGVSP_FIELD_NAME], - }] : [], - } - } - - const variants = Object.values(values[VARIANTS_FIELD_NAME] || {}).filter(v => v) - - const formattedValues = { - familyGuid: ownProps.family.familyGuid, - tags: values[TAG_FIELD_NAME], - variant: [variant, ...variants], - } - - return dispatch(updateVariantTags(formattedValues)) - }, + onSubmit: values => dispatch(createManualVariant(ownProps.family.familyGuid, values)), }) const CreateVariantButton = connect(mapStateToProps, mapDispatchToProps)(BaseCreateVariantButton) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 3f796b1df5..2d4b134359 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -5,7 +5,6 @@ import { connect } from 'react-redux' import { Link } from 'react-router-dom' import { Field } from 'react-final-form' import { Label, Popup, Form, Input, Loader } from 'semantic-ui-react' -import orderBy from 'lodash/orderBy' import { SearchInput, YearSelector, RadioButtonGroup, ButtonRadioGroup, Select } from 'shared/components/form/Inputs' import { helpLabel, validators } from 'shared/components/form/FormHelpers' @@ -32,7 +31,7 @@ import { import { snakecaseToTitlecase } from 'shared/utils/stringUtils' import { updateIndividual } from 'redux/rootReducer' -import { getSamplesByGuid, getMmeSubmissionsByGuid, getIGVSamplesByFamilySampleIndividual } from 'redux/selectors' +import { getDatasetsByIndividual, getMmeSubmissionsByGuid, getIGVSamplesByFamilySampleIndividual } from 'redux/selectors' import { HPO_FORM_FIELDS } from '../HpoTerms' import { CASE_REVIEW_STATUS_MORE_INFO_NEEDED, CASE_REVIEW_STATUS_OPTIONS, CASE_REVIEW_TABLE_NAME, INDIVIDUAL_DETAIL_FIELDS, @@ -149,10 +148,10 @@ MmeStatusLabel.propTypes = { mmeSubmission: PropTypes.object, } -const DataDetails = React.memo(({ loadedSamples, individual, mmeSubmission }) => ( +const DataDetails = React.memo(({ loadedDatasets, individual, mmeSubmission }) => (
- {loadedSamples.map( - sample =>
, + {loadedDatasets.map( + dataset =>
, )} {individual.rnaSample && ( DataDetails.propTypes = { mmeSubmission: PropTypes.object, individual: PropTypes.object, - loadedSamples: PropTypes.arrayOf(PropTypes.object), + loadedDatasets: PropTypes.arrayOf(PropTypes.object), } const formatGene = gene => `${gene.gene} ${gene.comments ? ` (${gene.comments.trim()})` : ''}` @@ -548,7 +547,7 @@ class IndividualRow extends React.PureComponent { project: PropTypes.object.isRequired, individual: PropTypes.object.isRequired, mmeSubmission: PropTypes.object, - samplesByGuid: PropTypes.object.isRequired, + datasets: PropTypes.arrayOf(PropTypes.object), alignmentSample: PropTypes.object, dispatchUpdateIndividual: PropTypes.func, dispatchUpdateIndividualIGV: PropTypes.func, @@ -575,17 +574,15 @@ class IndividualRow extends React.PureComponent { render() { const { - project, individual, mmeSubmission, samplesByGuid, tableName, updateIndividualPedigree, alignmentSample, + project, individual, mmeSubmission, datasets, tableName, updateIndividualPedigree, alignmentSample, dispatchUpdateIndividualIGV, } = this.props - const { displayName, sex, affected, createdDate, sampleGuids } = individual + const { displayName, sex, affected, createdDate } = individual - let loadedSamples = sampleGuids.map( - sampleGuid => samplesByGuid[sampleGuid], - ) - loadedSamples = orderBy(loadedSamples, [s => s.loadedDate], 'desc') // only show active or first/ last inactive samples - loadedSamples = loadedSamples.filter((sample, i) => sample.isActive || i === 0 || i === loadedSamples.length - 1) + const loadedDatasets = (datasets || []).reverse().filter( + ({ isActive }, i) => isActive || i === 0 || i === datasets.length - 1, + ) const leftContent = ( @@ -622,7 +619,7 @@ class IndividualRow extends React.PureComponent { const rightContent = editCaseReview ? : ( @@ -646,7 +643,7 @@ export { IndividualRow as IndividualRowComponent } const mapStateToProps = (state, ownProps) => ({ project: getCurrentProject(state), - samplesByGuid: getSamplesByGuid(state), + datasets: getDatasetsByIndividual(state)[ownProps.individual.individualGuid], mmeSubmission: getMmeSubmissionsByGuid(state)[ownProps.individual.mmeSubmissionGuid], alignmentSample: ( getIGVSamplesByFamilySampleIndividual(state)[ownProps.individual.familyGuid]?.alignment || {} diff --git a/ui/pages/Project/constants.js b/ui/pages/Project/constants.js index 4c2e4c11c6..be1991a1f7 100644 --- a/ui/pages/Project/constants.js +++ b/ui/pages/Project/constants.js @@ -367,18 +367,16 @@ export const FAMILY_SORT_OPTIONS = [ { value: SORT_BY_DATA_LOADED_DATE, name: 'Date Loaded', - createSortKeyGetter: (individualsByGuid, samplesByFamily) => (family) => { - const loadedSamples = samplesByFamily[family.familyGuid] || [] - return loadedSamples.length ? loadedSamples[loadedSamples.length - 1].loadedDate : '2000-01-01T01:00:00.000Z' - }, + createSortKeyGetter: (individualsByGuid, minMaxDatasetsByFamily) => family => ( + minMaxDatasetsByFamily[family.familyGuid][1]?.loadedDate || '2000-01-01T01:00:00.000Z' + ), }, { value: SORT_BY_DATA_FIRST_LOADED_DATE, name: 'Date First Loaded', - createSortKeyGetter: (individualsByGuid, samplesByFamily) => (family) => { - const loadedSamples = samplesByFamily[family.familyGuid] || [] - return loadedSamples.length ? loadedSamples[0].loadedDate : '2000-01-01T01:00:00.000Z' - }, + createSortKeyGetter: (individualsByGuid, minMaxDatasetsByFamily) => family => ( + minMaxDatasetsByFamily[family.familyGuid][0]?.loadedDate || '2000-01-01T01:00:00.000Z' + ), }, { value: SORT_BY_ANALYSIS_STATUS, @@ -506,12 +504,12 @@ export const CASE_REVIEW_INDIVIDUAL_EXPORT_DATA = [ export const SAMPLE_EXPORT_DATA = [ { header: 'Family ID', field: 'familyId' }, { header: 'Individual ID', field: 'individualId' }, - { header: 'Sample ID', field: 'sampleId' }, { header: 'Loaded Date', field: 'loadedDate' }, { header: 'Sample Type', field: 'sampleType' }, + { header: 'Dataset Type', field: 'datasetType' }, ] -export const TAG_FIELD_NAME = 'tags' +const TAG_FIELD_NAME = 'tags' export const TAG_FORM_FIELD = { name: TAG_FIELD_NAME, diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index 4a406efbc7..c2181b8dfe 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -145,6 +145,13 @@ export const loadFamilySavedVariants = familyGuid => loadSavedVariants({ familyG export const unloadProject = () => dispatch => dispatch({ type: UPDATE_CURRENT_PROJECT, newValue: null }) +export const createManualVariant = (familyGuid, values) => dispatch => new HttpRequestHelper( + `/api/saved_variant/create_manual/${familyGuid}`, + (responseJson) => { + dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) + }, +).post(values) + export const updateFamilies = values => (dispatch, getState) => { const action = values.delete ? 'delete' : 'edit' return new HttpRequestHelper(`/api/project/${getState().currentProjectGuid}/${action}_families`, diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 2e36795d83..172b8ad08b 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -17,12 +17,13 @@ import { import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' import { - getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getSamplesByGuid, getGenesById, getUser, + getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, - getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, - getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, + getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, + getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, + getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, } from 'redux/selectors' import { @@ -134,11 +135,9 @@ export const getProjectAnalysisGroupFamilySizeHistogram = createSelector( export const getProjectAnalysisGroupDataLoadedFamilySizeHistogram = createSelector( getProjectAnalysisGroupFamiliesByGuid, - getSamplesByFamily, - (familiesByGuid, samplesByFamily) => getFamilySizeHistogram(Object.values(familiesByGuid).map(((family) => { - const sampleIndividuals = new Set((samplesByFamily[family.familyGuid] || []).filter( - sample => sample.isActive, - ).map(sample => sample.individualGuid)) + getActiveDatasetsByFamily, + (familiesByGuid, datasetsByFamily) => getFamilySizeHistogram(Object.values(datasetsByFamily).map(((family) => { + const sampleIndividuals = new Set((datasetsByFamily[family.familyGuid] || []).map(sample => sample.individualGuid)) const hasSampleParents = (family.parents || []).reduce( (acc, { individualGuid, maternalGuid, paternalGuid }) => { const hasSampleMaternal = sampleIndividuals.has(maternalGuid) @@ -456,16 +455,16 @@ export const getVisibleFamilies = createSelector( export const getVisibleFamiliesInSortedOrder = createSelector( getVisibleFamilies, getIndividualsByGuid, - getSamplesByFamily, + getMinMaxDatasetsByFamily, getFamiliesSortOrder, getFamiliesSortDirection, - (visibleFamilies, individualsByGuid, samplesByFamily, familiesSortOrder, familiesSortDirection) => { + (visibleFamilies, individualsByGuid, minMaxDatasetsByFamily, familiesSortOrder, familiesSortDirection) => { if (!familiesSortOrder || !FAMILY_SORT_LOOKUP[familiesSortOrder] || visibleFamilies.some(({ familyId }) => !familyId)) { // families have been loaded without any core fields return visibleFamilies } - const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, samplesByFamily) + const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, minMaxDatasetsByFamily) return visibleFamilies.slice(0).sort((a, b) => getSortKey(a).localeCompare(getSortKey(b)) * familiesSortDirection) }, ) @@ -481,50 +480,40 @@ export const getEntityExportConfig = ({ projectName, tableName, fileName, fields const getFamiliesExportData = createSelector( getVisibleFamiliesInSortedOrder, - getSamplesByFamily, + getMinMaxDatasetsByFamily, getNotesByFamilyType, - (visibleFamilies, samplesByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { + (visibleFamilies, minMaxDatasetsByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { ...family, ...FAMILY_NOTES_FIELDS.reduce((noteAcc, { id, noteType }) => ( { ...noteAcc, [id]: (notesByFamilyType[family.familyGuid] || {})[noteType] }), {}), - [FAMILY_FIELD_FIRST_SAMPLE]: (samplesByFamily[family.familyGuid] || [])[0], + [FAMILY_FIELD_FIRST_SAMPLE]: (minMaxDatasetsByFamily[family.familyGuid] || [])[0], }], []), ) -const getSamplesByIndividual = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).reduce((acc, sample) => { - if (!acc[sample.individualGuid]) { - acc[sample.individualGuid] = [] - } - acc[sample.individualGuid].push(sample) - return acc - }, {}), -) - const getIndividualsExportData = createSelector( getVisibleFamiliesInSortedOrder, getSortedIndividualsByFamily, - getSamplesByIndividual, - (families, individualsByFamily, samplesByIndividual) => families.reduce((acc, family) => [ + getActiveDatasetsByIndividual, + (families, individualsByFamily, activeDatasetsByIndividual) => families.reduce((acc, family) => [ ...acc, ...(individualsByFamily[family.familyGuid] || []).map(individual => ({ ...individual, [FAMILY_FIELD_ID]: family.familyId, - [INDIVIDUAL_HAS_DATA_FIELD]: ( - samplesByIndividual[individual.individualGuid] || []).some(({ isActive }) => isActive), + [INDIVIDUAL_HAS_DATA_FIELD]: (activeDatasetsByIndividual[individual.individualGuid] || []).length > 0, }))], []), ) const getSamplesExportData = createSelector( getVisibleFamiliesInSortedOrder, - getIndividualsByGuid, - getSamplesByFamily, - (visibleFamilies, individualsByGuid, samplesByFamily) => visibleFamilies.reduce((acc, family) => [ - ...acc, ...(samplesByFamily[family.familyGuid] || []).map(sample => ({ - ...sample, - [FAMILY_FIELD_ID]: family.familyId, - [INDIVIDUAL_FIELD_ID]: individualsByGuid[sample.individualGuid]?.individualId, - }))], []), + getSortedIndividualsByFamily, + getDatasetsByIndividual, + (visibleFamilies, individualsByFamily, datasetsByIndividual) => visibleFamilies.reduce((acc, family) => [ + ...acc, ...(individualsByFamily[family.familyGuid] || []).reduce((indivAcc, { individualGuid, individualId }) => ([ + ...indivAcc, ...(datasetsByIndividual[individualGuid] || []).map(sample => ({ + ...sample, + [FAMILY_FIELD_ID]: family.familyId, + [INDIVIDUAL_FIELD_ID]: individualId, + })), + ]), [])], []), ) export const getProjectExportUrls = createSelector( diff --git a/ui/pages/Project/selectors.test.js b/ui/pages/Project/selectors.test.js index 502fdc28f4..ce3ed0dc11 100644 --- a/ui/pages/Project/selectors.test.js +++ b/ui/pages/Project/selectors.test.js @@ -47,8 +47,10 @@ test('getProjectExportUrls', () => { expect(exportFamilies.length).toEqual(2) expect(exportFamilies[0].familyGuid).toEqual('F011652_2') expect(exportFamilies[1].familyGuid).toEqual('F011652_1') - expect(exportFamilies[0].firstSample.sampleGuid).toEqual('S2310656_wal_mc16200_mc16203') - expect(exportFamilies[1].firstSample).toEqual(undefined) + expect(exportFamilies[0].firstSample.datasetType).toEqual('SNV_INDEL') + expect(exportFamilies[0].firstSample.sampleType).toEqual('WES') + expect(exportFamilies[0].firstSample.loadedDate).toEqual('2018-03-13') + expect(exportFamilies[1].firstSample).toEqual(null) expect(exportFamilies[0].analysisNotes.length).toEqual(2) expect(exportFamilies[0].caseNotes).toEqual(undefined) diff --git a/ui/pages/Report/components/VariantMetadata.jsx b/ui/pages/Report/components/VariantMetadata.jsx index 828c65f416..b077b70c36 100644 --- a/ui/pages/Report/components/VariantMetadata.jsx +++ b/ui/pages/Report/components/VariantMetadata.jsx @@ -10,7 +10,7 @@ const VIEW_ALL_PAGES = [ const COLUMNS = [ { name: 'participant_id' }, - ...VARIANT_METADATA_COLUMNS.slice(0, -1), + ...VARIANT_METADATA_COLUMNS, { name: 'variant_type' }, { name: 'allele_balance_or_heteroplasmy_percentage' }, { name: 'Clinvar allele ID', format: ({ clinvar }) => clinvar?.alleleId }, @@ -22,7 +22,7 @@ const COLUMNS = [ { name: 'additional_family_members_with_variant' }, { name: 'method_of_discovery' }, { name: 'Submitted to MME', format: ({ MME }) => (MME ? 'Yes' : 'No') }, - ...VARIANT_METADATA_COLUMNS.slice(-1), + { name: 'ClinGen_allele_ID' }, { name: 'tags', format: ({ tags }) => (tags || []).join('; ') }, ] diff --git a/ui/pages/SummaryData/components/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx similarity index 55% rename from ui/pages/SummaryData/components/VariantLookup.jsx rename to ui/pages/Search/VariantLookup.jsx index 961f236a2a..ccbb4f6982 100644 --- a/ui/pages/SummaryData/components/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -1,25 +1,41 @@ import React from 'react' import { connect } from 'react-redux' import PropTypes from 'prop-types' -import { Grid, Header, Label, Table } from 'semantic-ui-react' +import { Grid, Header, Icon, Label, Popup, Table } from 'semantic-ui-react' import { RECEIVE_DATA } from 'redux/utils/reducerUtils' +import { navigateSavedHashedSearch } from 'redux/rootReducer' import { getVlmEnabled } from 'redux/selectors' import { QueryParamsEditor } from 'shared/components/QueryParamEditor' import StateDataLoader from 'shared/components/StateDataLoader' import SendEmailButton from 'shared/components/buttons/SendEmailButton' import FormWrapper from 'shared/components/form/FormWrapper' -import { helpLabel } from 'shared/components/form/FormHelpers' -import { BaseSemanticInput, InlineToggle } from 'shared/components/form/Inputs' +import { helpLabel, validators } from 'shared/components/form/FormHelpers' +import { BaseSemanticInput, InlineToggle, AlignedCheckboxGroup } from 'shared/components/form/Inputs' +import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' import FamilyReads from 'shared/components/panel/family/FamilyReads' import FamilyVariantTags from 'shared/components/panel/variants/FamilyVariantTags' import Variants, { Variant, StyledVariantRow } from 'shared/components/panel/variants/Variants' import { FamilyVariantIndividuals } from 'shared/components/panel/variants/VariantIndividuals' -import { GENOME_VERSION_FIELD, GENOME_VERSION_37, GENOME_VERSION_38 } from 'shared/utils/constants' -import { sendVlmContactEmail } from '../reducers' -import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail } from '../selectors' +import { + GENOME_VERSION_FIELD, + GENOME_VERSION_37, + GENOME_VERSION_38, + GENE_SEARCH_FREQUENCIES, + GROUPED_VEP_CONSEQUENCES, + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +} from 'shared/utils/constants' +import { snakecaseToTitlecase } from 'shared/utils/stringUtils' +import { sendVlmContactEmail } from './reducers' +import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail, getSearchedVariantsIsLoading } from './selectors' -const FIELDS = [ +const LOOKUP_FIELDS = [ { name: 'variantId', label: helpLabel('Variant ID', ( @@ -56,6 +72,47 @@ const FIELDS = [ }, ] +const validateAnnotations = (value, { search }) => ( + value || Object.values(search.annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' +) + +const CONSEQUENCE_FIELDS = [ + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +].map((group, i) => ({ + name: `search.annotations.${group}`, + component: AlignedCheckboxGroup, + groupLabel: snakecaseToTitlecase(group), + options: GROUPED_VEP_CONSEQUENCES[group], + format: value => value || [], + inline: true, + validate: i === 0 ? validateAnnotations : undefined, +})) + +const GENE_LOOKUP_FIELDS = [ + { validate: validators.required, ...GENOME_VERSION_FIELD, name: 'allGenomeProjectFamilies' }, + { + name: 'search.locus.rawItems', + label: 'Gene', + control: AwesomeBarFormInput, + categories: ['genes'], + fluid: true, + placeholder: 'Search for gene', + validate: validators.required, + }, + ...CONSEQUENCE_FIELDS, +] + +const INITIAL_GENE_LOOKUP_VALUES = { + includeNoAccessProjects: true, + search: { freqs: GENE_SEARCH_FREQUENCIES }, +} + const VlmDisplay = ({ vlmMatches }) => ( {Object.entries(vlmMatches || {}).map( @@ -178,12 +235,12 @@ BaseLookupVariant.propTypes = { vlmDefaultContactEmails: PropTypes.object, } -const mapStateToProps = (state, ownProps) => ({ +const mapVariantStateToProps = (state, ownProps) => ({ familiesByContactEmail: getVlmFamiliesByContactEmail(state, ownProps), vlmDefaultContactEmails: getVlmDefaultContactEmails(state, ownProps), }) -const LookupVariant = connect(mapStateToProps)(BaseLookupVariant) +const LookupVariant = connect(mapVariantStateToProps)(BaseLookupVariant) const VariantDisplay = ({ variants }) => ( (variants || [])[0]?.lookupFamilyGuids ? : @@ -193,47 +250,37 @@ VariantDisplay.propTypes = { variants: PropTypes.arrayOf(PropTypes.object), } -const onSubmit = updateQueryParams => (data) => { +const onQueryParamSubmit = (updateQueryParams, data) => { updateQueryParams(data) return Promise.resolve() } const passThroughResponse = response => response -const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled }) => ( - - - - -
- +const BaseVariantLookupResults = ({ queryParams, receiveData, vlmEnabled }) => ([ + ...(vlmEnabled ? [( + + +
+ + + - - {vlmEnabled && ( - - -
- - - - - - )} - {vlmEnabled && ( - - -
- - - - )} - + ), ( + + +
+ + + + )] : []), ( + - -) + ), +]) -VariantLookup.propTypes = { +BaseVariantLookupResults.propTypes = { receiveData: PropTypes.func, - updateQueryParams: PropTypes.func, queryParams: PropTypes.object, vlmEnabled: PropTypes.bool, } -const mapGlobalStateToProps = state => ({ +const mapResultsStateToProps = state => ({ vlmEnabled: getVlmEnabled(state), }) -const mapDispatchToProps = dispatch => ({ +const mapResultsDispatchToProps = dispatch => ({ receiveData: (updatesById) => { dispatch({ type: RECEIVE_DATA, updatesById }) return updatesById }, }) -const WrappedVariantLookup = ({ vlmEnabled, ...props }) => ( - - - +const VariantLookupResults = connect(mapResultsStateToProps, mapResultsDispatchToProps)(BaseVariantLookupResults) + +const LOOKUP_HEADER = { content: 'Lookup Variant' } +const GENE_LOOKUP_HEADER = { + content: 'Lookup Variants in Gene', + subheader: ( + + Lookup up all rare variants in seqr in a given gene, regardless of whether or not they are in your + projects. +
+ Variants are only returned if they have a gnomAD Allele Frequency < 3% + and have a seqr global Allele Count < 3000. +
+ For more granular control over the returned variants, we recommend using the main Search page instead of this + lookup   + } + content={ + 'On the Search page, select your desired genome build in the "Include All Projects" section, select the ' + + '"Include External Projects" checkbox next to it, and enter your gene of interest in the "Location" search ' + + 'box along with your other desired search filters.' + } + hoverable + /> +
+ +
+ +
+ ), +} + +const BaseVariantLookup = ({ match, formProps, onSubmit, ...props }) => ( + + + +
+
+ + + + + + + + + + {!match.params.gene && } + ) -WrappedVariantLookup.propTypes = { - vlmEnabled: PropTypes.bool, +BaseVariantLookup.propTypes = { + match: PropTypes.object, + formProps: PropTypes.object, + onSubmit: PropTypes.func, } -export default connect(mapGlobalStateToProps, mapDispatchToProps)(WrappedVariantLookup) +const mapStateToProps = (state, ownProps) => ({ + formProps: ownProps.match.params.gene ? { + showErrorPanel: true, + loading: getSearchedVariantsIsLoading(state), + initialValues: INITIAL_GENE_LOOKUP_VALUES, + fields: GENE_LOOKUP_FIELDS, + } : { + fields: LOOKUP_FIELDS, + initialValues: ownProps.queryParams, + }, +}) + +const mapDispatchToProps = (dispatch, ownProps) => ({ + onSubmit: data => (ownProps.match.params.gene ? + dispatch(navigateSavedHashedSearch(data)) : + onQueryParamSubmit(ownProps.updateQueryParams, data) + ), +}) + +const VariantLookup = connect(mapStateToProps, mapDispatchToProps)(BaseVariantLookup) + +export default props => ( + + + +) diff --git a/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx b/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx index fce85b7b04..8790eaa5ee 100644 --- a/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx +++ b/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx @@ -6,10 +6,11 @@ import { Form, Button } from 'semantic-ui-react' import { getProjectsByGuid, getProjectAnalysisGroupFamilyGuidsByGuid, + getProjectDatasetTypes, } from 'redux/selectors' import { Multiselect, ButtonRadioGroup } from 'shared/components/form/Inputs' import { SOLVED_FAMILY_STATUS_OPTIONS } from 'shared/utils/constants' -import { getProjectFamilies, getSearchContextIsLoading, getFamilyOptions, getAnalysisGroupOptions, getProjectDatasetTypes } from '../../selectors' +import { getProjectFamilies, getSearchContextIsLoading, getFamilyOptions, getAnalysisGroupOptions } from '../../selectors' import { loadProjectFamiliesContext } from '../../reducers' import { ProjectFilter } from './ProjectsField' diff --git a/ui/pages/Search/reducers.js b/ui/pages/Search/reducers.js index 8d3e38a32b..67cbd5ea9a 100644 --- a/ui/pages/Search/reducers.js +++ b/ui/pages/Search/reducers.js @@ -219,6 +219,10 @@ export const loadGeneBreakdown = searchHash => (dispatch, getState) => { } } +export const sendVlmContactEmail = values => () => new HttpRequestHelper( + '/api/send_vlm_email', +).post(values) + // reducers export const reducers = { diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 6b49def59a..6fea91ec0d 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -12,10 +12,11 @@ import { getSearchFamiliesByHash, getGenesById, getSearchesByHash, - getSamplesGroupedByProjectGuid, - getSamplesByFamily, + getProjectDatasetTypes, + getActiveDatasetsByFamily, + getSortedIndividualsByFamily, } from 'redux/selectors' -import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' +import { getVariantMainGeneId, getVariantSummary, FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' import { compareObjects } from 'shared/utils/sortUtils' import { compHetGene } from 'shared/components/panel/variants/VariantUtils' @@ -196,28 +197,15 @@ export const getLocusListOptions = createListEqualSelector( }, ) -const getSampleDatasetTypes = samples => ([ - ...new Set((samples || []).filter(({ isActive }) => isActive).map(({ datasetType }) => datasetType)), -]) - -export const getProjectDatasetTypes = createSelector( - getProjectsByGuid, - getSamplesGroupedByProjectGuid, - (projectsByGuid, samplesByProjectGuid) => Object.values(projectsByGuid).reduce( - (acc, { projectGuid, datasetTypes }) => ({ - ...acc, - [projectGuid]: datasetTypes || getSampleDatasetTypes(Object.values(samplesByProjectGuid[projectGuid] || {})), - }), {}, - ), -) - export const getDatasetTypes = createSelector( (state, props) => props.projectFamilies, getProjectDatasetTypes, - getSamplesByFamily, - (projectFamilies, projectDatasetTypes, samplesByFamily) => { + getActiveDatasetsByFamily, + (projectFamilies, projectDatasetTypes, datasetsByFamily) => { const isSingleFamily = (projectFamilies || []).length === 1 && projectFamilies[0].familyGuids?.length === 1 - const datasetTypes = isSingleFamily ? getSampleDatasetTypes(samplesByFamily[projectFamilies[0].familyGuids[0]]) : ( + const datasetTypes = isSingleFamily ? [ + ...new Set((datasetsByFamily[projectFamilies[0].familyGuids[0]] || []).map(({ datasetType }) => datasetType)), + ] : ( projectFamilies || [] ).reduce((acc, { projectGuid }) => new Set([ ...acc, ...(projectDatasetTypes[projectGuid] || [])]), new Set()) @@ -291,3 +279,32 @@ export const getSearchGeneBreakdownValues = createSelector( }), })), ) + +export const getVlmFamiliesByContactEmail = createSelector( + getSortedIndividualsByFamily, + (state, ownProps) => ownProps.variant, + (individualsByFamily, variant) => (variant.lookupFamilyGuids || []).reduce((acc, familyGuid) => { + const individual = individualsByFamily[familyGuid]?.[0] + const contactEmail = individual?.projectGuid ? 'internal' : (individual?.vlmContactEmail || 'disabled') + return { ...acc, [contactEmail]: [...(acc[contactEmail] || []), familyGuid] } + }, {}), +) + +export const getVlmDefaultContactEmails = createSelector( + getVlmFamiliesByContactEmail, + getGenesById, + getUser, + (state, ownProps) => ownProps.variant, + (familiesByContactEmail, genesById, user, variant) => { + const gene = genesById[getVariantMainGeneId(variant)]?.geneSymbol + const subject = `${gene || variant.variantId} variant match in seqr` + const defaultEmailContent = `harboring ${getVariantSummary(variant)} in ${gene || 'no genes'} (${window.location.href}). + \n\nWe have identified the variant in a case with [replace with phenotype]. + \n\n[List your specific questions for the researcher here.] + \n\nWe appreciate your assistance and look forward to hearing more from you.\n\nBest wishes,\n${user.displayName}` + return Object.entries(familiesByContactEmail).reduce((acc, [to, familyGuids]) => ({ + ...acc, + [to]: { to, subject, body: `Dear researcher,\n\nWe are interested in learning more about your ${familyGuids.length} cases in seqr ${defaultEmailContent}` }, + }), {}) + }, +) diff --git a/ui/pages/Search/selectors.test.js b/ui/pages/Search/selectors.test.js index 78b77f1bc7..b8dd139d22 100644 --- a/ui/pages/Search/selectors.test.js +++ b/ui/pages/Search/selectors.test.js @@ -64,7 +64,7 @@ test('getLocusListOptions', () => { test('getDatasetTypes', () => { expect(getDatasetTypes(STATE, { projectFamilies: [{ projectGuid: PROJECT_GUID }] })).toEqual('SNV_INDEL,SV') - expect(getDatasetTypes(STATE, EXPECTED_INITAL_SEARCH)).toEqual('SNV_INDEL') + expect(getDatasetTypes(STATE, EXPECTED_INITAL_SEARCH)).toEqual('SNV_INDEL,SV') }) test('getSearchGeneBreakdownValues', () => { diff --git a/ui/pages/SummaryData/SummaryData.jsx b/ui/pages/SummaryData/SummaryData.jsx index e683abcb51..01bc14758d 100644 --- a/ui/pages/SummaryData/SummaryData.jsx +++ b/ui/pages/SummaryData/SummaryData.jsx @@ -16,7 +16,6 @@ import LocusLists from './components/LocusLists' import ExternalAnalysis from './components/ExternalAnalysis' import Hpo from './components/Hpo' import IndividualMetadata from './components/IndividualMetadata' -import VariantLookup from './components/VariantLookup' const GenePage = ({ match }) => ( match.params.geneId ? : @@ -27,7 +26,6 @@ GenePage.propTypes = { } const SUMMARY_DATA_PAGES = [ - { path: 'variant_lookup', component: VariantLookup }, { path: 'gene_info', params: '/:geneId?', component: GenePage }, { path: 'gene_lists', component: LocusLists }, { path: 'saved_variants', component: SavedVariants }, diff --git a/ui/pages/SummaryData/components/IndividualMetadata.test.js b/ui/pages/SummaryData/components/IndividualMetadata.test.js index 1166facee6..c7a4be4264 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.test.js +++ b/ui/pages/SummaryData/components/IndividualMetadata.test.js @@ -76,8 +76,6 @@ const DATA = [ 'phenotype_contribution-2': 'Full', 'partial_contribution_explained-1': 'HP:0000501|HP:0000365', 'partial_contribution_explained-2': '', - 'ClinGen_allele_ID-1': 'CA1501729', - 'ClinGen_allele_ID-2': null, }, ] @@ -93,19 +91,19 @@ test('IndividualMetadata render and export', () => { 'filter_flags', 'consanguinity', 'family_history', 'genetic_findings_id-1', 'variant_reference_assembly-1', 'chrom-1', 'pos-1', 'chrom_end-1', 'pos_end-1', 'ref-1', 'alt-1', 'gene_of_interest-1', 'gene_id-1', 'seqr_chosen_consequence-1', 'transcript-1', 'hgvsc-1', 'hgvsp-1', 'zygosity-1', 'copy_number-1', 'sv_name-1', 'validated_name-1', 'sv_type-1', 'variant_inheritance-1', 'gene_known_for_phenotype-1', - 'phenotype_contribution-1', 'partial_contribution_explained-1', 'notes-1', 'ClinGen_allele_ID-1', + 'phenotype_contribution-1', 'partial_contribution_explained-1', 'notes-1', 'genetic_findings_id-2', 'variant_reference_assembly-2', 'chrom-2', 'pos-2', 'chrom_end-2', 'pos_end-2', 'ref-2', 'alt-2', 'gene_of_interest-2', 'gene_id-2', 'seqr_chosen_consequence-2', 'transcript-2', 'hgvsc-2', 'hgvsp-2', 'zygosity-2', 'copy_number-2', 'sv_name-2', 'validated_name-2', 'sv_type-2', 'variant_inheritance-2', 'gene_known_for_phenotype-2', - 'phenotype_contribution-2', 'partial_contribution_explained-2', 'notes-2', 'ClinGen_allele_ID-2']) + 'phenotype_contribution-2', 'partial_contribution_explained-2', 'notes-2']) expect(exportConfig.processRow(DATA[0])).toEqual([ 'Test Reprocessed Project', 'R0003_test', '12', 'F000012_12', 'NA20889', 'I000017_na20889', '', '', '', '', 'Self', 'Female (XXX)', 'Ashkenazi Jewish', 'Affected', 'HP:0011675 (Arrhythmia)|HP:0001509 ()', '', 'Yes', null, 'OMIM:616126', 'Immunodeficiency 38', 'Autosomal recessive', null, null, undefined, 'Waiting for data', 'Tier 1', 'WES', '2017-02-05', '', undefined, 'Yes', 'NA20889_1_248367227', undefined, '1', 248367227, null, null, 'TC', 'T', 'OR4G11P', 'ENSG00000240361', 'intron_variant', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', null, undefined, undefined, undefined, - 'unknown', 'Candidate', 'Partial', 'HP:0000501|HP:0000365', undefined, 'CA1501729', 'NA20889_1_249045487_DEL', undefined, + 'unknown', 'Candidate', 'Partial', 'HP:0000501|HP:0000365', undefined, 'NA20889_1_249045487_DEL', undefined, '12', '49045487', null, '49045898', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, 'Heterozygous', 1, 'DEL:chr12:49045487-49045898', 'DEL:chr12:49045123-49045456', 'Deletion', - 'unknown', 'Candidate', 'Full', '', undefined, null]) + 'unknown', 'Candidate', 'Full', '', undefined]) }) diff --git a/ui/pages/SummaryData/reducers.js b/ui/pages/SummaryData/reducers.js index a3111e8086..761f79b87e 100644 --- a/ui/pages/SummaryData/reducers.js +++ b/ui/pages/SummaryData/reducers.js @@ -78,10 +78,6 @@ export const updateExternalAnalysis = values => dispatch => new HttpRequestHelpe }, ).post(values) -export const sendVlmContactEmail = values => () => new HttpRequestHelper( - '/api/summary_data/send_vlm_email', -).post(values) - export const reducers = { successStoryLoading: loadingReducer(REQUEST_SUCCESS_STORY, RECEIVE_SUCCESS_STORY), successStoryRows: createSingleValueReducer(RECEIVE_SUCCESS_STORY, []), diff --git a/ui/pages/SummaryData/selectors.js b/ui/pages/SummaryData/selectors.js index 236dd98249..9676010112 100644 --- a/ui/pages/SummaryData/selectors.js +++ b/ui/pages/SummaryData/selectors.js @@ -1,8 +1,3 @@ -import { createSelector } from 'reselect' - -import { getSortedIndividualsByFamily, getGenesById, getUser } from 'redux/selectors' -import { getVariantMainGeneId, getVariantSummary } from 'shared/utils/constants' - export const getSuccessStoryLoading = state => state.successStoryLoading.isLoading export const getSuccessStoryLoadingError = state => state.successStoryLoading.errorMessage export const getSuccessStoryRows = state => state.successStoryRows @@ -11,32 +6,3 @@ export const getMmeLoadingError = state => state.mmeLoading.errorMessage export const getMmeMetrics = state => state.mmeMetrics export const getMmeSubmissions = state => state.mmeSubmissions export const getExternalAnalysisUploadStats = state => state.externalAnalysisUploadStats - -export const getVlmFamiliesByContactEmail = createSelector( - getSortedIndividualsByFamily, - (state, ownProps) => ownProps.variant, - (individualsByFamily, variant) => (variant.lookupFamilyGuids || []).reduce((acc, familyGuid) => { - const individual = individualsByFamily[familyGuid]?.[0] - const contactEmail = individual?.projectGuid ? 'internal' : (individual?.vlmContactEmail || 'disabled') - return { ...acc, [contactEmail]: [...(acc[contactEmail] || []), familyGuid] } - }, {}), -) - -export const getVlmDefaultContactEmails = createSelector( - getVlmFamiliesByContactEmail, - getGenesById, - getUser, - (state, ownProps) => ownProps.variant, - (familiesByContactEmail, genesById, user, variant) => { - const gene = genesById[getVariantMainGeneId(variant)]?.geneSymbol - const subject = `${gene || variant.variantId} variant match in seqr` - const defaultEmailContent = `harboring ${getVariantSummary(variant)} in ${gene || 'no genes'} (${window.location.href}). - \n\nWe have identified the variant in a case with [replace with phenotype]. - \n\n[List your specific questions for the researcher here.] - \n\nWe appreciate your assistance and look forward to hearing more from you.\n\nBest wishes,\n${user.displayName}` - return Object.entries(familiesByContactEmail).reduce((acc, [to, familyGuids]) => ({ - ...acc, - [to]: { to, subject, body: `Dear researcher,\n\nWe are interested in learning more about your ${familyGuids.length} cases in seqr ${defaultEmailContent}` }, - }), {}) - }, -) diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index b93f60411d..d28acefe1f 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -225,12 +225,12 @@ export const updateVariantTags = (values, tagType = 'tags') => { export const updateVariantClassification = values => updateSavedVariant(values, `${values.variant.variantGuid}/update_acmg_classification`) -export const updateVariantMainTranscript = (variantGuid, transcriptId) => dispatch => new HttpRequestHelper( - `/api/saved_variant/${variantGuid}/update_transcript/${transcriptId}`, +export const updateVariantMainTranscript = (variantGuid, transcript) => dispatch => new HttpRequestHelper( + `/api/saved_variant/${variantGuid}/update_transcript/${transcript.transcriptId}`, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) }, -).post() +).post(transcript) export const updateLocusList = values => (dispatch) => { let action = 'create' diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 702baebb9a..3fe2aace45 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -11,7 +11,7 @@ export const getFamiliesByGuid = state => state.familiesByGuid export const getFamilyNotesByGuid = state => state.familyNotesByGuid export const getFamilyDetailsLoading = state => state.familyDetailsLoading export const getIndividualsByGuid = state => state.individualsByGuid -export const getSamplesByGuid = state => state.samplesByGuid +const getSamplesByGuid = state => state.samplesByGuid export const getIgvSamplesByGuid = state => state.igvSamplesByGuid export const getAnalysisGroupsByGuid = state => state.analysisGroupsByGuid export const getAnalysisGroupIsLoading = state => state.analysisGroupsLoading.isLoading @@ -56,7 +56,6 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce(( }, {}) export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid) export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid) -export const getSamplesGroupedByProjectGuid = createSelector(getSamplesByGuid, groupEntitiesByProjectGuid) const groupByFamilyGuid = objs => objs.reduce((acc, o) => { if (!acc[o.familyGuid]) { @@ -66,6 +65,53 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { return acc }, {}) +export const getProjectDatasetTypes = createSelector( + getProjectsByGuid, + getSamplesByGuid, + (projectsByGuid, samplesByGuid) => { + const projectDatasetTypes = Object.values(projectsByGuid).reduce( + (acc, { projectGuid, datasetTypes }) => ({ ...acc, [projectGuid]: datasetTypes }), {}, + ) + const sampleDatasetTypes = Object.values(samplesByGuid).reduce( + (acc, { projectGuid, datasetType, isActive }) => { + if (projectDatasetTypes[projectGuid] || !isActive) { + return acc + } + if (!acc[projectGuid]) { + acc[projectGuid] = new Set() + } + acc[projectGuid].add(datasetType) + return acc + }, {}, + ) + return { ...projectDatasetTypes, ...sampleDatasetTypes } + }, +) + +export const getDatasetsByIndividual = createSelector( + getSamplesByGuid, + samplesByGuid => Object.values(samplesByGuid).sort( + (a, b) => a.loadedDate.localeCompare(b.loadedDate), + ).reduce((acc, sample) => { + const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive, sampleType, datasetType, loadedDate: loadedDate.split('T')[0] }) + return acc + }, {}), +) + +export const getActiveDatasetsByIndividual = createSelector( + getDatasetsByIndividual, + datasetsByIndividual => Object.entries(datasetsByIndividual).reduce((acc, [individualGuid, datasets]) => ({ + ...acc, + [individualGuid]: datasets.filter(({ isActive }) => isActive).map( + ({ sampleType, datasetType, loadedDate }) => ({ sampleType, datasetType, loadedDate, individualGuid }), + ), + }), {}), +) + export const getNotesByFamilyType = createSelector( getFamilyNotesByGuid, notesByGuid => Object.values(notesByGuid).reduce((acc, note) => { @@ -131,23 +177,46 @@ export const getSortedIndividualsByFamily = createSelector( groupByFamilyGuid, ) -const getSortedSamples = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).sort((a, b) => a.loadedDate.localeCompare(b.loadedDate)), +export const getActiveDatasetsByFamily = createSelector( + getFamiliesByGuid, + getActiveDatasetsByIndividual, + (familiesByGuid, activeDatasetsByIndividual) => Object.entries(familiesByGuid).reduce( + (acc, [familyGuid, { individualGuids = [] }]) => ({ + ...acc, + [familyGuid]: individualGuids.reduce( + (familyAcc, individualGuid) => [...familyAcc, ...(activeDatasetsByIndividual[individualGuid] || [])], [], + ), + }), + {}, + ), ) -export const getSamplesByFamily = createSelector( - getSortedSamples, - sortedSamples => groupByFamilyGuid(sortedSamples || []), +export const getMinMaxDatasetsByFamily = createSelector( + getFamiliesByGuid, + getDatasetsByIndividual, + (familiesByGuid, datasetsByIndividual) => Object.entries(familiesByGuid).reduce( + (acc, [familyGuid, { individualGuids }]) => ({ + ...acc, + [familyGuid]: individualGuids.reduce(([minLoaded, maxLoaded], individualGuid) => { + const individualDatasets = datasetsByIndividual[individualGuid] || [] + if (individualDatasets.length === 0) { + return [minLoaded, maxLoaded] + } + const endIndex = individualDatasets.length - 1 + return [ + minLoaded?.loadedDate < individualDatasets[0]?.loadedDate ? minLoaded : individualDatasets[0], + maxLoaded?.loadedDate > individualDatasets[endIndex]?.loadedDate ? maxLoaded : individualDatasets[endIndex], + ] + }, [null, null]), + }), + {}, + ), ) export const getHasActiveSearchSampleByFamily = createSelector( - getSamplesByFamily, - samplesByFamily => Object.entries(samplesByFamily).reduce( - (acc, [familyGuid, familySamples]) => ({ - ...acc, - [familyGuid]: familySamples.some(({ isActive }) => isActive), - }), {}, + getActiveDatasetsByFamily, + activeDatasetsByFamily => Object.entries(activeDatasetsByFamily).reduce( + (acc, [familyGuid, familyDatasets]) => ({ ...acc, [familyGuid]: familyDatasets.length > 0 }), {}, ), ) @@ -428,8 +497,8 @@ const isAnalysedBy = (family, analysedByFilter, user, analysedByOptions) => { export const familyPassesFilters = createSelector( getUser, - getSamplesByFamily, - (user, samplesByFamily) => ( + getActiveDatasetsByFamily, + (user, activeDatasetsByFamily) => ( family, groupedFilters, analysedByOptions, categoryFilters = CATEGORY_FAMILY_FILTERS, ) => { if (groupedFilters.analysedBy && !isAnalysedBy(family, groupedFilters.analysedBy, user, analysedByOptions)) { @@ -439,7 +508,7 @@ export const familyPassesFilters = createSelector( const filters = categoryFilters[key]?.filter( opt => groupVals.includes(opt.value) && opt.createFilter, ).map(opt => opt.createFilter) - return !filters?.length || filters.some(filter => filter(family, user, samplesByFamily)) + return !filters?.length || filters.some(filter => filter(family, user, activeDatasetsByFamily)) }) }, ) diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index 1afbc17906..f56839d732 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -26,6 +26,8 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => {
seqr
{Object.keys(user).length ? [ , + , + , , (user.isAnalyst || user.isPm) ? : null, (user.isDataManager || user.isPm) ? : null, diff --git a/ui/shared/components/page/PageHeader.jsx b/ui/shared/components/page/PageHeader.jsx index 269b2ebd14..5cb7d96fd5 100644 --- a/ui/shared/components/page/PageHeader.jsx +++ b/ui/shared/components/page/PageHeader.jsx @@ -33,6 +33,7 @@ export const GenePageHeader = connect(mapGeneStateToProps)(BaseGenePageHeader) const NO_HEADER_PAGES = [ '/dashboard', '/create_project_from_workspace', '/workspace', '/users', '/login', '/accept_policies', ...PUBLIC_PAGES, + '/variant_lookup', ] const NO_HEADER_PAGE_TITLES = { diff --git a/ui/shared/components/panel/family/FamilyFields.jsx b/ui/shared/components/panel/family/FamilyFields.jsx index 2f3834dbde..a2a529f0c2 100644 --- a/ui/shared/components/panel/family/FamilyFields.jsx +++ b/ui/shared/components/panel/family/FamilyFields.jsx @@ -7,7 +7,7 @@ import styled from 'styled-components' import { loadUserOptions, updateFamily } from 'redux/rootReducer' import { loadProjectAnalysisGroups } from 'redux/utils/reducerUtils' import { - getSamplesByFamily, + getMinMaxDatasetsByFamily, getUserOptionsIsLoading, getHasActiveSearchSampleByFamily, getUserOptions, @@ -28,22 +28,22 @@ const NoWrap = styled.div` white-space: nowrap; ` -const BaseFirstSample = React.memo(({ firstFamilySample, compact, hasActiveVariantSample }) => ( +const BaseFirstSample = React.memo(({ firstFamilyDataset, compact, hasActiveVariantSample }) => ( )) BaseFirstSample.propTypes = { - firstFamilySample: PropTypes.object, + firstFamilyDataset: PropTypes.object, compact: PropTypes.bool, hasActiveVariantSample: PropTypes.bool, } const mapSampleDispatchToProps = (state, ownProps) => ({ - firstFamilySample: (getSamplesByFamily(state)[ownProps.familyGuid] || [])[0], + firstFamilyDataset: (getMinMaxDatasetsByFamily(state)[ownProps.familyGuid] || [])[0], hasActiveVariantSample: getHasActiveSearchSampleByFamily(state)[ownProps.familyGuid], }) diff --git a/ui/shared/components/panel/variants/Annotations.jsx b/ui/shared/components/panel/variants/Annotations.jsx index 6d0a58613a..fbb99e52fd 100644 --- a/ui/shared/components/panel/variants/Annotations.jsx +++ b/ui/shared/components/panel/variants/Annotations.jsx @@ -352,7 +352,7 @@ const variantSearchLinks = (variant, mainTranscript, genesById, user) => { const seqrSearchLink = ( Variant Lookup diff --git a/ui/shared/components/panel/variants/Transcripts.jsx b/ui/shared/components/panel/variants/Transcripts.jsx index 5406ebbe5b..c049d9dae8 100644 --- a/ui/shared/components/panel/variants/Transcripts.jsx +++ b/ui/shared/components/panel/variants/Transcripts.jsx @@ -143,7 +143,7 @@ const transcriptIdDetails = (transcript, variant, { transcriptsById, project, up transcript.transcriptId === variant.selectedMainTranscriptId ?