From 47bf1853d33d836231a7adbb4a28d48ea320fb39 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 16:05:25 -0500 Subject: [PATCH 001/252] transfer families remove condition --- .../transfer_families_to_different_project.py | 3 +- ...fer_families_to_different_project_tests.py | 29 +++++-------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 60d4780d51..356a0a615c 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand from seqr.models import Project, Family, VariantTag, VariantTagType -from seqr.utils.search.utils import backend_specific_call from seqr.utils.search.add_data_utils import trigger_delete_families_search import logging @@ -35,7 +34,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, trigger_delete_families_search)(from_project, list(families.values_list('guid', flat=True))) + trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True))) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 914ae2507f..b2feeecd53 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -2,13 +2,16 @@ import responses from seqr.models import Family, VariantTagType, VariantTag, Sample -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -class TransferFamiliesTest(object): +class TransferFamiliesClickhouseTest(AnvilAuthenticationTestCase): + fixtures = ['users', '1kg_project'] - DEACTIVATE_SEARCH = True - LOGS = [] + LOGS = [ + ('Disabled search for 7 samples in the following 1 families: 2', None), + ('Triggered Delete Families', {'detail': {'project_guid': 'R0001_1kg', 'family_guids': ['F000002_2', 'F000004_4']}}), + ] @responses.activate def test_command(self): @@ -44,24 +47,8 @@ def test_command(self): samples = Sample.objects.filter(individual__family=family) self.assertEqual(samples.count(), 7) - self.assertEqual(samples.filter(is_active=True).count(), 0 if self.DEACTIVATE_SEARCH else 7) + self.assertEqual(samples.filter(is_active=True).count(), 0) family = Family.objects.get(family_id='4') self.assertEqual(family.project.guid, 'R0003_test') self.assertEqual(family.individual_set.count(), 1) - - -class TransferFamiliesLocalTest(TransferFamiliesTest, AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - DEACTIVATE_SEARCH = False - - -class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - ES_HOSTNAME = '' - LOGS = [ - ('Disabled search for 7 samples in the following 1 families: 2', None), - ('Triggered Delete Families', {'detail': {'project_guid': 'R0001_1kg', 'family_guids': ['F000002_2', 'F000004_4']}}), - ] From a7857a809a4432cdebce479380c76ac589f117a3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 16:08:07 -0500 Subject: [PATCH 002/252] remove ui flag --- seqr/views/react_app.py | 1 - seqr/views/react_app_tests.py | 1 - 2 files changed, 2 deletions(-) diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 8d754b8c57..82480db004 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -55,7 +55,6 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'version': '{}-{}'.format(SEQR_VERSION, ui_version), 'hijakEnabled': DEBUG or False, 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, - 'elasticsearchEnabled': backend_specific_call(True, False), 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], }} diff --git a/seqr/views/react_app_tests.py b/seqr/views/react_app_tests.py index 10e2d48975..f363edcb09 100644 --- a/seqr/views/react_app_tests.py +++ b/seqr/views/react_app_tests.py @@ -24,7 +24,6 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, 'version': mock.ANY, 'hijakEnabled': False, 'oauthLoginProvider': self.OAUTH_PROVIDER, - 'elasticsearchEnabled': bool(self.ES_HOSTNAME), 'vlmEnabled': vlm_enabled, 'warningMessages': [{'id': 1, 'header': 'Warning!', 'message': 'A sample warning'}], }) From a8b3e09987d98c9173e1639af96e7e24b28167cb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 16:36:19 -0500 Subject: [PATCH 003/252] conditional clean up --- seqr/views/apis/family_api.py | 4 +--- seqr/views/apis/saved_variant_api.py | 16 ++++------------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index a273ad34a1..538774d430 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -12,7 +12,6 @@ from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json @@ -57,9 +56,8 @@ def family_page_data(request, family_guid): add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] - additional_fields = backend_specific_call([],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( - 'xpos', 'xpos_end', 'gene_ids', *additional_fields, + 'xpos', 'xpos_end', 'gene_ids', 'key', 'dataset_type', svType=F('saved_variant_json__svType') ) gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 6fa1fd5b29..c81cf9b6af 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -9,7 +9,7 @@ from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import backend_specific_call, es_only +from seqr.utils.search.utils import es_only from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -127,7 +127,7 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) + gene_id = _variant_gene_id(saved_variants[0], genome_version) create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -137,16 +137,8 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): def _variant_gene_id(variant, genome_version): - if variant.selected_main_transcript_id and len(variant.gene_ids) > 1: - return next( - gene_id for gene_id, transcripts in variant.saved_variant_json['transcripts'].items() - if any(t['transcriptId'] == variant.selected_main_transcript_id for t in transcripts)) - return variant.gene_ids[0] - - -def _clickhouse_variant_gene_id(variant, genome_version): - if not (variant.key and variant.selected_main_transcript_id) or len(variant.gene_ids) == 1: - return _variant_gene_id(variant, genome_version) + if len(variant.gene_ids) == 1 or not variant.selected_main_transcript_id: + return variant.gene_ids[0] return get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap( ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]), mapped_expression='x.geneId', output_field=ArrayField(StringField()), From c47fc1ee81329ad7f7a5be2e5b1c92a3b2fbfb13 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 16:43:09 -0500 Subject: [PATCH 004/252] clean up conditionals --- seqr/views/utils/individual_utils.py | 15 +++++---------- seqr/views/utils/orm_to_json_utils.py | 11 ++++------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b5e2a68629..b9e46d1a21 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -8,7 +8,6 @@ from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.search.add_data_utils import trigger_rebuild_gt_stats -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \ update_family_from_json from seqr.views.utils.orm_to_json_utils import _get_json_for_individuals, _get_json_for_families, get_json_for_family_notes @@ -88,9 +87,9 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected and not skip_gt_stats_rebuild: - backend_specific_call(lambda *args: True, trigger_rebuild_gt_stats)(project, user) + trigger_rebuild_gt_stats(project, user) if updated_sex: - backend_specific_call(lambda *args: True, SexDict.reload)() + SexDict.reload() pedigree_json = None if get_update_json: @@ -215,7 +214,7 @@ def check_project_individuals_deletable(project, individual_guids=None): if individual_guids is not None: individuals_to_delete = individuals_to_delete.filter(guid__in=individual_guids) - errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples)(individuals_to_delete) + errors = _validate_no_sumissions_no_search_samples(individuals_to_delete) return errors, individuals_to_delete @@ -227,15 +226,11 @@ def _validate_delete_individuals(individuals_to_delete, error_type, query): return errors -def _validate_no_submissions(individuals_to_delete): +def _validate_no_sumissions_no_search_samples(individuals_to_delete): return _validate_delete_individuals( individuals_to_delete, 'MME submission', dict(matchmakersubmission__isnull=False, matchmakersubmission__deleted_date__isnull=True) - ) - - -def _validate_no_sumissions_no_search_samples(individuals_to_delete): - return _validate_no_submissions(individuals_to_delete) + _validate_delete_individuals( + ) + _validate_delete_individuals( individuals_to_delete, 'search sample', dict(sample__is_active=True) ) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index a75940022c..59e6d23b88 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -422,11 +422,9 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo additional_fields = [] additional_fields += additional_model_fields or [] if add_details: - from seqr.utils.search.utils import backend_specific_call - additional_fields += ['saved_variant_json'] + backend_specific_call( - lambda x: [], - lambda gv: ['key', 'genotypes', 'dataset_type'] + ([] if gv else ['family__project__genome_version']), - )(genome_version) + additional_fields += ['saved_variant_json', 'key', 'genotypes', 'dataset_type'] + ( + [] if genome_version else ['family__project__genome_version'] + ) results = get_json_for_queryset( saved_variants, guid_key='variantGuid', additional_values=sv_additional_values, @@ -434,8 +432,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo ) if add_details: - from seqr.utils.search.utils import backend_specific_call - backend_specific_call(lambda *args: None, _add_clickhouse_annotations)(results, genome_version) + _add_clickhouse_annotations(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) From 74240d538b669fdef7346b2e2dbba2ee4d4b374b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 16:47:32 -0500 Subject: [PATCH 005/252] clean up conditionals --- seqr/views/utils/variant_utils.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index f1be32a486..67dbda59c9 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -15,8 +15,7 @@ from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.utils import backend_specific_call, variant_dataset_type +from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos @@ -132,13 +131,11 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use if new_variant_keys: if load_new_variant_data: genome_version = Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() - new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) + new_variant_data = _search_new_saved_variants(new_variant_keys, genome_version) else: new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} if load_new_variant_keys: - new_variant_data = backend_specific_call( - lambda o, _: o, _get_clickhouse_variant_keys, - )(new_variant_data, project.genome_version) + new_variant_data = _get_clickhouse_variant_keys(new_variant_data, project.genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): @@ -221,7 +218,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ return updated_tag -def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: User, genome_version: str) -> dict[tuple[int, str], dict]: +def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_version: str) -> dict[tuple[int, str], dict]: family_ids = set() variant_families = defaultdict(list) for family_id, variant_id in family_variant_ids: @@ -231,9 +228,9 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) search_variants_by_id = { - v['variantId']: v for v in backend_specific_call( - _get_es_variants, _get_clickhouse_variants, - )(samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) + v['variantId']: v for v in _get_clickhouse_variants( + samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), genome_version=genome_version, + ) } new_variants = {} @@ -256,11 +253,7 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U return new_variants -def _get_es_variants(samples: Sample.objects, families_by_id: dict[int, Family], *args, **kwargs): - return get_es_variants_for_variant_ids(samples.filter(individual__family_id__in=families_by_id.keys()), *args, **kwargs) - - -def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: +def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], variant_ids: list[str], genome_version: str = None) -> list[dict]: variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_ids, reverse=True) families = list(families_by_id.values()) prefetch_related_objects(families, 'project') @@ -351,7 +344,7 @@ def _get_variants_reference_data_response(variants, genome_versions, get_family_ for var in variant: for gene_id, transcripts in var.get('transcripts', {}).items(): gene_ids.add(gene_id) - if backend_specific_call(lambda v: True, _requires_transcript_metadata)(variant): + if _requires_transcript_metadata(variant): transcript_ids.update([t['transcriptId'] for t in transcripts if t.get('transcriptId')]) if get_family_genes: for family_guid in var.get('familyGuids', []): @@ -376,7 +369,7 @@ def _get_variants_reference_data_response(variants, genome_versions, get_family_ if any(genome_version == OMIM_GENOME_VERSION for genome_version in genome_versions): response['omimIntervals'] = _get_omim_intervals(variants) - backend_specific_call(lambda *args: None, _add_sample_count_stats)(response, genome_versions) + _add_sample_count_stats(response, genome_versions) return response From c41edeebf2aecb6c080aa8ca376578240158c5e6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Feb 2026 18:05:49 -0500 Subject: [PATCH 006/252] fix tests --- .../fixtures/clickhouse_saved_variants.json | 50 +++++++ seqr/views/apis/saved_variant_api_tests.py | 135 ++++++++---------- seqr/views/utils/test_utils.py | 65 ++++----- 3 files changed, 144 insertions(+), 106 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 347de13572..75f470d78b 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -110,6 +110,56 @@ [1, ["frameshift_variant"], "ENSG00000228198"] ] } +}, { + "model": "clickhouse_search.annotationsgrch37snvindel", + "pk": 113, + "fields": { + "key": 113, + "xpos": 15062456358, + "chrom": "15", + "pos": 62456358, + "ref": "A", + "alt": "C", + "variant_id": "15-62456358-A-C", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "hgmd": [null, null], + "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0] + ], + "sorted_transcript_consequences": [] + } +}, { + "model": "clickhouse_search.annotationsgrch37snvindel", + "pk": 114, + "fields": { + "key": 114, + "xpos": 15062456406, + "chrom": "15", + "pos": 62456406, + "ref": "G", + "alt": "A", + "variant_id": "15-62456406-G-A", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "hgmd": [null, null], + "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null], + "populations": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0] + ], + "sorted_transcript_consequences": [] + } }, { "model": "clickhouse_search.entriesgrch37snvindel", "pk": 101, diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 5a51ff12f5..7fee738b12 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -30,68 +30,62 @@ 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', } SAVED_VARIANT_RESPONSE_KEYS = { - *VARIANT_TAG_RESPONSE_KEYS, 'familiesByGuid', 'omimIntervals', + *VARIANT_TAG_RESPONSE_KEYS, 'familiesByGuid', 'omimIntervals', 'totalSampleCounts', 'genesById', 'locusListsByGuid', 'rnaSeqData', 'mmeSubmissionsByGuid', 'transcriptsById', 'phenotypeGeneScores', } COMPOUND_HET_3_JSON = { + 'key': 113, 'alt': 'C', 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', 'mainTranscriptId': None, - 'originalAltAlleles': ['C'], 'populations': {'callset': {'ac': 17, 'af': 0.607, 'an': 28}}, 'pos': 62456358, 'predictions': {'cadd': 12.34}, 'ref': 'A', 'transcripts': {}, 'xpos': 15062456358, - 'projectGuid': 'R0001_1kg', 'familyGuids': ['F000001_1'], 'variantId': '15-62456358-A-C', 'acmgClassification': None, + 'CAID': None, + 'rsid': None, + 'clinvar': None, + 'hgmd': None, + 'liftedOverGenomeVersion': '38', + 'liftedOverChrom': None, + 'liftedOverPos': None, } COMPOUND_HET_4_JSON = { + 'key': 114, 'alt': 'A', 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', 'mainTranscriptId': None, - 'originalAltAlleles': ['A'], 'populations': {'callset': {'ac': 1, 'af': 0.033, 'an': 8686}}, 'pos': 62456406, 'predictions': {'cadd': 13.56}, 'ref': 'G', 'transcripts': {}, 'xpos': 15062456406, - 'projectGuid': 'R0001_1kg', 'familyGuids': ['F000001_1'], 'variantId': '15-62456406-G-A', 'acmgClassification': None, -} - -COMPOUND_HET_5_JSON = { - 'alt': 'C', - 'chrom': '16', - 'genotypes': {}, - 'genomeVersion': '37', - 'mainTranscriptId': None, - 'originalAltAlleles': ['C'], - 'populations': {'callset': {'ac': 18, 'af': 0.563, 'an': 32}}, - 'pos': 31096164, - 'predictions': {'cadd': 7.099}, - 'ref': 'G', - 'transcripts': {}, - 'xpos': 16031096164, - 'projectGuid': 'R0001_1kg', - 'familyGuids': ['F000001_1'], - 'variantId': '16-31096164-G-C', - 'acmgClassification': None, + 'CAID': None, + 'rsid': None, + 'clinvar': None, + 'hgmd': None, + 'liftedOverGenomeVersion': '38', + 'liftedOverChrom': None, + 'liftedOverPos': None, } CREATE_VARIANT_JSON = { + 'key': 123, 'alt': 'A', 'chrom': '2', 'genotypes': {}, @@ -161,15 +155,15 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) variants = response_json['savedVariantsByGuid'] self.assertSetEqual(set(variants.keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID}) variant = variants[VARIANT_GUID] - self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), self.SAVED_VARIANT_DETAIL_FIELDS) - fields = {'mainTranscriptId', 'mmeSubmissions'} - fields.update(self.SAVED_VARIANT_DETAIL_FIELDS) + fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} + self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), fields) + fields.update({'mainTranscriptId', 'mmeSubmissions'}) self.assertSetEqual(set(variant.keys()), fields) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) @@ -221,7 +215,7 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() response_keys = {'projectsByGuid'} - response_keys.update(self.SAVED_VARIANT_RESPONSE_KEYS) + response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) project = response_json['projectsByGuid'][PROJECT_GUID] @@ -232,7 +226,7 @@ def test_saved_variant_data(self): response = self.client.get('{}?includeLocusLists=true'.format(url)) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) locus_list_fields.update(LOCUS_LIST_FIELDS) locus_list_fields.update(PA_LOCUS_LIST_FIELDS) @@ -247,7 +241,7 @@ def test_saved_variant_data(self): family_context_response_keys = { 'individualsByGuid', 'familyNotesByGuid', 'igvSamplesByGuid', 'projectsByGuid' } - family_context_response_keys.update(self.SAVED_VARIANT_RESPONSE_KEYS) + family_context_response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), family_context_response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2'}) @@ -266,7 +260,7 @@ def test_saved_variant_data(self): # get variants with no tags for whole project response = self.client.get('{}?includeNoteVariants=true'.format(url)) self.assertEqual(response.status_code, 200) - no_families_response_keys = {*self.SAVED_VARIANT_RESPONSE_KEYS} + no_families_response_keys = {*SAVED_VARIANT_RESPONSE_KEYS} no_families_response_keys.remove('familiesByGuid') no_families_response_keys.remove('transcriptsById') self.assertSetEqual(set(response.json().keys()), no_families_response_keys) @@ -322,7 +316,7 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) variants = response_json['savedVariantsByGuid'] self.assertSetEqual( set(variants.keys()), @@ -377,7 +371,7 @@ def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - create_variant_json = self._format_create_variant(CREATE_VARIANT_JSON) + create_variant_json = CREATE_VARIANT_JSON create_variant_request_body = { 'searchHash': 'd380ed0fd28c3127d07a64ea2ba907d7', 'familyGuid': 'F000001_1', @@ -433,14 +427,17 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _format_create_variant(self, variant_json, **kwargs): - return variant_json - - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, **kwargs): - for field in ['xpos', 'ref', 'alt']: + def _assert_created_basic_variant(self, saved_variant, variant_json, gene_ids): + for field in ['xpos', 'ref', 'alt', 'genotypes']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL'): + self._assert_created_basic_variant(saved_variant, variant_json, gene_ids) + self.assertEqual(variant_json['key'], saved_variant.key) + self.assertEqual(dataset_type, saved_variant.dataset_type) + self.assertDictEqual({}, saved_variant.saved_variant_json) + def test_create_saved_sv_variant(self): # SVs are only supported on build 38 Project.objects.filter(id=1).update(genome_version='38') @@ -448,7 +445,8 @@ def test_create_saved_sv_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - variant_json = self._format_create_variant({ + variant_json = { + 'key': 123, 'chrom': '2', 'genotypes': {}, 'genomeVersion': '38', @@ -466,7 +464,7 @@ def test_create_saved_sv_variant(self): 'svType': 'DUP', 'variantId': 'batch_123_DUP', 'acmgClassification': None, - }) + } request_body = { 'familyGuid': 'F000001_1', @@ -531,23 +529,26 @@ def test_create_saved_compound_hets(self): saved_compound_het_3 = SavedVariant.objects.get(guid=new_compound_het_3_guid, family__guid='F000001_1') saved_compound_het_4 = SavedVariant.objects.get(guid=new_compound_het_4_guid, family__guid='F000001_1') - self.assertDictEqual(COMPOUND_HET_3_JSON, saved_compound_het_3.saved_variant_json) - self.assertDictEqual(COMPOUND_HET_4_JSON, saved_compound_het_4.saved_variant_json) + self._assert_created_variant(saved_compound_het_3, COMPOUND_HET_3_JSON) + self._assert_created_variant(saved_compound_het_4, COMPOUND_HET_4_JSON) - expected_compound_het_3_json = { - 'variantGuid': new_compound_het_3_guid, + expected_json = { 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], + 'predictions': mock.ANY, + 'populations': mock.ANY, + } + expected_compound_het_3_json = { + 'variantGuid': new_compound_het_3_guid, + **COMPOUND_HET_3_JSON, + **expected_json, } - expected_compound_het_3_json.update(self._format_create_variant(COMPOUND_HET_3_JSON, key=None)) expected_compound_het_4_json = { 'variantGuid': new_compound_het_4_guid, - 'selectedMainTranscriptId': None, - 'noteGuids': [], - 'functionalDataGuids': [], + **COMPOUND_HET_4_JSON, + **expected_json, } - expected_compound_het_4_json.update(self._format_create_variant(COMPOUND_HET_4_JSON, key=None)) response_json = response.json() response_compound_het_3_json = response_json['savedVariantsByGuid'][new_compound_het_3_guid] response_compound_het_4_json = response_json['savedVariantsByGuid'][new_compound_het_4_guid] @@ -563,6 +564,10 @@ def test_create_saved_compound_hets(self): self.assertListEqual(["Review"], [vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__contains=new_compound_het_4_guid)]) + def test_create_manual_variant(self): + # TODO no longer have code coverage for creating variant with no key, add a real test using a representative manual variant body + self.assertEqual(1, 2) + def test_create_update_and_delete_variant_note(self): create_variant_note_url = reverse(create_variant_note_handler, args=[VARIANT_GUID]) self.check_collaborator_login(create_variant_note_url, request_data={'familyGuid': 'F000001_1'}) @@ -654,7 +659,7 @@ def test_create_partially_saved_compound_het_variant_note(self): self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) request_body = { - 'variant': [COMPOUND_HET_5_JSON, { + 'variant': [COMPOUND_HET_4_JSON, { 'variantId': '21-3343353-GAGA-G', 'xpos': 21003343353, 'ref': 'GAGA', 'alt': 'G', 'variantGuid': 'SV0000001_2103343353_r0390_100', 'tagGuids': ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'], 'noteGuids': []}, @@ -672,15 +677,17 @@ def test_create_partially_saved_compound_het_variant_note(self): compound_het_5_guid = compound_het_guids[0] saved_compound_het_5 = SavedVariant.objects.get(guid=compound_het_5_guid, family__guid='F000001_1') - self.assertDictEqual(COMPOUND_HET_5_JSON, saved_compound_het_5.saved_variant_json) + self._assert_created_variant(saved_compound_het_5, COMPOUND_HET_4_JSON) expected_compound_het_5_json = { + **COMPOUND_HET_4_JSON, 'variantGuid': compound_het_5_guid, 'selectedMainTranscriptId': None, 'tagGuids': [], 'functionalDataGuids': [], + 'predictions': mock.ANY, + 'populations': mock.ANY, } - expected_compound_het_5_json.update(self._format_create_variant(COMPOUND_HET_5_JSON, key=None)) response_json = response.json() response_compound_het_5_json = response_json['savedVariantsByGuid'][compound_het_5_guid] note_guids = response_compound_het_5_json.pop('noteGuids') @@ -1046,14 +1053,7 @@ def test_update_variant_acmg_classification(self): # Tests for AnVIL access disabled class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data'] - - SAVED_VARIANT_RESPONSE_KEYS = SAVED_VARIANT_RESPONSE_KEYS - SAVED_VARIANT_DETAIL_FIELDS = SAVED_VARIANT_DETAIL_FIELDS - - def _assert_created_variant(self, saved_variant, variant_json, **kwargs): - super()._assert_created_variant(saved_variant, variant_json, **kwargs) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] def assert_no_list_ws_has_al(self, acl_call_count): @@ -1068,9 +1068,6 @@ def assert_no_list_ws_has_al(self, acl_call_count): class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] - SAVED_VARIANT_RESPONSE_KEYS = {*SAVED_VARIANT_RESPONSE_KEYS, 'totalSampleCounts'} - SAVED_VARIANT_DETAIL_FIELDS = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} - @classmethod def setUpTestData(cls): super().setUpTestData() @@ -1144,13 +1141,3 @@ def test_update_variant_main_transcript(self): def test_update_variant_acmg_classification(self): super(AnvilSavedVariantAPITest, self).test_update_variant_acmg_classification() assert_no_list_ws_has_al(self, 2) - - def _format_create_variant(self, variant_json, key=123, **kwargs): - return {**variant_json, 'key': key} - - def _assert_created_variant(self, saved_variant, variant_json, dataset_type='SNV_INDEL', **kwargs): - super()._assert_created_variant(saved_variant, variant_json, **kwargs) - self.assertEqual(variant_json['key'], saved_variant.key) - self.assertEqual(dataset_type, saved_variant.dataset_type) - self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) - self.assertDictEqual({}, saved_variant.saved_variant_json) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index f2af38d493..6c0de7e544 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -19,6 +19,36 @@ WINDOW_REGEX_TEMPLATE = 'window\.{key}=(?P[^)<]+)' +class DifferentDbTransactionSupportMixin(object): + + @classmethod + def _databases_support_transactions(cls): + return True + + @classmethod + def _rollback_atomics(cls, atomics): + """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" + for db_name in reversed(cls._databases_names()): + if connections[db_name].features.supports_transactions: + transaction.set_rollback(True, using=db_name) + atomics[db_name].__exit__(None, None, None) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + for db_name in cls._databases_names(): + if not connections[db_name].features.supports_transactions: + call_command( + "flush", + verbosity=0, + interactive=False, + database=db_name, + reset_sequences=False, + allow_cascade=False, + inhibit_post_migrate=False, + ) + + class AuthenticationTestMixin(object): SUPERUSER = 'superuser' ANALYST = 'analyst' @@ -271,8 +301,8 @@ def assert_json_logs(self, user, expected, offset=0): def assert_no_logs(self): self.assertEqual(self._log_stream.getvalue(), '') -class AuthenticationTestCase(AuthenticationTestMixin, TestCase): - databases = ['default', 'reference_data'] +class AuthenticationTestCase(DifferentDbTransactionSupportMixin, AuthenticationTestMixin, TestCase): + databases = '__all__' def setUp(self): self.set_up_test() @@ -520,36 +550,6 @@ def get_group_members_side_effect(user, group, use_sa_credentials=False): return {} -class DifferentDbTransactionSupportMixin(object): - - @classmethod - def _databases_support_transactions(cls): - return True - - @classmethod - def _rollback_atomics(cls, atomics): - """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" - for db_name in reversed(cls._databases_names()): - if connections[db_name].features.supports_transactions: - transaction.set_rollback(True, using=db_name) - atomics[db_name].__exit__(None, None, None) - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - for db_name in cls._databases_names(): - if not connections[db_name].features.supports_transactions: - call_command( - "flush", - verbosity=0, - interactive=False, - database=db_name, - reset_sequences=False, - allow_cascade=False, - inhibit_post_migrate=False, - ) - - class AnvilAuthenticationTestMixin(AuthenticationTestMixin): ES_HOSTNAME = '' @@ -599,6 +599,7 @@ def set_up_test(self): self.addCleanup(patcher.stop) super().set_up_test() if self.CLICKHOUSE_HOSTNAME and not self.SKIP_RESET_VARIANT_JSON: + # TODO update fixture data to match SavedVariant.objects.filter(key__isnull=False).update(saved_variant_json={}) @classmethod From 68cfcc5df2d039c7ab6b855eb90659cd6d9e3a0b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 14:17:15 -0500 Subject: [PATCH 007/252] test manual variants --- seqr/views/apis/saved_variant_api_tests.py | 100 ++++++++++++++++++++- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7fee738b12..9b29de4a34 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -428,13 +428,13 @@ def test_create_saved_variant(self): self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) def _assert_created_basic_variant(self, saved_variant, variant_json, gene_ids): - for field in ['xpos', 'ref', 'alt', 'genotypes']: + for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL'): self._assert_created_basic_variant(saved_variant, variant_json, gene_ids) - self.assertEqual(variant_json['key'], saved_variant.key) + self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) self.assertEqual(dataset_type, saved_variant.dataset_type) self.assertDictEqual({}, saved_variant.saved_variant_json) @@ -565,8 +565,100 @@ def test_create_saved_compound_hets(self): saved_variants__guid__contains=new_compound_het_4_guid)]) def test_create_manual_variant(self): - # TODO no longer have code coverage for creating variant with no key, add a real test using a representative manual variant body - self.assertEqual(1, 2) + create_saved_variant_url = reverse(create_saved_variant_handler) + self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) + + manual_variant_request_body = { + 'familyGuid': 'F000001_1', + 'tags': [{'name': 'Review'}], + 'variant': [{ + 'alt': 'A', + 'ref': 'GAG', + 'chrom': '7', + 'pos': 53417465, + 'genomeVersion': '37', + 'variantId': '7-53417465-GAG-A', + 'mainTranscriptId': 'ENST00000459627', + 'transcripts': { + 'ENSG00000277258': [{'transcriptId': 'ENST00000459627', 'hgvsc':'c.156GAG>A', 'hgvsp': 'p.Leu52Phe'}], + }, + 'genotypes': { + 'I000001_na19675': {'numAlt': 1}, + 'I000002_na19678': {'numAlt': 0}, + 'I000003_na19679': {'numAlt': 0}, + }, + }], + } + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 200) + + self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) + variant_guid = next(iter(response.json()['savedVariantsByGuid'])) + + saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') + variant_json = {'xpos': 7053417465} + variant_json.update(manual_variant_request_body['variant'][0]) + self._assert_created_basic_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258']) + self.assertDictEqual(variant_json, saved_variant.saved_variant_json) + self.assertDictEqual({}, saved_variant.genotypes) + + base_variant_json = { + 'variantGuid': variant_guid, + 'familyGuids': ['F000001_1'], + 'acmgClassification': None, + 'selectedMainTranscriptId': None, + 'key': None, + 'noteGuids': [], + 'functionalDataGuids': [], + } + variant_json.update(base_variant_json) + response_json = response.json() + response_variant_json = response_json['savedVariantsByGuid'][variant_guid] + tag_guids = response_variant_json.pop('tagGuids') + self.assertEqual(len(tag_guids), 1) + tag = response_json['variantTagsByGuid'][tag_guids[0]] + self.assertEqual('Review', tag['name']) + self.assertDictEqual(variant_json, response_variant_json) + + # Create manual SV + manual_variant_request_body['variant'] = [{ + 'chrom': '7', + 'pos': 10417443, + 'end': 11719832, + 'genomeVersion': '37', + 'variantId': 'Deletion:chr7:(?_10417443)_(11719832_?)', + 'svName': 'Deletion:chr7:(?_10417443)_(11719832_?)', + 'svType': 'DEL', + 'transcripts': {'ENSG00000240361': []}, + 'genotypes': { + 'I000001_na19675': {'cn': 1}, + 'I000002_na19678': {'cn': 2}, + 'I000003_na19679': {'cn': 2}, + }, + }] + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 200) + + self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) + sv_variant_guid = next(iter(response.json()['savedVariantsByGuid'])) + + saved_sv_variant = SavedVariant.objects.get(guid=sv_variant_guid, family__guid='F000001_1') + sv_variant_json = {'xpos': 7010417443} + sv_variant_json.update(manual_variant_request_body['variant'][0]) + self._assert_created_basic_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361']) + self.assertDictEqual(sv_variant_json, saved_sv_variant.saved_variant_json) + self.assertDictEqual({}, saved_sv_variant.genotypes) + + sv_variant_json.update(base_variant_json) + sv_variant_json.update({ + 'variantGuid': sv_variant_guid, + 'tagGuids': [mock.ANY], + 'ref': None, + 'alt': None, + }) + response_json = response.json() + response_variant_json = response_json['savedVariantsByGuid'][sv_variant_guid] + self.assertDictEqual(sv_variant_json, response_variant_json) def test_create_update_and_delete_variant_note(self): create_variant_note_url = reverse(create_variant_note_handler, args=[VARIANT_GUID]) From f9d287a453ebde87aab771ecf21341272e4d63e4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 14:26:38 -0500 Subject: [PATCH 008/252] more clean up --- seqr/views/apis/family_api_tests.py | 8 ++------ seqr/views/react_app.py | 1 - seqr/views/utils/project_context_utils.py | 13 ++++--------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index fdf1045bb8..3dd6e69d91 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -16,7 +16,7 @@ FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \ SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS -from seqr.models import FamilyAnalysedBy, AnalysisGroup +from seqr.models import FamilyAnalysedBy, AnalysisGroup, Sample FAMILY_GUID = 'F000001_1' FAMILY_GUID2 = 'F000002_2' @@ -312,11 +312,6 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 400) - self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active MME submission: NA19675_1']) - - with mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', ''): - response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) - self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to delete individuals with active MME submission: NA19675_1', 'Unable to delete individuals with active search sample: HG00731, HG00732, HG00733, NA19675_1, NA19678', @@ -324,6 +319,7 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): # Test success MatchmakerSubmission.objects.update(deleted_date=datetime.now()) + Sample.objects.update(is_active=False) response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 200) diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 82480db004..0b40591a39 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -15,7 +15,6 @@ VLM_CLIENT_ID, ) from seqr.models import WarningMessage -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import get_json_for_user, get_json_for_current_user from seqr.views.utils.permissions_utils import login_active_required diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 8e96d02f1a..727b872a8a 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -8,7 +8,6 @@ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample from seqr.utils.gene_utils import get_genes from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ get_json_for_analysis_groups, get_json_for_samples, get_json_for_locus_lists, \ get_json_for_family_notes @@ -129,10 +128,10 @@ def families_discovery_tags(families, genome_version, project=None): Q(gene_ids__len=1) | (Q(selected_main_transcript_id__isnull=True) & ~Q(dataset_type__startswith='SV')) ).values_list('family_guid', F('gene_ids__0'))) try: - backend_specific_call(_get_no_key_selected_transcript_gene_id, _get_clickhouse_selected_transcript_gene_id)( + _get_clickhouse_selected_transcript_gene_id( family_discovery_genes, discovery_variants.filter(selected_main_transcript_id__isnull=False, gene_ids__len__gt=1), - genome_version=genome_version, + genome_version, ) except Exception as e: logger.error(f'Error loading discovery genes from clickhouse: {e}', None) @@ -155,18 +154,14 @@ def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): ), None) -def _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants, **kwargs): +def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): family_discovery_genes += [ (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts)) - for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values_list( + for family_guid, selected_main_transcript_id, transcripts in discovery_variants.filter(key__isnull=True).values_list( 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts', ) ] - -def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): - _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants.filter(key__isnull=True)) - tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( keys=ArrayAgg('key', distinct=True), variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')), From 47487dd862baeb484050e9aee71793dc6507732f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 14:35:51 -0500 Subject: [PATCH 009/252] more clean up --- seqr/views/apis/summary_data_api_tests.py | 45 +++++++++-------------- seqr/views/utils/anvil_metadata_utils.py | 30 +++++---------- 2 files changed, 26 insertions(+), 49 deletions(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index f8048a3966..97d5506b68 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -629,6 +629,22 @@ def _assert_expected_new_saved_variant(self, new_saved_variant): self.assertEqual(new_saved_variant.ref, 'GAGA') self.assertEqual(new_saved_variant.alt, 'G') self.assertListEqual(new_saved_variant.gene_ids, ['ENSG00000135953', 'ENSG00000228198']) + self.assertEqual(new_saved_variant.key, 101) + self.assertEqual(new_saved_variant.dataset_type, 'SNV_INDEL') + self.assertDictEqual(new_saved_variant.genotypes, { + 'I000004_hg00731': { + 'ab': 0, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000004_hg00731', 'sampleType': 'WES', + }, + 'I000005_hg00732': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000005_hg00732', 'sampleType': 'WES', + }, + 'I000006_hg00733': { + 'ab': 0, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'filters': [], + 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', + }, + }) def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False): self.assertEqual(response.status_code, 200) @@ -828,7 +844,7 @@ def _assert_expected_vlm_email(self, response, mock_email, additional_logs=None) # Tests for AnVIL access disabled class LocalSummaryDataAPITest(AuthenticationTestCase, SummaryDataAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = ['NA21234', 'NA21987', 'NA21654'] HAS_AIRTABLE = False @@ -840,14 +856,6 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assertEqual(response.status_code, 200) self._has_expected_metadata_response(response, expected_individuals) - @mock.patch('seqr.views.utils.variant_utils.get_es_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) - def test_bulk_update_family_external_analysis(self, *args, **kwargs): - super().test_bulk_update_family_external_analysis(*args, **kwargs) - - def _assert_expected_new_saved_variant(self, new_saved_variant): - super()._assert_expected_new_saved_variant(new_saved_variant) - self.assertDictEqual(new_saved_variant.saved_variant_json, PARSED_VARIANTS[1]) - def assert_has_expected_calls(self, users, skip_group_call_idxs=None): calls = [mock.call(user) for user in users] @@ -878,22 +886,3 @@ def test_saved_variants_page(self): ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') - - def _assert_expected_new_saved_variant(self, new_saved_variant): - super()._assert_expected_new_saved_variant(new_saved_variant) - self.assertEqual(new_saved_variant.key, 101) - self.assertEqual(new_saved_variant.dataset_type, 'SNV_INDEL') - self.assertDictEqual(new_saved_variant.genotypes, { - 'I000004_hg00731': { - 'ab': 0, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000004_hg00731', 'sampleType': 'WES', - }, - 'I000005_hg00732': { - 'ab': 0, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000005_hg00732', 'sampleType': 'WES', - }, - 'I000006_hg00733': { - 'ab': 0, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'filters': [], - 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', - }, - }) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index d8e464ec44..48f85865a2 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -16,7 +16,7 @@ from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples, backend_specific_call +from seqr.utils.search.utils import get_search_samples from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY @@ -345,9 +345,7 @@ def _get_parsed_saved_discovery_variants_by_family( variant_json_fields.append('clinvar') variants = [] gene_ids = set() - variant_json_by_guid = backend_specific_call( - _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, - )(project_saved_variants, include_clinvar) + variant_json_by_guid = _get_variant_json_by_guid(project_saved_variants, include_clinvar) for variant in project_saved_variants: chrom, pos = get_chrom_pos(variant.xpos) @@ -405,31 +403,21 @@ def _get_parsed_saved_discovery_variants_by_family( return saved_variants_by_family -def _get_variant_json_by_guid(saved_variants, *args, **kwargs): - variant_json_by_guid = {} - for v in saved_variants: - main_transcript = _get_variant_main_transcript(v) - gene_id = main_transcript.get('geneId') - gene_ids = [gene_id] if gene_id else v.gene_ids - variant_json_by_guid[v.guid] = { - **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, - } - - return variant_json_by_guid - - -def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar): +def _get_variant_json_by_guid(saved_variants, include_clinvar): variant_json_by_guid = {} variant_keys_by_search_type = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - no_key_variants = [] for v in saved_variants: if v.key: end_chrom, end = get_chrom_pos(v.xpos_end) variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids} variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id)) else: - no_key_variants.append(v) - variant_json_by_guid.update(_get_variant_json_by_guid(no_key_variants)) + main_transcript = _get_variant_main_transcript(v) + gene_id = main_transcript.get('geneId') + gene_ids = [gene_id] if gene_id else v.gene_ids + variant_json_by_guid[v.guid] = { + **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, + } for genome_version, keys_by_dataset_type in variant_keys_by_search_type.items(): for dataset_type, key_map in keys_by_dataset_type.items(): From d47439bf21cd5c59c2e6bb51b15c7ca4d1399c49 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 14:48:15 -0500 Subject: [PATCH 010/252] status clean up --- seqr/utils/search/utils.py | 13 ++------- seqr/views/status.py | 16 ----------- seqr/views/status_tests.py | 54 ++++++-------------------------------- 3 files changed, 10 insertions(+), 73 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f10eb0ec09..ac4baf4567 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -12,9 +12,9 @@ from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS -from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ +from seqr.utils.search.elasticsearch.es_utils import \ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ - es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS + es_backend_enabled, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -75,15 +75,6 @@ def backend_specific_call(es_func, clickhouse_func): return clickhouse_func -def ping_search_backend(): - # Clickhouse backend does not need special uptime testing, will be checked with the other database connection pings - backend_specific_call(ping_elasticsearch, lambda: None)() - - -def ping_search_backend_admin(): - backend_specific_call(ping_kibana, lambda: True)() - - def _get_filtered_search_samples(search_filter, active_only=True): samples = Sample.objects.filter(**search_filter) if active_only: diff --git a/seqr/views/status.py b/seqr/views/status.py index d0e6502a1e..36a1244bf6 100644 --- a/seqr/views/status.py +++ b/seqr/views/status.py @@ -3,7 +3,6 @@ import redis from settings import SEQR_VERSION, REDIS_SERVICE_HOSTNAME, REDIS_SERVICE_PORT, DATABASES -from seqr.utils.search.utils import ping_search_backend, ping_search_backend_admin from seqr.views.utils.json_utils import create_json_response logger = logging.getLogger(__name__) @@ -30,21 +29,6 @@ def status_view(request): secondary_services_ok = False logger.error('Redis connection error: {}'.format(str(e))) - # Test search backend connection - try: - ping_search_backend() - except Exception as e: - dependent_services_ok = False - logger.error('Search backend connection error: {}'.format(str(e))) - - # Test search admin view connection - try: - ping_search_backend_admin() - except Exception as e: - secondary_services_ok = False - logger.error('Search Admin connection error: {}'.format(str(e))) - - return create_json_response( {'version': SEQR_VERSION, 'dependent_services_ok': dependent_services_ok, 'secondary_services_ok': secondary_services_ok}, status= 200 if dependent_services_ok else 400 diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index f2cbc40e18..7ef6af7c3f 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -5,23 +5,13 @@ from requests import HTTPError from seqr.views.status import status_view -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses @mock.patch('seqr.views.status.redis.StrictRedis') @mock.patch('seqr.views.status.logger') -class ElasticsearchStatusTest(TestCase): +class StatusTest(TestCase): databases = '__all__' - SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' - HAS_KIBANA = True - ES_HOSTNAME = 'testhost' - - def setUp(self): - patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) - patcher.start() - self.addCleanup(patcher.stop) - def _post_teardown(self): for conn in connections.all(): conn.connection = None @@ -30,13 +20,11 @@ def _post_teardown(self): conn.connection.pool.closed = False conn.connection.is_closed = False - @urllib3_responses.activate def test_status_error(self, mock_logger, mock_redis): for conn in connections.all(): conn.ensure_connection() conn.connection.close() mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) url = reverse(status_view) response = self.client.get(url) @@ -50,49 +38,23 @@ def test_status_error(self, mock_logger, mock_redis): mock.call('Database "clickhouse" connection error: connection already closed'), mock.call('Redis connection error: Bad connection'), ] - if self.SEARCH_BACKEND_ERROR: - calls.append(mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}')) - if self.HAS_KIBANA: - calls.append(mock.call('Search Admin connection error: Kibana Error 400: Bad Request')) mock_logger.error.assert_has_calls(calls) - @urllib3_responses.activate def test_status(self, mock_logger, mock_redis): url = reverse(status_view) - mock_redis.return_value.ping.side_effect = None - urllib3_responses.add(urllib3_responses.HEAD, '/', status=200) - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=500 if self.HAS_KIBANA else 200) - + mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') response = self.client.get(url) self.assertEqual(response.status_code, 200) - if self.HAS_KIBANA: - self.assertDictEqual( - response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': False}) - mock_logger.error.assert_has_calls([ - mock.call('Search Admin connection error: Kibana Error 500: Internal Server Error'), - ]) - - mock_logger.reset_mock() - urllib3_responses.replace_json('/status', {'success': True}, method=urllib3_responses.HEAD, status=200) + self.assertDictEqual( + response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': False}) + mock_logger.error.assert_called_with('Redis connection error: Bad connection') - response = self.client.get(url) + mock_logger.reset_mock() + mock_redis.return_value.ping.side_effect = None + response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertDictEqual( response.json(), {'version': 'v1.0', 'dependent_services_ok': True, 'secondary_services_ok': True}) mock_logger.error.assert_not_called() - self._assert_expected_requests() - - def _assert_expected_requests(self): - self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) - - -class ClickhouseStatusTest(ElasticsearchStatusTest): - - SEARCH_BACKEND_ERROR = None - HAS_KIBANA = False - ES_HOSTNAME = '' - - def _assert_expected_requests(self): - self.assertEqual(len(urllib3_responses.calls), 0) From 82e4699fcb14aa7df6d4cb278fe731b39f796117 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 14:59:17 -0500 Subject: [PATCH 011/252] single variant clean up --- seqr/utils/search/utils.py | 18 +++--------------- seqr/views/apis/variant_search_api.py | 2 +- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f10eb0ec09..dc0d242789 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -133,26 +133,14 @@ def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_ids_dataset_type([parsed_variant_id]) samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) - variant = backend_specific_call( - _get_es_variant_by_id, - _get_clickhouse_variant_by_id, - )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type=dataset_type, user=user) + variant = get_clickhouse_variant_by_id( + variant_id, parsed_variant_id, samples, family.project.genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], + ) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return variant -def _get_es_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, user=None, **kwargs): - variants = get_es_variants_for_variant_ids(samples, genome_version, [variant_id], user) - return variants[0] if variants else None - - -def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type=None, **kwargs): - return get_clickhouse_variant_by_id( - variant_id, parsed_variant_id, samples, genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], - ) - - @clickhouse_only def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): cache_fields = ['variant_lookup_results', variant_id, genome_version] diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ed322793fb..23fcd16bf7 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -143,7 +143,7 @@ def query_single_variant_handler(request, variant_id): family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(family, variant_id, user=request.user) + variant = get_single_variant(family, variant_id) response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) From e2ce13e3c1439e0419e27f63b5bc5a0dda503092 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 15:04:31 -0500 Subject: [PATCH 012/252] backend specific clean up --- seqr/utils/search/utils.py | 39 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index dc0d242789..189c703a17 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -282,12 +282,10 @@ def _query_variants(search_model, user, previous_search_results, genome_version, raise InvalidSearchException('Too many genes/intervals') parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) if not (genes or intervals): - rs_ids, variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) + variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) if invalid_items: raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) - if rs_ids and variant_ids: - raise InvalidSearchException('Invalid variant notation: found both variant IDs and rsIDs') - parsed_search.update({'rs_ids': rs_ids, 'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) + parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) if variant_ids: num_results = len(variant_ids) @@ -301,9 +299,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, exclude.pop('previousSearch', None) exclude_previous_hash = exclude.pop('previousSearchHash', None) if exclude_previous_hash: - parsed_search.update(backend_specific_call( - lambda *args: {}, _get_clickhouse_exclude_keys, - )(exclude_previous_hash, user, genome_version)) + parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user, genome_version)) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): @@ -418,25 +414,21 @@ def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): def _parse_variant_items(search_json): raw_items = search_json.get('rawVariantItems') if not raw_items: - return None, None, None, None + return None, None, None invalid_items = [] variant_ids = [] parsed_variant_ids = [] - rs_ids = [] for item in raw_items.replace(',', ' ').split(): - if item.startswith('rs') and backend_specific_call(True, False): - rs_ids.append(item) + variant_id = item.lstrip('chr') + parsed_variant_id = parse_variant_id(variant_id) + if parsed_variant_id: + parsed_variant_ids.append(parsed_variant_id) + variant_ids.append(variant_id) else: - variant_id = item.lstrip('chr') - parsed_variant_id = parse_variant_id(variant_id) - if parsed_variant_id: - parsed_variant_ids.append(parsed_variant_id) - variant_ids.append(variant_id) - else: - invalid_items.append(item) + invalid_items.append(item) - return rs_ids, variant_ids, parsed_variant_ids, invalid_items + return variant_ids, parsed_variant_ids, invalid_items def parse_variant_id(variant_id): @@ -560,10 +552,6 @@ def _validate_search(search, samples, previous_search_results): if search.get('no_access_project_genome_version'): raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - backend_specific_call(lambda *args: None, _validate_clickhouse_search)(samples, has_location_filter, search) - - -def _validate_clickhouse_search(samples, has_location_filter, search): variant_samples = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) if not has_location_filter and variant_samples.values('individual__family__project_id').distinct().count() > 1: raise InvalidSearchException('Location must be specified to search across multiple projects') @@ -576,7 +564,6 @@ def _validate_clickhouse_search(samples, has_location_filter, search): def _filter_inheritance_family_samples(samples, inheritance_filter): family_groups = defaultdict(set) - sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type') individual_affected_status = inheritance_filter.get('affected') or {} genotype_filter = None if inheritance_filter.get(Individual.AFFECTED_STATUS_AFFECTED) else inheritance_filter.get('genotype') for sample in samples: @@ -587,7 +574,7 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): is_filtered_family = affected_status == Individual.AFFECTED_STATUS_AFFECTED if is_filtered_family: - family_groups[sample.individual.family_id].add(getattr(sample, sample_group_field)) + family_groups[sample.individual.family_id].add(sample.dataset_type) if not family_groups: raise InvalidSearchException( @@ -596,7 +583,7 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ) return [ - s for s in samples if getattr(s, sample_group_field) not in family_groups[s.individual.family_id] + s for s in samples if s.dataset_type not in family_groups[s.individual.family_id] ] From ab85fabe9ba54e55d6298d2e37d6f531fc4c35b0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 15:10:27 -0500 Subject: [PATCH 013/252] gene count clean up --- seqr/utils/search/utils.py | 43 +++++--------------------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 189c703a17..62304b476f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -13,7 +13,7 @@ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ - get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ + get_es_variants, process_es_previously_loaded_results, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -360,50 +360,17 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - return backend_specific_call( - _get_es_variant_query_gene_counts, - _get_clickhouse_variant_query_gene_counts, - )(search_model, user) - - -def _get_es_variant_query_gene_counts(search_model, user): - previous_search_results = _get_cached_search_results(search_model) - if previous_search_results.get('gene_aggs'): - return previous_search_results['gene_aggs'] - - if len(previous_search_results.get('all_results', [])) == previous_search_results.get('total_results'): - return _get_gene_aggs_for_cached_variants( - previous_search_results['all_results'], - lambda v: next(( - [gene_id] for gene_id, transcripts in v['transcripts'].items() - if any(t['transcriptId'] == v['mainTranscriptId'] for t in transcripts) - ), []) if v['mainTranscriptId'] else [], - ) - - previously_loaded_results = process_es_previously_loaded_gene_aggs(previous_search_results) - if previously_loaded_results is not None: - return previously_loaded_results - - genome_version = _get_search_genome_version(search_model) - gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) - return gene_counts - - -def _get_clickhouse_variant_query_gene_counts(search_model, user): previous_search_results = _get_any_sort_cached_results(search_model) or {} if len(previous_search_results.get('all_results', [])) != previous_search_results.get('total_results'): genome_version = _get_search_genome_version(search_model) _query_variants(search_model, user, previous_search_results, genome_version) - return _get_gene_aggs_for_cached_variants([ + flat_variants = [ v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) - ], lambda v: v['transcripts'].keys() if 'transcripts' in v else {t['geneId'] for t in v['sortedTranscriptConsequences']}) - - -def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): + ] gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - for var in variants: - gene_ids = get_variant_genes(var) + for var in flat_variants: + gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} for gene_id in gene_ids: gene_aggs[gene_id]['total'] += 1 for family_guid in var['familyGuids']: From 74b62ec390ea455ba633bd7caf9a4c6fdd8397e0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 15:18:37 -0500 Subject: [PATCH 014/252] cache clean up --- seqr/utils/search/utils.py | 52 ++++++++++---------------------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 62304b476f..6334c3c479 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -13,7 +13,7 @@ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ - get_es_variants, process_es_previously_loaded_results, \ + get_es_variants, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -179,33 +179,14 @@ def _get_any_sort_cached_results(search_model): return safe_redis_get_wildcard_json(cache_key) -def _get_cached_search_results(search_model, sort=None): - cache_key = _get_search_cache_key(search_model, sort=sort) - return safe_redis_get_json(cache_key) or {} - - def _validate_export_variant_count(total_variants): if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') -def _get_elasticsearch_previous_search_results(search_model, sort, page, num_results, load_all, **kwargs): - previous_search_results = _get_cached_search_results(search_model, sort=sort) - start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) - - cached_page = None - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - cached_page = loaded_results[start_index:end_index] - - if not cached_page: - cached_page = process_es_previously_loaded_results(previous_search_results, start_index, end_index) - - return previous_search_results, cached_page, num_results - - -def _get_clickhouse_previous_search_results(search_model, sort, page, num_results, load_all, genome_version=None): - previous_search_results = _get_cached_search_results(search_model, sort=sort) +def _get_previous_search_results(search_model, sort, page, num_results, load_all, genome_version): + cache_key = _get_search_cache_key(search_model, sort=sort) + previous_search_results = safe_redis_get_json(cache_key) or {} if not previous_search_results: unsorted_results = _get_any_sort_cached_results(search_model) if unsorted_results: @@ -215,16 +196,7 @@ def _get_clickhouse_previous_search_results(search_model, sort, page, num_result cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) - cached_page = None - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) - - return previous_search_results, cached_page, num_results - - -def _get_result_range(page, num_results, total_results, load_all): + total_results = previous_search_results.get('total_results') if load_all: num_results = total_results or MAX_EXPORT_VARIANTS _validate_export_variant_count(num_results) @@ -236,15 +208,19 @@ def _get_result_range(page, num_results, total_results, load_all): if end_index > MAX_VARIANTS: raise InvalidSearchException(f'Unable to load more than {MAX_VARIANTS} variants ({end_index} requested)') - return start_index, end_index, num_results + cached_page = None + loaded_results = previous_search_results.get('all_results') or [] + if len(loaded_results) >= end_index: + cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) + + return previous_search_results, cached_page, num_results def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): genome_version = _get_search_genome_version(search_model) - previous_search_results, cached_page, num_results = backend_specific_call( - _get_elasticsearch_previous_search_results, - _get_clickhouse_previous_search_results, - )(search_model, sort, page, num_results, load_all, genome_version=genome_version) + previous_search_results, cached_page, num_results = _get_previous_search_results( + search_model, sort, page, num_results, load_all, genome_version, + ) if cached_page is not None: return cached_page, previous_search_results.get('total_results') From a327bae8d39d81534db22b19dce76153a646430f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 15:22:50 -0500 Subject: [PATCH 015/252] search clean up --- seqr/utils/search/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 6334c3c479..7b6fa97fef 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -13,7 +13,6 @@ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ - get_es_variants, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -234,7 +233,7 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, return variants, total_results -def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, **kwargs): +def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, page=1, **kwargs): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() @@ -296,7 +295,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, _validate_search(parsed_search, samples, previous_search_results) - variant_results = backend_specific_call(get_es_variants, get_clickhouse_variants)( + variant_results = get_clickhouse_variants( samples, parsed_search, user, previous_search_results, genome_version, sort=sort, num_results=num_results, **kwargs, ) From a2dd91d5e046e75279f63c3f19e2fec40e536b5f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:12:41 -0500 Subject: [PATCH 016/252] remove es util tests --- .../search/elasticsearch/es_utils_tests.py | 3 +- seqr/utils/search/search_utils_tests.py | 280 +++++------------- seqr/utils/search/utils.py | 2 +- 3 files changed, 70 insertions(+), 215 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index a4bfc081ef..85651ec04f 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -12,8 +12,9 @@ from urllib3_mock import Responses from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults +from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, get_es_variants_for_variant_ids, InvalidSearchException + get_variant_query_gene_counts, InvalidSearchException from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException from seqr.utils.search.utils import _get_liftover diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 42703de4ee..fd59adb7c5 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -48,13 +48,15 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) -class SearchUtilsTests(SearchTestHelper): +class SearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchTestHelper): + databases = '__all__' + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] - CACHED_VARIANTS = PARSED_VARIANTS + PARSED_VARIANTS - HAS_GENE_AGG = False + PARSED_CACHED_VARIANTS = [VARIANT1, VARIANT2, VARIANT3, VARIANT4] + CACHED_VARIANTS = [format_cached_variant(v) for v in PARSED_CACHED_VARIANTS] - def set_up(self): - super(SearchUtilsTests, self).set_up() + def setUp(self): + self.set_up() self.non_affected_search_samples = Sample.objects.filter(guid__in=[ 'S000137_na20874', @@ -65,6 +67,7 @@ def set_up(self): ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) + @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') def test_variant_lookup(self, mock_variant_lookup): mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] variants = variant_lookup(self.user, '1-10439-AC-A', '38', affected_only=True) @@ -105,7 +108,9 @@ def test_variant_lookup(self, mock_variant_lookup): mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) + @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') def test_get_single_variant(self, mock_get_variants_for_ids): + mock_get_variants_for_ids.return_value = PARSED_VARIANTS[0] family = self.families.filter(guid='F000002_2').first() variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) self.assertDictEqual(variant, PARSED_VARIANTS[0]) @@ -127,11 +132,9 @@ def test_get_single_variant(self, mock_get_variants_for_ids): get_single_variant(family, '10-10334333-A-G') self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, variant_id, parsed_variant_id, expected_samples, **kwargs): - mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', [variant_id], self.user, - ) - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) + def _assert_expected_get_single_variant_call(self, mock_call, variant_id, parsed_variant_id, expected_samples, dataset_type='SNV_INDEL'): + mock_call.assert_called_with(variant_id, parsed_variant_id, mock.ANY, '37', dataset_type) + self.assertSetEqual(set(mock_call.call_args.args[2]), expected_samples) @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @@ -148,7 +151,7 @@ def _test_invalid_search_params(self, search_func): self.search_model.search['locus']['rawVariantItems'] = 'rs9876,chr2-1234-A-C' with self.assertRaises(InvalidSearchException) as cm: search_func(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), self.INVALID_RSID_ERROR) + self.assertEqual(str(cm.exception), 'Invalid variants: rs9876') self.search_model.search['locus']['rawItems'] = 'chr27:1234-5678,2:40-400000000, ENSG00012345' with self.assertRaises(InvalidSearchException) as cm: @@ -285,7 +288,7 @@ def test_invalid_search_query_variants(self): self._test_invalid_search_params(query_variants) def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, - rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', + variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', dataset_type=None, secondary_dataset_type=None, exclude_keys=None, exclude_key_pairs=None, exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): genes = intervals = None @@ -310,7 +313,6 @@ def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_ if not genes: expected_search.update({ 'parsed_variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, 'variant_ids': variant_ids, }) expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) @@ -352,7 +354,7 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid self.assertSetEqual(set(mock_get_variants.call_args.args[0]), set(searched_samples)) self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - + @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') def test_query_variants(self, mock_get_variants): parsed_variants = [{**v, 'key': (i+1) * 1000 } for i, v in enumerate(PARSED_VARIANTS)] def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): @@ -464,10 +466,30 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v previous_results_model = VariantSearchResults.objects.create(variant_search=previous_search_model, search_hash='abc1234') previous_results_model.families.set(self.families) query_variants(self.results_model, user=self.user) - self._test_exclude_previous_search( + self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], + exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, ) + self.assertEqual(mock_get_variants.call_count, 2) + call_args, call_kwargs, _ = self._get_expected_search_call(results_cache, inheritance_mode='de_novo', sort=None, num_results=100) + mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)]) + + # Test when previous results are cached + mock_get_variants.reset_mock() + self.mock_redis.get.side_effect = [ + None, + json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), + ] + self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] + query_variants(self.results_model, user=self.user) + self._test_expected_search_call( + mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, + inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], + exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]}, + exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]}, + ) + self.assertEqual(mock_get_variants.call_count, 1) del self.search_model.search['exclude'] self.search_model.search['exclude_svs'] = True @@ -490,7 +512,7 @@ def _test_locus_query_variants(self, mock_get_variants, results_cache): query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=2, skip_genotype_filter=False, - rs_ids=[], variant_ids=['1-248367227-TC-T', '2-103343353-GAGA-G'], + variant_ids=['1-248367227-TC-T', '2-103343353-GAGA-G'], parsed_variant_ids=[('1', 248367227, 'TC', 'T'), ('2', 103343353, 'GAGA', 'G')], dataset_type='SNV_INDEL', omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733'], ) @@ -509,11 +531,9 @@ def _test_locus_query_variants(self, mock_get_variants, results_cache): has_gene_search=True, omitted_sample_guids=MITO_SAMPLES, ) - def _test_exclude_previous_search(self, mock_get_variants, *args, num_searches=1, **kwargs): - self._test_expected_search_call(mock_get_variants, *args, **kwargs) - self.assertEqual(mock_get_variants.call_count, num_searches) - def test_cached_query_variants(self): + Project.objects.filter(id=1).update(genome_version='38') + self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS}) variants, total = query_variants(self.results_model, user=self.user) self._assert_expected_cached_variants(variants, 4) @@ -523,192 +543,6 @@ def test_cached_query_variants(self): self._assert_expected_cached_variants(variants, 2) self.assertEqual(total, 4) - def _assert_expected_cached_variants(self, variants, num_results): - self.assertListEqual(variants, self.CACHED_VARIANTS[:num_results]) - - def test_invalid_search_get_variant_query_gene_counts(self): - self._test_invalid_search_params(get_variant_query_gene_counts) - - def test_get_variant_query_gene_counts(self, mock_get_variants): - def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = self.GENE_AGG_ALL_RESULTS - return None - def _mock_get_gene_counts(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['gene_aggs'] = GENE_COUNTS - return GENE_COUNTS - - mock_get_variants.side_effect = _mock_get_variants if hasattr(self, 'GENE_AGG_ALL_RESULTS') else _mock_get_gene_counts - - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, GENE_COUNTS) - results_cache = {'all_results': self.GENE_AGG_ALL_RESULTS} if hasattr(self, 'GENE_AGG_ALL_RESULTS') else {'gene_aggs': gene_counts} - self.assert_cached_results(results_cache) - kwargs = dict(sort=None, num_results=100) - if self.HAS_GENE_AGG: - kwargs['gene_agg'] = True - self._test_expected_search_call( - mock_get_variants, results_cache, **kwargs, - ) - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - HAS_GENE_AGG = True - INVALID_RSID_ERROR = 'Invalid variant notation: found both variant IDs and rsIDs' - - def setUp(self): - self.set_up() - - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): - return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) - - def test_variant_lookup(self, *args, **kwargs): - with self.assertRaises(ValueError) as cm: - super().test_variant_lookup(mock.MagicMock()) - self.assertEqual(str(cm.exception), 'variant_lookup is disabled without the clickhouse backend') - - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') - def test_get_single_variant(self, mock_get_variants_for_ids): - mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] - super(ElasticsearchSearchUtilsTests, self).test_get_single_variant(mock_get_variants_for_ids) - - @mock.patch('seqr.utils.search.utils.get_es_variants') - def test_query_variants(self, mock_get_variants): - super(ElasticsearchSearchUtilsTests, self).test_query_variants(mock_get_variants) - - def _test_locus_query_variants(self, mock_get_variants, results_cache): - self.search_model.search['locus'] = {'rawVariantItems': 'rs9876'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - rs_ids=['rs9876'], variant_ids=[], parsed_variant_ids=[], omitted_sample_guids=SV_SAMPLES, dataset_type='SNV_INDEL', - ) - - super()._test_locus_query_variants(mock_get_variants, results_cache) - - def test_cached_query_variants(self): - super(ElasticsearchSearchUtilsTests, self).test_cached_query_variants() - - self.set_cache({ - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}, - {'null': [PARSED_VARIANTS[1]]} - ], - 'total_results': 3, - }) - - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, [PARSED_VARIANTS[0]] + [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT] + [PARSED_VARIANTS[1]]) - self.assertEqual(total, 3) - - variants, total = query_variants(self.results_model, user=self.user, num_results=2) - self.assertListEqual(variants, [PARSED_VARIANTS[0]] + [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) - self.assertEqual(total, 3) - - @mock.patch('seqr.utils.search.utils.get_es_variants') - def test_get_variant_query_gene_counts(self, mock_get_variants): - super(ElasticsearchSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_get_variants) - - def test_cached_get_variant_query_gene_counts(self): - cached_gene_counts = { - 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} - } - self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, cached_gene_counts) - - self.set_cache({'all_results': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, 'total_results': 2}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 1, 'families': {'F000003_3': 1, 'F000011_11': 1}}, - 'ENSG00000228198': {'total': 1, 'families': {'F000003_3': 1, 'F000011_11': 1}}, - }) - - self.set_cache({ - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}, - {'null': [PARSED_VARIANTS[1]]} - ], - 'loaded_variant_counts': { - 'test_index_second': {'loaded': 1, 'total': 1}, - 'test_index_second_compound_het': {'total': 0, 'loaded': 0}, - 'test_index': {'loaded': 1, 'total': 1}, - 'test_index_compound_het': {'total': 2, 'loaded': 2}, - }, - 'total_results': 4, - }) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000002_2': 1}}, - 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}} - }) - - def _test_invalid_no_location_search_params(self): - # Elasticsearch has no limits on no-location searches - pass - - -class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): - databases = '__all__' - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] - - PARSED_CACHED_VARIANTS = [VARIANT1, VARIANT2, VARIANT3, VARIANT4] - CACHED_VARIANTS = [format_cached_variant(v) for v in PARSED_CACHED_VARIANTS] - GENE_AGG_ALL_RESULTS = CACHED_VARIANTS + [format_cached_variant(PROJECT_2_VARIANT2)] - INVALID_RSID_ERROR = 'Invalid variants: rs9876' - - def setUp(self): - self.set_up() - - def _assert_expected_cached_variants(self, variants, num_results): - self.assertListEqual( - json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), - self.PARSED_CACHED_VARIANTS[:num_results], - ) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') - def test_get_single_variant(self, mock_call): - mock_call.return_value = PARSED_VARIANTS[0] - super().test_get_single_variant(mock_call) - - def _assert_expected_get_single_variant_call(self, mock_call, variant_id, parsed_variant_id, expected_samples, dataset_type='SNV_INDEL', **kwargs): - mock_call.assert_called_with(variant_id, parsed_variant_id, mock.ANY, '37', dataset_type) - self.assertSetEqual(set(mock_call.call_args.args[2]), expected_samples) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_query_variants(self, mock_call): - super().test_query_variants(mock_call) - - def _test_exclude_previous_search(self, mock_get_variants, *args, **kwargs): - super()._test_exclude_previous_search( - mock_get_variants, *args, **kwargs, - exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, num_searches=2, - ) - call_args, call_kwargs, _ = self._get_expected_search_call(*args, inheritance_mode='de_novo', sort=None, num_results=100) - mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)]) - - # Test when previous results are cached - mock_get_variants.reset_mock() - self.mock_redis.get.side_effect = [ - None, - json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), - ] - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] - query_variants(self.results_model, user=self.user) - super()._test_exclude_previous_search( - mock_get_variants, *args, **kwargs, num_searches=1, - exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]}, - exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]}, - ) - - def test_cached_query_variants(self): - Project.objects.filter(id=1).update(genome_version='38') - super().test_cached_query_variants() - cache_key_prefix = f'search_results__{self.results_model.guid}' self.mock_redis.get.side_effect = [None, json.dumps({'total_results': 4, 'all_results': self.CACHED_VARIANTS})] self.mock_redis.keys.return_value = [f'{cache_key_prefix}__xpos', f'{cache_key_prefix}__gnomad'] @@ -725,13 +559,37 @@ def test_cached_query_variants(self): ]) self.mock_redis.keys.assert_called_with(pattern=f'{cache_key_prefix}__*') self.assert_cached_results( - {'all_results': [format_cached_variant(v) for v in [VARIANT4, VARIANT3, VARIANT2, VARIANT1]], 'total_results': 4}, + {'all_results': [format_cached_variant(v) for v in [VARIANT4, VARIANT3, VARIANT2, VARIANT1]], + 'total_results': 4}, sort='cadd', ) + def _assert_expected_cached_variants(self, variants, num_results): + self.assertListEqual( + json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), + self.PARSED_CACHED_VARIANTS[:num_results], + ) + + def test_invalid_search_get_variant_query_gene_counts(self): + self._test_invalid_search_params(get_variant_query_gene_counts) + @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_get_variant_query_gene_counts(self, mock_call): - super().test_get_variant_query_gene_counts(mock_call) + def test_get_variant_query_gene_counts(self, mock_get_variants): + gene_agg_all_results = self.CACHED_VARIANTS + [format_cached_variant(PROJECT_2_VARIANT2)] + def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): + previous_search_results['all_results'] = gene_agg_all_results + return None + + mock_get_variants.side_effect = _mock_get_variants + + gene_counts = get_variant_query_gene_counts(self.results_model, self.user) + self.assertDictEqual(gene_counts, GENE_COUNTS) + results_cache = {'all_results': gene_agg_all_results} + self.assert_cached_results(results_cache) + kwargs = dict(sort=None, num_results=100) + self._test_expected_search_call( + mock_get_variants, results_cache, **kwargs, + ) def test_cached_get_variant_query_gene_counts(self): self.set_cache({'all_results': self.CACHED_VARIANTS + [SV_VARIANT1], 'total_results': 5}) @@ -742,7 +600,3 @@ def test_cached_get_variant_query_gene_counts(self): 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, }) - - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_variant_lookup(self, mock_call): - super().test_variant_lookup(mock_call) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 7b6fa97fef..c5a0fdea60 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -233,7 +233,7 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, return variants, total_results -def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, page=1, **kwargs): +def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, **kwargs): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() From 85a602395f4abffaccc0784bed5ef9cab5adf86c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:20:07 -0500 Subject: [PATCH 017/252] exception clean up --- seqr/utils/middleware.py | 5 +---- seqr/utils/search/utils.py | 9 +-------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 33d22532a3..0b77e1b1f0 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -11,7 +11,7 @@ import json import traceback -from seqr.utils.search.utils import ERROR_LOG_EXCEPTIONS, SEARCH_EXCEPTION_ERROR_MAP, SEARCH_EXCEPTION_MESSAGE_MAP +from seqr.utils.search.utils import SEARCH_EXCEPTION_ERROR_MAP from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.terra_api_utils import TerraAPIException @@ -47,7 +47,6 @@ def __init__(self, errors, warnings=None): EXCEPTION_MESSAGE_MAP = { TerraAPIException: lambda e: LOGIN_URL if e.status_code == 401 else str(e), } -EXCEPTION_MESSAGE_MAP.update(SEARCH_EXCEPTION_MESSAGE_MAP) def _get_exception_status_code(exception): @@ -75,8 +74,6 @@ class JsonErrorMiddleware(MiddlewareMixin): def process_exception(request, exception): exception_json = _get_core_exception_json(exception) status = _get_exception_status_code(exception) - if exception.__class__ in ERROR_LOG_EXCEPTIONS: - exception_json['log_error'] = True if DEBUG or status == 500: traceback_message = traceback.format_exc() exception_json['traceback'] = traceback_message diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index c5a0fdea60..0f8c4c2a45 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -13,7 +13,7 @@ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ - es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS + es_backend_enabled, ping_kibana from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -26,13 +26,6 @@ class InvalidSearchException(Exception): SEARCH_EXCEPTION_ERROR_MAP = { InvalidSearchException: 400, } -SEARCH_EXCEPTION_ERROR_MAP.update(ES_EXCEPTION_ERROR_MAP) - -SEARCH_EXCEPTION_MESSAGE_MAP = {} -SEARCH_EXCEPTION_MESSAGE_MAP.update(ES_EXCEPTION_MESSAGE_MAP) - -ERROR_LOG_EXCEPTIONS = set() -ERROR_LOG_EXCEPTIONS.update(ES_ERROR_LOG_EXCEPTIONS) DATASET_TYPES_LOOKUP = { data_types[0]: data_types for data_types in [ From 9b6df59be91d459f52bf3e18816a595762c9a6e8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:26:36 -0500 Subject: [PATCH 018/252] remove utility tests --- .../search/elasticsearch/es_utils_tests.py | 3796 ----------------- 1 file changed, 3796 deletions(-) delete mode 100644 seqr/utils/search/elasticsearch/es_utils_tests.py diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py deleted file mode 100644 index 85651ec04f..0000000000 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ /dev/null @@ -1,3796 +0,0 @@ -from copy import deepcopy -import mock -import jmespath -import json -import re -from collections import defaultdict -from datetime import timedelta -from django.test import TestCase -from elasticsearch.exceptions import ConnectionTimeout, TransportError -from sys import maxsize -from urllib3.exceptions import ReadTimeoutError -from urllib3_mock import Responses - -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, InvalidSearchException -from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status -from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException -from seqr.utils.search.utils import _get_liftover -from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_SV_WGS_VARIANT,\ - PARSED_MITO_VARIANT, TRANSCRIPT_2, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT - - -# The responses library for mocking requests does not work with urllib3 (which is used by elasticsearch) -# The urllib3_mock library works for those requests, but it has limited functionality, so this extension adds helper -# methods for easier usage -class Urllib3Responses(Responses): - def add_json(self, url, json_response, method=None, match_querystring=True, **kwargs): - if not method: - method = self.GET - body = json.dumps(json_response) - self.add(method, url, match_querystring=match_querystring, content_type='application/json', body=body, **kwargs) - - def replace_json(self, url, *args, **kwargs): - existing_index = next(i for i, match in enumerate(self._urls) if match['url'] == url) - self.add_json(url, *args, **kwargs) - self._urls[existing_index] = self._urls.pop() - - def call_request_json(self, index=-1): - return json.loads(self.calls[index].request.body) - - def activate(self, func): - def wrapper(*args, **kwargs): - with self: - return func(*args, **kwargs) - return wrapper - -urllib3_responses = Urllib3Responses() - - -INDEX_NAME = 'test_index' -SECOND_INDEX_NAME = 'test_index_second' -HG38_INDEX_NAME = 'test_index_hg38' -SV_INDEX_NAME = 'test_index_sv' -SV_WGS_INDEX_NAME = 'test_index_sv_wgs' -MITO_WGS_INDEX_NAME = 'test_index_mito_wgs' -INDEX_ALIAS = '377e97bd791cf92a78296bc184e0976a' -ALIAS_MAP = {INDEX_ALIAS: ','.join([INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME, MITO_WGS_INDEX_NAME])} -SUB_INDICES = ['sub_index_1', 'sub_index_2'] -SECOND_SUB_INDICES = ['sub_index_a', 'sub_index_b'] - -ES_VARIANTS = [ - { - '_source': { - 'homozygote_count': 3, - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '1-248367227-TC-T' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': 0, - 'cadd_PHRED': '25.9', - 'splice_ai_delta_score': 0.75, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': 125568, - 'g1k_AN': None, - 'topmed_AF': 0.00016724, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'TC', - 'exac_AC_Hom': 0, - 'topmed_AC': 21, - 'dbnsfp_REVEL_score': None, - 'dbnsfp_VEST4_score': '0.335;0.341;0.38', - 'primate_ai_score': None, - 'variantId': '1-248367227-TC-T', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'nonsense_mediated_decay', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Gene3D:1', - ], - 'canonical': None, - 'transcript_rank': 1, - 'cdna_end': 143, - 'lof_filter': None, - 'hgvs': 'ENSP00000413641.1:p.Leu48del', - 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', - 'cdna_start': 141, - 'transcript_id': 'ENST00000428085', - 'protein_id': 'ENSP00000413641', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000413641.1:p.Leu48del', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant', - 'inframe_deletion', - 'NMD_transcript_variant' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'screen_region_type' : [ - 'dELS', - 'CTCF-bound' - ], - 'gnomad_non_coding_constraint_z_score': 1.01272, - 'hgmd_class': 'DM', - 'AC': 2, - 'exac_AN_Adj': 121308, - 'mpc_MPC': None, - 'AF': 0.063, - 'alt': 'T', - 'clinvar_clinical_significance': 'Pathogenic/Likely_pathogenic', - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': 0.0004590314436538903, - 'gnomad_genomes_FAF_AF': 0.000437, - 'exac_AF': 0.00006589, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 8, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': 0, - 'gnomad_genomes_AN': 30946, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.0009151523074911753, - 'gnomad_genomes_Hemi': None, - 'xpos': 1248367227, - 'start': 248367227, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245930, - 'contig': '1', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.0006726888333653661, - 'gnomad_exomes_AC': 16, - 'dbnsfp_FATHMM_pred': 'T', - 'dbnsfp_fathmm_MKL_coding_pred': 'D', - 'gnomad_exomes_AF': 0.00006505916317651364, - 'gnomad_genomes_AF': 0.00012925741614425127, - 'gnomad_genomes_AC': 4, - 'genotypes': [ - { - 'num_alt': 2, - 'ab': 1, - 'dp': 74, - 'gq': 99, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 88, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - ], - 'samples_num_alt_1': ['NA20885'], - 'samples_num_alt_2': ['NA20870'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3'], SECOND_INDEX_NAME: ['F000011_11']}, - }, - { - '_source': { - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '2-103343353-GAGA-G' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': None, - 'cadd_PHRED': None, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': None, - 'g1k_AN': None, - 'topmed_AF': None, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'GAGA', - 'exac_AC_Hom': 0, - 'topmed_AC': None, - 'dbnsfp_REVEL_score': None, - 'primate_ai_score': 1, - 'variantId': '2-103343353-GAGA-G', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'PROSITE_profiles:PS50850', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 421, - 'lof_filter': None, - 'hgvs': 'ENSP00000258436.5:p.Leu126del', - 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', - 'cdna_start': 419, - 'transcript_id': 'ENST00000258436', - 'protein_id': 'ENSP00000258436', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000258436.5:p.Leu126del', - 'major_consequence': 'inframe_deletion', - 'consequence_terms': [ - 'inframe_deletion' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'hgmd_class': None, - 'AC': 1, - 'exac_AN_Adj': 121336, - 'mpc_MPC': None, - 'AF': 0.031, - 'alt': 'G', - 'clinvar_clinical_significance': None, - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': None, - 'gnomad_genomes_FAF_AF': None, - 'exac_AF': 0.00004942, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 6, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': None, - 'gnomad_genomes_AN': None, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'dbnsfp_VEST4_score': '.;.;.;.', - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.00016269686320447742, - 'gnomad_genomes_Hemi': None, - 'xpos': 2103343353, - 'start': 103343353, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245714, - 'contig': '2', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.000242306760358614, - 'gnomad_exomes_AC': 6, - 'dbnsfp_FATHMM_pred': None, - 'gnomad_exomes_AF': 0.000024418633044922146, - 'gnomad_genomes_AF': None, - 'gnomad_genomes_AC': None, - 'genotypes': [ - { - 'num_alt': 1, - 'ab': 0.70212764, - 'dp': 50, - 'gq': 46, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - { - 'num_alt': 2, - 'ab': 0, - 'dp': 67, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00732', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00733', - } - ], - 'samples_num_alt_1': ['NA20870', 'HG00733', 'NA20885'], - 'samples_num_alt_2': ['HG00732'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3', 'F000002_2'], SECOND_INDEX_NAME: ['F000011_11']}, - }, -] -BUILD_38_NO_LIFTOVER_ES_VARIANT = deepcopy(ES_VARIANTS[1]) -BUILD_38_NO_LIFTOVER_ES_VARIANT['_source'].update({ - 'start': 103343363, - 'xpos': 2103343363, - 'variantId': '2-103343363-GAGA-G' -}) -BUILD_38_ES_VARIANT = deepcopy(BUILD_38_NO_LIFTOVER_ES_VARIANT) -BUILD_38_ES_VARIANT['_source']['rg37_locus'] = {'contig': '2', 'position': 103343353} - -ES_SV_VARIANT = { - '_source': { - 'genotypes': [ - { - 'qs': 33, - 'cn': 1, - 'defragged': False, - 'sample_id': 'HG00731', - 'num_exon': 2, - 'start': 49045487, - 'end': 49045899, - 'geneIds': ['ENSG00000228198'], - 'prev_call': False, - 'prev_overlap': False, - 'new_call': True, - }, - { - 'qs': 80, - 'cn': 2, - 'defragged': False, - 'sample_id': 'HG00733', - 'num_exon': 1, - 'start': 49045987, - 'end': 49045890, - 'geneIds': ['ENSG00000228198', 'ENSG00000135953'], - 'prev_call': False, - 'prev_overlap': True, - 'new_call': False, - } - ], - 'xpos': 1049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 1049045587, - 'num_exon': 1, - 'pos': 49045487, - 'StrVCTVRE_score': 0.374, - 'svType': 'INS', - 'xstop': 9049045898, - 'variantId': 'prefix_19107_DEL', - 'samples': ['HG00731'], - 'sc': 7, - 'contig': '1', - 'bothsides_support': True, - 'sortedTranscriptConsequences': [ - { - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_id': 'ENSG00000135953' - }, - { - 'gene_id': 'ENSG00000186092' - }, - ], - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_INDEX_NAME: ['F000002_2']}, - } - -ES_SV_WGS_VARIANT = { - '_source': { - 'genotypes': [ - { - 'gq': 33, - 'sample_id': 'NA21234', - 'num_alt': 1, - 'prev_num_alt': 2, - } - ], - 'xpos': 2049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 2049045387, - 'pos': 49045387, - 'svType': 'CPX', - 'xstop': 20012345678, - 'variantId': 'prefix_19107_CPX', - 'algorithms': ['wham', 'manta'], - 'sc': 7, - 'contig': '2', - 'rg37_locus': {'contig': '2', 'position': 49272526}, - 'rg37_locus_end': {'contig': '20', 'position': 12326326}, - 'sortedTranscriptConsequences': [ - { - 'gene_symbol': 'OR4F5', - 'major_consequence': 'DUP_PARTIAL', - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_symbol': 'FBXO28', - 'major_consequence': 'MSV_EXON_OVR', - 'gene_id': 'ENSG00000228199' - }, - { - 'gene_symbol': 'FAM131C', - 'major_consequence': 'DUP_LOF', - 'gene_id': 'ENSG00000228201' - }, - { - "gene_symbol": "H3-2", - "gene_id": None, - "major_consequence": "NEAREST_TSS" - }, - ], - 'cpx_intervals': [{'type': 'DUP', 'chrom': '2', 'start': 1000, 'end': 3000}, - {'type': 'INV', 'chrom': '20', 'start': 11000, 'end': 13000}], - 'sv_type_detail': 'dupINV', - 'gnomad_svs_ID': 'gnomAD-SV_v2.1_BND_1_1', - 'gnomad_svs_AF': 0.00679, - 'gnomad_svs_AC': 22, - 'gnomad_svs_AN': 3240, - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_WGS_INDEX_NAME: ['F000014_14']}, - } - -ES_MITO_WGS_VARIANT = { - "_source" : { - "genotypes" : [ - { - "num_alt": 2, - "gq": 60.0, - "hl": 1.0, - "mito_cn": 319.03225806451616, - "contamination": 0.0, - "dp": 5139.0, - "sample_id": "HG00731" - }, - ], - "samples_gq_60_to_65" : [ - "HG00731" - ], - "samples_num_alt_2" : [ "HG00731" ], - "AC" : 0, - "AC_het" : 1, - "AF" : 0.0, - "AF_het" : 3.968253968253968E-4, - "alt" : "A", - "AN" : 2520, - "clinvar_allele_id" : None, - "clinvar_clinical_significance" : "Likely_pathogenic", - "clinvar_gold_stars" : None, - "codingGeneIds" : [ - "ENSG00000198840" - ], - "common_low_heteroplasmy" : False, - "contig" : "M", - "dbnsfp_SIFT_pred" : "D", - "dbnsfp_MutationTaster_pred" : "N", - "dbnsfp_FATHMM_pred" : "T", - "dbnsfp_GERP_RS" : "5.07", - "dbnsfp_phastCons100way_vertebrate" : "0.958000", - "docId" : "M-10195-C-A", - "domains" : [ - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Gene3D:1", - "PANTHER:PTHR11058", - "Pfam:PF00507" - ], - "end" : 10195, - "filters" : [ ], - "geneIds" : [ - "ENSG00000198840" - ], - "gnomad_mito_AN": 56421, - "gnomad_mito_AC": 1368, - "gnomad_mito_AC_het": 3, - "gnomad_mito_AF": 0.024246292, - "gnomad_mito_AF_het": 5.317169E-5, - "gnomad_mito_max_hl": 1.0, - "hap_defining_variant" : False, - "helix_AC": 1312, - "helix_AC_het": 5, - "helix_AF": 0.0033268193, - "helix_AF_het": 4.081987E-5, - "helix_max_hl": 0.90441, - "high_constraint_region" : True, - "hmtvar_hmtVar" : 0.71, - "mainTranscript_biotype" : "protein_coding", - "mainTranscript_canonical" : 1, - "mainTranscript_category" : "missense", - "mainTranscript_cdna_start" : 137, - "mainTranscript_cdna_end" : 137, - "mainTranscript_codons" : "cCc/cAc", - "mainTranscript_gene_id" : "ENSG00000198840", - "mainTranscript_gene_symbol" : "MT-ND3", - "mainTranscript_hgvs" : "p.Pro46His", - "mainTranscript_hgvsc" : "ENST00000361227.2:c.137C>A", - "mainTranscript_major_consequence" : "missense_variant", - "mainTranscript_major_consequence_rank" : 11, - "mainTranscript_transcript_id" : "ENST00000361227", - "mainTranscript_amino_acids" : "P/H", - "mainTranscript_domains" : "Gene3D:1,ENSP_mappings:5xtc,PANTHER:PTHR11058,ENSP_mappings:5xtd,Pfam:PF00507", - "mainTranscript_hgvsp" : "ENSP00000355206.2:p.Pro46His", - "mainTranscript_polyphen_prediction" : "probably_damaging", - "mainTranscript_protein_id" : "ENSP00000355206", - "mainTranscript_sift_prediction" : "deleterious_low_confidence", - "mitimpact_apogee" : 0.42, - "mitomap_pathogenic" : True, - "pos" : 10195, - "ref" : "C", - "rg37_locus" : { - "contig" : "MT", - "position" : 10195 - }, - "rsid" : None, - "sortedTranscriptConsequences" : [ - { - "biotype" : "protein_coding", - "canonical" : 1, - "cdna_start" : 137, - "cdna_end" : 137, - "codons" : "cCc/cAc", - "gene_id" : "ENSG00000198840", - "gene_symbol" : "MT-ND3", - "hgvsc" : "ENST00000361227.2:c.137C>A", - "hgvsp" : "ENSP00000355206.2:p.Pro46His", - "transcript_id" : "ENST00000361227", - "amino_acids" : "P/H", - "polyphen_prediction" : "probably_damaging", - "protein_id" : "ENSP00000355206", - "protein_start" : 46, - "sift_prediction" : "deleterious_low_confidence", - "consequence_terms" : [ - "missense_variant" - ], - "domains" : [ - "Gene3D:1", - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Pfam:PF00507", - "PANTHER:PTHR11058", - "PANTHER:PTHR11058" - ], - "major_consequence" : "missense_variant", - "category" : "missense", - "hgvs" : "p.Pro46His", - "major_consequence_rank" : 11, - "transcript_rank" : 0 - } - ], - "start" : 10195, - "transcriptConsequenceTerms" : [ - "missense_variant" - ], - "transcriptIds" : [ - "ENST00000361227" - ], - "variantId" : "M-10195-C-A", - "xpos" : 25000010195, - "xstart" : 25000010195, - "xstop" : 25000010195 - }, - 'matched_queries': {MITO_WGS_INDEX_NAME: ['F000002_2']}, -} - -OR2M3_COMPOUND_HET_ES_VARIANTS = deepcopy(ES_VARIANTS) -transcripts = OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] -transcripts[0]['major_consequence'] = 'frameshift_variant' -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] = [transcripts[1], transcripts[0]] -OR2M3_COMPOUND_HET_ES_VARIANTS[0]['_source']['rg37_locus'] = {'contig': '1', 'position': 248367217} -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['rg37_locus'] = {'contig': '2', 'position': 103343343} -MFSD9_COMPOUND_HET_ES_VARIANTS = deepcopy(OR2M3_COMPOUND_HET_ES_VARIANTS) -for var in MFSD9_COMPOUND_HET_ES_VARIANTS: - var['_source']['variantId'] = '{}-het'.format(var['_source']['variantId']) -EXTRA_FAMILY_ES_VARIANTS = deepcopy(ES_VARIANTS) + [deepcopy(ES_VARIANTS[0])] -EXTRA_FAMILY_ES_VARIANTS[2]['matched_queries'][INDEX_NAME] = ['F000005_5'] -MISSING_SAMPLE_ES_VARIANTS = deepcopy(ES_VARIANTS) -MISSING_SAMPLE_ES_VARIANTS[1]['_source']['samples_num_alt_1'] = [] - -ES_SV_COMP_HET_VARIANT = deepcopy(ES_SV_VARIANT) -ES_SV_COMP_HET_VARIANT['_source']['xpos'] = 2101343374 -ES_SV_COMP_HET_VARIANT['_source']['start'] = 101343374 -ES_SV_COMP_HET_VARIANT['_source']['xstop'] = 1104943628 -ES_SV_COMP_HET_VARIANT['_source']['end'] = 104943628 -ES_SV_COMP_HET_VARIANT['_source']['num_exon'] = 2 -ES_SV_COMP_HET_VARIANT['_source']['variantId'] = 'prefix_191011_DEL' -ES_SV_COMP_HET_VARIANT['_source']['svType'] = 'DEL' -for gen in ES_SV_COMP_HET_VARIANT['_source']['genotypes']: - gen.update({'start': None, 'end': None, 'num_exon': None}) - gen.pop('geneIds') - -COMPOUND_HET_INDEX_VARIANTS = { - INDEX_NAME: { - 'ENSG00000135953': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228198': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228199': EXTRA_FAMILY_ES_VARIANTS, - }, - SECOND_INDEX_NAME: { - 'ENSG00000135953': MFSD9_COMPOUND_HET_ES_VARIANTS, 'ENSG00000228198': OR2M3_COMPOUND_HET_ES_VARIANTS, - }, - '{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME): {'ENSG00000135953': MISSING_SAMPLE_ES_VARIANTS}, - SV_INDEX_NAME: {'ENSG00000228198': [ES_SV_VARIANT], 'ENSG00000135953': []}, - '{},{}'.format(INDEX_NAME, SV_INDEX_NAME): {'ENSG00000228198': [ES_SV_VARIANT, ES_SV_COMP_HET_VARIANT, ES_VARIANTS[1]], 'ENSG00000135953': []}, -} - -INDEX_ES_VARIANTS = { - INDEX_NAME: ES_VARIANTS, - SECOND_INDEX_NAME: [ES_VARIANTS[1]], - SV_INDEX_NAME: [ES_SV_VARIANT], - SV_WGS_INDEX_NAME: [ES_SV_WGS_VARIANT], - HG38_INDEX_NAME: [BUILD_38_ES_VARIANT, BUILD_38_NO_LIFTOVER_ES_VARIANT], - MITO_WGS_INDEX_NAME: [ES_MITO_WGS_VARIANT], -} -INDEX_ES_VARIANTS.update({k: ES_VARIANTS for k in SUB_INDICES + SECOND_SUB_INDICES}) - -PARSED_ANY_AFFECTED_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_ANY_AFFECTED_VARIANTS[1]['familyGuids'] = ['F000003_3'] -PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes'] = {'I000007_na20870': PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes']['I000007_na20870']} - -PARSED_COMPOUND_HET_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_COMPOUND_HET_VARIANTS[0]['_sort'] = [1248367327] -PARSED_COMPOUND_HET_VARIANTS[1]['_sort'] = [2103343453] -PARSED_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000003_3'] - -PARSED_SV_COMPOUND_HET_VARIANTS = [deepcopy(PARSED_SV_VARIANT), deepcopy(PARSED_COMPOUND_HET_VARIANTS[1])] -PARSED_SV_COMPOUND_HET_VARIANTS[0].update({ - '_sort': [2101343474], - 'xpos': 2101343374, - 'pos': 101343374, - 'end': 104943628, - 'variantId': 'prefix_191011_DEL', - 'svType': 'DEL', -}) -del PARSED_SV_COMPOUND_HET_VARIANTS[0]['svSourceDetail'] -PARSED_SV_COMPOUND_HET_VARIANTS[0]['transcripts']['ENSG00000186092'] = [{'geneId': 'ENSG00000186092'}] -for gen in PARSED_SV_COMPOUND_HET_VARIANTS[0]['genotypes'].values(): - gen.update({'start': None, 'end': None, 'numExon': None, 'geneIds': None}) -PARSED_SV_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000002_2'] - -PARSED_COMPOUND_HET_VARIANTS_PROJECT_2 = deepcopy(PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT) -for variant in PARSED_COMPOUND_HET_VARIANTS_PROJECT_2: - variant.update({ - 'variantId': '{}-het'.format(variant['variantId']), - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': variant['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '37', - 'selectedMainTranscriptId': None, - }) - variant['clinvar']['version'] = None - -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[1]['selectedMainTranscriptId'] = None - -PARSED_CADD_VARIANTS = deepcopy(PARSED_NO_CONSEQUENCE_FILTER_VARIANTS) -PARSED_CADD_VARIANTS[0]['_sort'][0] = -25.9 -PARSED_CADD_VARIANTS[1]['_sort'][0] = maxsize - - -PARSED_MULTI_INDEX_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_MULTI_INDEX_VARIANT['familyGuids'].append('F000011_11') -PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'] = { - 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, 'sampleType': 'WES', -} - -PARSED_HG38_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_HG38_VARIANT.update({ - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverPos': PARSED_MULTI_INDEX_VARIANT['pos'], - 'liftedOverChrom': PARSED_MULTI_INDEX_VARIANT['chrom'], - 'pos': PARSED_MULTI_INDEX_VARIANT['pos'] + 10, - 'xpos': PARSED_MULTI_INDEX_VARIANT['xpos'] + 10, - 'variantId': PARSED_MULTI_INDEX_VARIANT['variantId'].replace( - str(PARSED_MULTI_INDEX_VARIANT['pos']), str(PARSED_MULTI_INDEX_VARIANT['pos'] + 10) - ), - '_sort': [PARSED_MULTI_INDEX_VARIANT['_sort'][0] + 10], -}) -PARSED_HG38_VARIANT['clinvar']['version'] = None - -PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -for guid, genotype in PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT = deepcopy(PARSED_VARIANTS[1]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT_0 = deepcopy(PARSED_VARIANTS[0]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS = deepcopy(PARSED_COMPOUND_HET_VARIANTS) -for variant in PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS: - for guid, genotype in variant['genotypes'].items(): - variant['genotypes'][guid] = dict(otherSample=genotype, **genotype) - - -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT.update({ - 'familyGuids': ['F000003_3', 'F000011_11'], - 'genotypes': { - ind_guid: PARSED_MULTI_INDEX_VARIANT['genotypes'][ind_guid] - for ind_guid in ['I000007_na20870', 'I000015_na20885'] - }, -}) -MAPPING_FIELDS = [ - 'start', - 'end', - 'rsid', - 'originalAltAlleles', - 'filters', - 'xpos', - 'alt', - 'ref', - 'contig', - 'variantId', - 'dbnsfp_MutationTaster_pred', - 'mpc_MPC', - 'dbnsfp_DANN_score', - 'eigen_Eigen_phred', - 'dbnsfp_REVEL_score', - 'splice_ai_delta_score', - 'splice_ai_splice_consequence', - 'dbnsfp_FATHMM_pred', - 'dbnsfp_fathmm_MKL_coding_pred', - 'dbnsfp_MutPred_score', - 'dbnsfp_VEST4_score', - 'primate_ai_score', - 'dbnsfp_SIFT_pred', - 'dbnsfp_Polyphen2_HVAR_pred', - 'cadd_PHRED', - 'gnomad_non_coding_constraint_z_score', - 'sortedTranscriptConsequences', - 'screen_region_type', - 'genotypes', - 'samples_no_call', - 'samples_num_alt_1', - 'samples_num_alt_2', - 'clinvar_clinical_significance', - 'clinvar_allele_id', - 'clinvar_variation_id', - 'clinvar_gold_stars', - 'hgmd_accession', - 'hgmd_class', - 'AC', - 'AF', - 'AN', - 'gnomad_genomes_AC', - 'gnomad_genomes_Hom', - 'gnomad_genomes_Hemi', - 'gnomad_genomes_AF', - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_genomes_AN', - 'gnomad_genomes_Het', - 'gnomad_genomes_ID', - 'gnomad_exomes_AC', - 'gnomad_exomes_Hom', - 'gnomad_exomes_Hemi', - 'gnomad_exomes_AF', - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_exomes_AN', - 'gnomad_exomes_Het', - 'gnomad_exomes_ID', - 'exac_AC_Adj', - 'exac_AC_Hom', - 'exac_AC_Hemi', - 'exac_AF_POPMAX', - 'exac_AF', - 'exac_AN_Adj', - 'exac_Het', - 'exac_ID', - 'topmed_AC', - 'topmed_Hom', - 'topmed_Hemi', - 'topmed_Het', - 'topmed_AF', - 'topmed_AN', - 'topmed_ID', - 'gnomad_genomes_FAF_AF', - 'rg37_locus', - 'rg37_locus_end', - 'xstop', - 'bothsides_support', -] - -SV_MAPPING_FIELDS = [ - 'start', - 'end', - 'xpos', - 'contig', - 'variantId', - 'sortedTranscriptConsequences', - 'genotypes', - 'samples', - 'samples_cn_0', - 'samples_cn_1', - 'samples_cn_2', - 'samples_cn_3', - 'samples_cn_gte_4', - 'sf', - 'sc', - 'sn', - 'num_exon', - 'svType', - 'StrVCTVRE_score', - 'sv_type_detail', - 'cpx_intervals', - 'gnomad_svs_AC', - 'gnomad_svs_Hom', - 'gnomad_svs_Hemi', - 'gnomad_svs_AF', - 'gnomad_svs_AN', - 'gnomad_svs_filter_AF', - 'gnomad_svs_Het', - 'gnomad_svs_ID', - 'bothsides_support', -] - -MITO_MAPPING_FIELDS = [ - "AC", - "AC_het", - "AF", - "AF_het", - "AN", - "alt", - "clinvar_allele_id", - "clinvar_clinical_significance", - "clinvar_gold_stars", - "common_low_heteroplasmy", - "contig", - "dbnsfp_FATHMM_pred", - "dbnsfp_MutationTaster_pred", - "dbnsfp_Polyphen2_HVAR_pred", - "dbnsfp_REVEL_score", - "dbnsfp_SIFT_pred", - "end", - "filters", - "genotypes", - "gnomad_mito_AC", - "gnomad_mito_AC_het", - "gnomad_mito_AF", - "gnomad_mito_AF_het", - "gnomad_mito_AN", - "gnomad_mito_max_hl", - "hap_defining_variant", - "helix_AC", - "helix_AC_het", - "helix_AF", - "helix_AF_het", - "helix_max_hl", - "high_constraint_region", - "hmtvar_hmtVar", - "mitimpact_apogee", - "mitomap_pathogenic", - "mitotip_mitoTIP", - "ref", - "rg37_locus", - "rsid", - "samples_num_alt_1", - "samples_num_alt_2", - "sortedTranscriptConsequences", - "start", - "variantId", - "xpos", - "xstop", -] - -MITO_SOURCE_ONLY_FIELDS = [ - 'callset_max_hl', - 'exac_max_hl', - 'gnomad_exomes_max_hl', - 'gnomad_genomes_max_hl', - 'gnomad_svs_max_hl', - 'sv_callset_max_hl', - 'topmed_max_hl', - 'helix_ID', - 'helix_Hemi', - 'helix_AN', - 'helix_filter_AF', - 'helix_Het', - 'helix_Hom', - 'gnomad_mito_filter_AF', - 'gnomad_mito_Hom', - 'gnomad_mito_ID', - 'gnomad_mito_Het', - 'gnomad_mito_Hemi', - 'callset_heteroplasmy_Het', - 'callset_heteroplasmy_ID', - 'callset_heteroplasmy_Hemi', - 'callset_heteroplasmy_filter_AF', - 'callset_heteroplasmy_Hom', - 'callset_heteroplasmy_max_hl', - 'gnomad_mito_heteroplasmy_filter_AF', - 'gnomad_mito_heteroplasmy_Hemi', - 'gnomad_mito_heteroplasmy_Hom', - 'gnomad_mito_heteroplasmy_ID', - 'gnomad_mito_heteroplasmy_Het', - 'helix_heteroplasmy_Het', - 'helix_heteroplasmy_ID', - 'helix_heteroplasmy_Hemi', - 'helix_heteroplasmy_filter_AF', - 'helix_heteroplasmy_Hom', -] - -SOURCE_FIELDS = { - 'homozygote_count', 'callset_Hemi', 'callset_Het', 'callset_ID', 'sv_callset_Hemi', - 'sv_callset_Hom', 'sv_callset_Het', 'sv_callset_ID', 'algorithms', -} -SOURCE_FIELDS.update(MAPPING_FIELDS) -SOURCE_FIELDS.update(SV_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_SOURCE_ONLY_FIELDS) -SOURCE_FIELDS -= { - 'samples_no_call', 'samples_cn_0', 'samples_cn_1', 'samples_cn_2', 'samples_cn_3', 'samples_cn_gte_4', 'topmed_Het', - 'gnomad_genomes_FAF_AF', -} - -FIELD_TYPE_MAP = { - 'cadd_PHRED': {'type': 'keyword'}, - 'primate_ai_score': {'type': 'float'}, - 'rg37_locus': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}}, - 'rg37_locus_end': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}} -} -MAPPING_PROPERTIES = {field: FIELD_TYPE_MAP.get(field, {'type': 'keyword'}) for field in MAPPING_FIELDS} - -CORE_INDEX_METADATA = { - INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'clinvar_version': '2023-03-05'}, - 'properties': MAPPING_PROPERTIES, - }, - SECOND_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'VARIANTS'}, - 'properties': MAPPING_PROPERTIES, - }, - SV_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'SV'}, - 'properties': {field: {'type': 'keyword'} for field in SV_MAPPING_FIELDS}, - }, - MITO_WGS_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'MITO'}, - 'properties': {field: {'type': 'keyword'} for field in MITO_MAPPING_FIELDS}, - }, -} -INDEX_METADATA = { - HG38_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SECOND_INDEX_NAME]), - SV_WGS_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SV_INDEX_NAME]), -} -for meta in INDEX_METADATA.values(): - meta['_meta']['genomeVersion'] = '38' -INDEX_METADATA.update(CORE_INDEX_METADATA) - -ALL_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples': ['HG00731', 'HG00732', 'HG00733']}}, - ]}} - ], - '_name': 'F000002_2' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20870']}}, - {'terms': {'samples_num_alt_2': ['NA20870']}}, - {'terms': {'samples': ['NA20870']}}, - ]}} - ], - '_name': 'F000003_3' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20874']}}, - {'terms': {'samples_num_alt_2': ['NA20874']}}, - {'terms': {'samples': ['NA20874']}}, - ]}} - ], - '_name': 'F000005_5' - }} - ] - } -} - -COMPOUND_HET_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -COMPOUND_HET_PATH_INHERITANCE_QUERY = deepcopy(COMPOUND_HET_INHERITANCE_QUERY) -for fam_q in COMPOUND_HET_PATH_INHERITANCE_QUERY['bool']['should']: - fam_quality_q = fam_q['bool']['must'][1] - fam_quality_q['bool'] = {'should': [ - deepcopy(fam_quality_q), - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}} - ]} - -RECESSIVE_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -ANNOTATION_QUERY = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}} - -REDIS_CACHE = {} -def _set_cache(k, v): - REDIS_CACHE[k] = v -MOCK_REDIS = mock.MagicMock() -MOCK_REDIS.get.side_effect = REDIS_CACHE.get -MOCK_REDIS.set.side_effect =_set_cache - -def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=None, index=INDEX_NAME): - parsed_hits = deepcopy(hits) - for hit in parsed_hits: - hit.update({ - '_index': index, - '_id': hit['_source']['variantId'], - '_type': 'structural_variant' if SV_INDEX_NAME in index or SV_WGS_INDEX_NAME in index else 'variant', - }) - matched_queries = hit.pop('matched_queries') - if include_matched_queries: - hit['matched_queries'] = [] - for subindex in index.split(','): - if subindex in matched_queries: - hit['_index'] = subindex - hit['matched_queries'] += matched_queries[subindex] - - if sort or increment_sort: - sort_key = sort[0] if sort else 'xpos' - if isinstance(sort_key, dict): - if '_script' in sort_key: - sort_key = sort_key['_script']['script']['params'].get('field', 'xpos') # pylint: disable=invalid-sequence-index - else: - sort_key = next(iter(sort_key.keys())) - sort_value = jmespath.search(sort_key, hit['_source']) - if sort_value is None: - sort_value = 'Infinity' - if increment_sort: - sort_value += 100 - hit['_sort'] = [sort_value] - return parsed_hits - - -def create_mock_response(search, index=INDEX_NAME): - index = ALIAS_MAP.get(index, index) - indices = index.split(',') - include_matched_queries = False - variant_id_filters = None - gene_ids_filters = set() - if 'query' in search: - for search_filter in search['query']['bool']['filter']: - if not variant_id_filters: - variant_id_filters = search_filter.get('terms', {}).get('variantId') - if not gene_ids_filters and not search.get('aggs'): - gene_ids_filters.update(search_filter.get('terms', {}).get('geneIds') or []) - possible_inheritance_filters = search_filter.get('bool', {}).get('should', []) + [search_filter] - if any('_name' in possible_filter.get('bool', {}) for possible_filter in possible_inheritance_filters): - include_matched_queries = True - break - - response_dict = { - 'took': 1, - 'hits': {'total': {'value': 5}, 'hits': []} - } - for index_name in sorted(indices): - index_hits = mock_hits( - INDEX_ES_VARIANTS[index_name], include_matched_queries=include_matched_queries, sort=search.get('sort'), - index=index_name) - if variant_id_filters: - index_hits = [hit for hit in index_hits if hit['_id'] in variant_id_filters] - elif gene_ids_filters: - index_hits = [hit for hit in index_hits if any( - gene_ids_filters.intersection({t['gene_id'] for t in hit['_source']['sortedTranscriptConsequences']}) - )] - response_dict['hits']['hits'] += index_hits - - try: - response_dict['hits']['hits'] = sorted(response_dict['hits']['hits'], key=lambda v: v['_sort']) - except (KeyError, TypeError): - pass - - if search.get('aggs'): - index_vars = COMPOUND_HET_INDEX_VARIANTS.get(index, {}) - buckets = [{'key': gene_id, 'doc_count': 3} for gene_id in ['ENSG00000135953', 'ENSG00000228198']] - if search['aggs']['genes']['aggs'].get('vars_by_gene'): - for bucket in buckets: - bucket['vars_by_gene'] = { - 'hits': { - 'hits': mock_hits(index_vars.get(bucket['key'], ES_VARIANTS), increment_sort=True, index=index) - }} - else: - for bucket in buckets: - doc_count = 0 - for sample_field in ['samples', 'samples_num_alt_1', 'samples_num_alt_2']: - gene_samples = defaultdict(int) - for var in index_vars.get(bucket['key'], ES_VARIANTS): - for sample in var['_source'].get(sample_field, []): - gene_samples[sample] += 1 - bucket[sample_field] = {'buckets': [{'key': k, 'doc_count': v} for k, v in gene_samples.items()]} - doc_count += sum(gene_samples.values()) - bucket['doc_count'] = doc_count - - response_dict['aggregations'] = {'genes': {'buckets': buckets}} - - if len(response_dict['hits']['hits']) == 0: - response_dict['hits']['total']['value'] = 0 - elif gene_ids_filters == {'ENSG00000186092'}: - response_dict['hits']['total']['value'] = 1 - - - return response_dict - -def get_indices_from_url(url): - return url.split('/')[1] - -def get_metadata_callback(request): - indices = get_indices_from_url(request.url).split(',') - response = {index: {'mappings': INDEX_METADATA[index]} for index in indices} - return 200, {}, json.dumps(response) - -def get_search_callback(request): - body = json.loads(request.body) - response = create_mock_response(body, get_indices_from_url(request.url)) - return 200, {}, json.dumps(response) - -def parse_msearch_body(body): - return [json.loads(row) for row in body.decode().split('\n') if row] - -def get_msearch_callback(request): - body = parse_msearch_body(request.body) - response = { - 'responses': [ - create_mock_response(exec_search, index=','.join(body[i-1]['index'])) - for i, exec_search in enumerate(body) if not exec_search.get('index')] - } - return 200, {}, json.dumps(response) - -def setup_search_responses(): - urllib3_responses.add_callback( - urllib3_responses.POST, '/_msearch', callback=get_msearch_callback, - content_type='application/json', match_querystring=True) - urllib3_responses.add_callback( - urllib3_responses.POST, re.compile('^/[,\w]+/_search$'), callback=get_search_callback, - content_type='application/json', match_querystring=True) - -def setup_responses(): - urllib3_responses.add_callback( - urllib3_responses.GET, re.compile('^/[,\w]+/_mapping$'), callback=get_metadata_callback, - content_type='application/json', match_querystring=True) - setup_search_responses() - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) -class EsUtilsTest(TestCase): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - Sample.objects.filter(sample_id='NA19678').update(is_active=False) - Sample.objects.filter(sample_id='NA20885').update(sample_type='WES') - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - - def assertExecutedSearch(self, filters=None, start_index=0, size=2, index=INDEX_NAME, expected_source_fields=SOURCE_FIELDS, call_index=-1, **kwargs): - executed_search = urllib3_responses.call_request_json(index=call_index) - searched_indices = get_indices_from_url(urllib3_responses.calls[call_index].request.url) - self.assertListEqual(sorted(searched_indices.split(',')), sorted(index.split(','))) - self.assertSameSearch( - executed_search, - dict(filters=filters, start_index=start_index, size=size, **kwargs), - expected_source_fields=expected_source_fields, - ) - - def assertExecutedSearches(self, searches): - executed_search = parse_msearch_body(urllib3_responses.calls[-1].request.body) - self.assertEqual(len(executed_search), len(searches) * 2) - for i, expected_search in enumerate(searches): - self.assertDictEqual(executed_search[i * 2], {'index': expected_search.get('index', INDEX_NAME).split(',')}) - self.assertSameSearch(executed_search[(i * 2) + 1], expected_search) - - def assertSameSearch(self, executed_search, expected_search_params, expected_source_fields=SOURCE_FIELDS): - expected_search = { - 'from': expected_search_params['start_index'], - 'size': expected_search_params['size'] - } - - if expected_search_params['filters']: - expected_search['query'] = { - 'bool': { - 'filter': expected_search_params['filters'] - } - } - - if expected_search_params.get('query'): - expected_search['query']['bool']['must'] = expected_search_params['query'] - - if not expected_search_params.get('unsorted'): - expected_search['sort'] = expected_search_params.get('sort') or ['xpos', 'variantId'] - - if expected_search_params.get('gene_aggs'): - expected_search['aggs'] = { - 'genes': {'terms': {'field': 'geneIds', 'min_doc_count': 2, 'size': 1001}, 'aggs': { - 'vars_by_gene': { - 'top_hits': {'sort': expected_search['sort'], '_source': mock.ANY, 'size': 100} - } - }}} - elif expected_search_params.get('gene_count_aggs'): - expected_search['aggs'] = {'genes': { - 'terms': {'field': 'mainTranscript_gene_id', 'size': 1001}, - 'aggs': expected_search_params['gene_count_aggs'] - }} - del expected_search['sort'] - else: - expected_search['_source'] = mock.ANY - self.assertDictEqual(executed_search, expected_search) - - if not expected_search_params.get('gene_count_aggs'): - source = executed_search['aggs']['genes']['aggs']['vars_by_gene']['top_hits']['_source'] \ - if expected_search_params.get('gene_aggs') else executed_search['_source'] - self.assertSetEqual(expected_source_fields, set(source)) - - def assertCachedResults(self, results_model, expected_results, sort='xpos'): - cache_key = 'search_results__{}__{}'.format(results_model.guid, sort) - self.assertIn(cache_key, REDIS_CACHE.keys()) - self.assertDictEqual(json.loads(REDIS_CACHE[cache_key]), expected_results) - MOCK_REDIS.expire.assert_called_with(cache_key, timedelta(weeks=2)) - - @urllib3_responses.activate - def test_get_es_variants_for_variant_ids(self): - setup_responses() - samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') - get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], - size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ) - - @urllib3_responses.activate - def test_get_single_es_variant(self): - setup_responses() - family = self.families.filter(guid='F000003_3').first() - variant = get_single_variant(family, '2-103343353-GAGA-G') - self.assertDictEqual(variant, {**PARSED_ANY_AFFECTED_VARIANTS[1], 'selectedMainTranscriptId': None}) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['2-103343353-GAGA-G']}}], - size=1, index=INDEX_NAME - ) - - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, 'prefix_19107_DEL') - self.assertDictEqual(variant, PARSED_SV_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['prefix_19107_DEL']}}], size=1, index=SV_INDEX_NAME, - ) - - variant = get_single_variant(family, 'M-10195-C-A') - self.assertDictEqual(variant, PARSED_MITO_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['M-10195-C-A']}}], size=1, index=MITO_WGS_INDEX_NAME, - ) - - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G') - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_gene_agg_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_search.logger') - @urllib3_responses.activate - def test_invalid_get_es_variants(self, mock_logger): - setup_responses() - mito_sample = Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME) - mito_sample.individual_id = 6 - mito_sample.save() - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - - results_model.families.set(self.families) - Sample.objects.filter(elasticsearch_index=INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), 'The following indices do not have the expected genome version 37: test_index_hg38 (38)', - ) - Sample.objects.filter(elasticsearch_index=HG38_INDEX_NAME).update(elasticsearch_index=INDEX_NAME) - - results_model.families.set(self.families) - search_model.search = { - 'inheritance': {'mode': 'compound_het'}, - 'locus': {'rawItems': 'WASH7P'}, - 'annotations': {'frameshift': ['frameshift_variant']}, - } - search_model.save() - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), - 'This search returned too many compound heterozygous variants. Please add stricter filters') - - with self.assertRaises(InvalidSearchException) as cm: - get_variant_query_gene_counts(results_model, None) - self.assertEqual(str(cm.exception), 'This search returned too many genes') - - search_model.search = {'qualityFilter': {'min_gq': 7}} - search_model.save() - with self.assertRaises(Exception) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Invalid gq filter 7') - - search_model.search = {} - search_model.save() - urllib3_responses.reset() - urllib3_responses.add( - urllib3_responses.POST, '/_msearch', body=ReadTimeoutError('', '', 'timeout')) - urllib3_responses.add_json('/_tasks?actions=*search&group_by=parents', {'tasks': { - 123: {'running_time_in_nanos': 10}, - 456: {'running_time_in_nanos': 10 ** 12}, - }}) - with self.assertRaises(ConnectionTimeout): - query_variants(results_model) - self.assertListEqual( - [call.request.url for call in urllib3_responses.calls], - ['/_msearch', '/_tasks?actions=%2Asearch&group_by=parents']) - mock_logger.error.assert_called_with('ES Query Timeout: Found 1 long running searches', None, detail=[ - {'task': {'running_time_in_nanos': 10 ** 12}, 'parent_task_id': '456'}, - ]) - - urllib3_responses.reset() - urllib3_responses.add_json('/_msearch', {'responses': [ - {'error': {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}} - ]}, method=urllib3_responses.POST) - - with self.assertRaises(TransportError) as cm: - query_variants(results_model) - self.assertDictEqual( - cm.exception.info, - {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}) - - _set_cache(f'index_metadata__{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}', None) - urllib3_responses.add( - urllib3_responses.GET, f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', body=Exception('Connection error')) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'test_index,test_index_mito_wgs,test_index_sv - Error accessing index: Connection error') - - urllib3_responses.replace_json(f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', {}) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') - - @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) - @urllib3_responses.activate - def test_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={'annotations': {'frameshift': ['frameshift_variant']}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_VARIANTS[1]) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY]) - - # does not save non-consecutive pages - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=2) - - # test pagination - variants, total_results = query_variants(results_model, page=2, num_results=2) - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': PARSED_VARIANTS + PARSED_VARIANTS, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=2, size=2) - - # test does not re-fetch page - urllib3_responses.reset() - variants, total_results = query_variants(results_model, page=1, num_results=3) - self.assertEqual(len(variants), 3) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS[:1]) - self.assertEqual(total_results, 5) - - # test load_all - setup_responses() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - # test does not re-fetch once all loaded - urllib3_responses.reset() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - @urllib3_responses.activate - def test_filtered_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'pathogenicity': { - 'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting'], - 'hgmd': ['disease_causing', 'likely_disease_causing'], - }, - 'annotations': { - 'in_frame': ['inframe_insertion', 'inframe_deletion'], - 'other': ['5_prime_UTR_variant', 'intergenic_variant'], - 'SCREEN': ['dELS', 'DNase-only'], - 'splice_ai': '0.8', - }, - 'freqs': { - 'callset': {'af': 0.1}, - 'exac': {'ac': 2, 'af': None}, - 'g1k': {'ac': None, 'af': 0.001}, - 'gnomad_exomes': {'af': 0.01, 'ac': 3, 'hh': 3}, - 'gnomad_genomes': {'af': 0.01, 'hh': 3}, - 'topmed': {'ac': 2, 'af': None}, - }, - 'qualityFilter': {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass', 'affected_only': True}, - 'in_silico': {'cadd': '11.5', 'sift': 'D', 'fathmm': 'D'}, - 'inheritance': {'mode': 'de_novo'}, - 'customQuery': {'term': {'customFlag': 'flagVal'}}, - 'exclude': {'rawItems': 'WASH7P, chr2:1234-5678, chr7:100-10100%10'}, - }) - - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - variants, total_results = query_variants(results_model, sort='cadd', num_results=2) - - self.assertListEqual(variants, PARSED_CADD_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(filters=[ - {'term': {'customFlag': 'flagVal'}}, - { - 'bool': { - 'must_not': [ - {'range': {'xpos': {'gte': 2000001234, 'lte': 2000005678}}}, - {'range': {'xstop': {'gte': 2000001234, 'lte': 2000005678}}}, - {'bool': {'must': [ - {'range': {'xpos': {'lte': 2000001234}}}, - {'range': {'xstop': {'gte': 2000005678}}}, - {'range': {'xpos': {'gte': 2000000001}}}, - {'range': {'xstop': {'lte': 2300000000}}}, - ]}}, - {'terms': {'geneIds': ['ENSG00000227232']}}, - {'bool': {'must': [ - {'range': {'xpos': {'gte': 7000000001, 'lte': 7000001100}}}, - {'range': {'xstop': {'gte': 7000009100, 'lte': 7000011100}}}]}}, - ] - } - }, - {'bool': {'should': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'exac_AC_Adj'}}]}}, - {'range': {'exac_AC_Adj': {'lte': 2}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hom'}}]}}, - {'range': {'gnomad_exomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hemi'}}]}}, - {'range': {'gnomad_exomes_Hemi': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hom'}}]}}, - {'range': {'gnomad_genomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hemi'}}]}}, - {'range': {'gnomad_genomes_Hemi': {'lte': 3}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'topmed_AC'}}]}}, - {'range': {'topmed_AC': {'lte': 2}}} - ]} - } - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }} - ] - }}, - - ]}}, - {'bool': {'should': [ - {'bool': {'must_not': [{'exists': {'field': 'cadd_PHRED'}}]}}, - {'range': {'cadd_PHRED': {'gte': 11.5}}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_SIFT_pred'}}]}}, - {'prefix': {'dbnsfp_SIFT_pred': 'D'}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_fathmm_MKL_coding_pred'}}]}}, - {'prefix': {'dbnsfp_fathmm_MKL_coding_pred': 'D'}}, - ]}}, - {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}}, - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'transcriptConsequenceTerms'}}]}}, - {'terms': { - 'transcriptConsequenceTerms': [ - '5_prime_UTR_variant', - 'inframe_deletion', - 'inframe_insertion', - 'intergenic_variant', - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*|Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))', - }}, - {'terms': {'hgmd_class': ['DM', 'DM?']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.8}}}, - {'terms': {'screen_region_type': ['dELS', 'DNase-only']}}, - ] - } - }, - {'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - }}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'HG00731'}}, - {'term': {'samples_ab_5_to_10': 'HG00731'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'HG00731'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_10_to_15': 'HG00731'}}, - ], - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*' - }}]}} - ], - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}} - ]}}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'NA20870'}}, - {'term': {'samples_ab_5_to_10': 'NA20870'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'NA20870'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - {'term': {'samples_gq_10_to_15': 'NA20870'}}, - ] - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }}]}} - ], - '_name': 'F000003_3' - }}, - ] - }} - ], sort=[{'cadd_PHRED': {'order': 'desc', 'unmapped_type': 'keyword'}}, 'xpos', 'variantId']) - - # Test sort does not error on pagination - query_variants(results_model, sort='cadd', num_results=2, page=2) - - @urllib3_responses.activate - def test_sv_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'new_structural_variants': ['NEW']}, - 'freqs': {'sv_callset': {'af': 0.1}}, - 'in_silico': {'strvctvre': '3.1', 'requireScore': True}, - 'qualityFilter': {'min_qs': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - - self.assertListEqual(variants, [PARSED_SV_VARIANT]) - self.assertExecutedSearch(filters=[ - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'sf'}}]}}, - {'range': {'sf': {'lte': 0.1}}} - ] - }}, - {'range': {'StrVCTVRE_score': {'gte': 3.1}}}, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_qs_0_to_10': 'HG00731'}}, - {'term': {'samples_qs_10_to_20': 'HG00731'}}, - {'term': {'samples_qs_0_to_10': 'HG00732'}}, - {'term': {'samples_qs_10_to_20': 'HG00732'}}, - {'term': {'samples_qs_0_to_10': 'HG00733'}}, - {'term': {'samples_qs_10_to_20': 'HG00733'}}, - ], - 'must': [{'terms': {'samples_new_call': ['HG00731', 'HG00732', 'HG00733']}}], - }}, - ], - '_name': 'F000002_2' - }} - ], index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_sv_wgs_get_es_variants(self): - self.families = Family.objects.filter(guid='F000014_14') - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DUP', 'CPX']}, - 'qualityFilter': {'min_gq_sv': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - self.assertListEqual(variants, [PARSED_SV_WGS_VARIANT]) - - self.assertExecutedSearch(filters=[ - {'terms': {'transcriptConsequenceTerms': ['CPX', 'DUP']}}, - {'bool': { - 'must': [{'bool': {'must': [{'term': {'samples': 'NA21234'}}, {'term': {'samples': 'NA21987'}}], - 'must_not': [{'term': {'samples': 'NA21654'}}]}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_sv_0_to_5': 'NA21234'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21234'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21234'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21234'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21654'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21654'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21654'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21654'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21987'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21987'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21987'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21987'}}, - ], - }} - ], - '_name': 'F000014_14' - }} - ], index=SV_WGS_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_get_es_variants(self): - setup_responses() - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, [PARSED_SV_VARIANT] + PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - self.assertExecutedSearches([ - dict(filters=[], start_index=0, size=5, index=SV_INDEX_NAME), - dict(filters=[], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['pathogenicity'] = {'clinvar': ['pathogenic']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - path_filter = {'regexp': { - 'clinvar_clinical_significance': '.*Pathogenic.*' - }} - self.assertExecutedSearches([ - dict(filters=[path_filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[path_filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - # test with dataset filtering applied - search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - filter = {'bool': {'should': [{'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}}, path_filter]}} - self.assertExecutedSearches([ - dict(filters=[filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['annotations'] = { - 'structural': ['DEL'], 'structural_consequence': ['MSV_EXON_OVERLAP', 'INTRAGENIC_EXON_DUP']} - search_model.search['pathogenicity'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - self.assertExecutedSearch( - filters=[{'terms': { - 'transcriptConsequenceTerms': ['DEL', 'DUP_LOF', 'INTRAGENIC_EXON_DUP', 'MSV_EXON_OVERLAP', 'MSV_EXON_OVR'] - }}], - start_index=0, size=5, index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_no_affected_inheritance_get_es_variants(self): - setup_responses() - # The family has multiple data types loaded but only one loaded in an affected individual - Sample.objects.get(individual_id=4, elasticsearch_index=SV_INDEX_NAME).delete() - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - - search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - - query_variants(results_model, num_results=2) - self.assertExecutedSearch(filters=[{'bool': { - 'must': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], - }}], '_name': 'F000002_2', - }}]) - - @urllib3_responses.activate - def test_compound_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_compound_het_get_es_variants_secondary_annotation(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant'], 'splice_ai': '0.5'}, - 'inheritance': {'mode': 'compound_het'}, - 'annotations_secondary': {'other': ['intron']}, - 'pathogenicity': {'clinvar': ['pathogenic'], 'hgmd': ['disease_causing']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - annotation_query = {'bool': {'should': [ - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}}, - {'terms': {'hgmd_class': ['DM']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.5}}}, - {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}}, - ]}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_PATH_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - # variants require both primary and secondary annotations - setup_responses() - del search_model.search['pathogenicity'] - del search_model.search['annotations']['splice_ai'] - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertIsNone(variants) - self.assertEqual(total_results, 0) - - annotation_query = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - @urllib3_responses.activate - def test_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10, 'vcf_filter': 'pass'}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS[1]) - self.assertEqual(total_results, 6) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 6, - }) - - pass_filter_query = {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}} - - self.assertExecutedSearches([ - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=0, size=2, - ), - ]) - - # test pagination - - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_MULTI_SAMPLE_VARIANT) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}, - {'null': [PARSED_VARIANTS[0]]}, {'null': [PARSED_MULTI_SAMPLE_VARIANT]}], - 'duplicate_doc_count': 1, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 4, 'total': 5}}, - 'total_results': 5, - }) - - self.assertExecutedSearches([dict(filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=2, size=4)]) - - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_multi_datatype_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'recessive'}, - 'annotations': {'frameshift': ['frameshift_variant'], 'structural': ['DEL']} - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 4) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[2][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[2][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - self.assertDictEqual(variants[3], PARSED_VARIANTS[1]) - - annotations_q = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - self.assertExecutedSearches([ - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotations_q, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - annotations_q, - { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - }}, - ] - } - } - ], start_index=0, size=10, - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL']}, - 'annotations_secondary': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - - annotation_secondary_query = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - - self.assertExecutedSearches([ - dict( - filters=[ - {'terms': {'transcriptConsequenceTerms': ['DEL']}}, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_comp_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL'], 'SCREEN': ['dELS']}, - 'annotations_secondary': {'structural_consequence': ['LOF']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - query_variants(results_model, num_results=10) - - annotation_secondary_query = {'bool': {'should': [ - {'terms': {'transcriptConsequenceTerms': ['DEL', 'LOF']}}, - {'terms': {'screen_region_type': ['dELS']}}]}} - - self.assertExecutedSearches([ - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(index=INDEX_NAME, size=2, filters=[ - {'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G']}}, ANNOTATION_QUERY]) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_ANY_AFFECTED_VARIANTS) - self.assertEqual(total_results, 10) - - self.assertExecutedSearches([dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], index=MITO_WGS_INDEX_NAME, start_index=0, size=2), dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], index=INDEX_NAME, start_index=0, size=2)]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_SEARCH_CLAUSES', 1) - @urllib3_responses.activate - def test_many_family_inheitance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 14) - self.assertDictEqual(variants[0], PARSED_MULTI_SAMPLE_VARIANT_0) - self.assertListEqual(variants[1], PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_VARIANT, PARSED_MITO_VARIANT], - 'grouped_results': [{'null': [PARSED_MULTI_SAMPLE_VARIANT_0]}, {'ENSG00000228198': PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, - MITO_WGS_INDEX_NAME: {'loaded': 1, 'total': 5}, - INDEX_NAME: {'loaded': 4, 'total': 10}}, - 'total_results': 14, - }) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - ] - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}], - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - } - }, - ], - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - } - }, - ], - start_index=0, - size=2, - ), - ]) - - @urllib3_responses.activate - def test_multi_project_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2'])) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[1]) - self.assertEqual(total_results, 11) - - self.assertCachedResults(results_model, { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 2, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 11, - }) - self.assertTrue('index_metadata__{}'.format(INDEX_NAME) in REDIS_CACHE) - self.assertTrue('index_metadata__{}'.format(SECOND_INDEX_NAME) in REDIS_CACHE) - - project_2_search = dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME) - project_1_search = dict( - filters=[ - ANNOTATION_QUERY, - RECESSIVE_INHERITANCE_QUERY, - ], start_index=0, size=2, index=INDEX_NAME) - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - '_name': 'F000011_11', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}} - ]}} - ] - }} - ], - gene_aggs=True, start_index=0, size=1, index=SECOND_INDEX_NAME, - ), - project_2_search, - dict( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, start_index=0, size=1, index=INDEX_NAME, - ), - project_1_search, - ]) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_VARIANTS[0], PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) - self.assertEqual(total_results, 9) - - cache_results = { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}, - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT} - ], - 'duplicate_doc_count': 4, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 4, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 9, - } - self.assertCachedResults(results_model, cache_results) - - project_2_search['start_index'] = 1 - project_2_search['size'] = 3 - project_1_search['start_index'] = 2 - self.assertExecutedSearches([project_2_search, project_1_search]) - - # If one project is fully loaded, only query the second project - cache_results['loaded_variant_counts'][INDEX_NAME]['total'] = 4 - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(cache_results)) - query_variants(results_model, num_results=2, page=3) - project_2_search['start_index'] = 2 - project_2_search['size'] = 4 - self.assertExecutedSearches([project_2_search]) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_MULTI_INDEX_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 4) - - cached_variants = expected_variants + [PARSED_MITO_VARIANT] - self.assertCachedResults(results_model, { - 'all_results': cached_variants, - 'duplicate_doc_count': 1, - 'total_results': 4, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME, MITO_WGS_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=6, - ) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - expected_variants = [PARSED_MITO_VARIANT, PARSED_VARIANTS[0]] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 3) - - self.assertCachedResults(results_model, { - 'all_results': cached_variants + cached_variants, - 'duplicate_doc_count': 2, - 'total_results': 3, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=8, - start_index=4, - ) - - # test skipping page fetches all consecutively - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, page=2) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=12, - ) - - @urllib3_responses.activate - def test_multi_project_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }, - ) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 14) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_SEARCHES', 1) - @urllib3_responses.activate - def test_multi_project_prefilter_indices_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'de_novo'}, - 'qualityFilter': {'min_gq': 10}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 14) - self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) - - gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} - prefilter_search = dict( - filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', - size=200, expected_source_fields=set(), - ) - sv_search = dict( - filters=[ - gene_filter, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }} - ], start_index=0, size=2, index=SV_INDEX_NAME) - num_calls = 3 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - # Search total is greater than returned hits, so proceed with regular multi-search - self.assertExecutedSearches([ - sv_search, - dict(filters=[ - gene_filter, - { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict(filters=[ - gene_filter, - {'bool': {'should': [ - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA19675'}}, - {'term': {'samples_num_alt_2': 'NA19675'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA19675'}}, - {'term': {'samples_gq_5_to_10': 'NA19675'}}, - ]}}], '_name': 'F000001_1'}}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], 'minimum_should_match': 1}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - ]}}], '_name': 'F000003_3'}}, - ]}}], start_index=0, size=2, index=INDEX_NAME), - ]) - - # Test successful prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000186092'] - search_model.search['locus']['rawItems'] = 'ENSG00000186092' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 1) - num_calls += 2 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - sv_search['query'] = [{'ids': {'values': ['prefix_19107_DEL']}}] - self.assertExecutedSearches([sv_search]) - - # Test no results in prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000269732'] - search_model.search['locus']['rawItems'] = 'ENSG00000269732' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 0) - # Only the prefliter search is run, no multi-search - self.assertEqual(len(urllib3_responses.calls), num_calls + 1) - self.assertExecutedSearch(**prefilter_search) - - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_VARIANTS', 3) - @urllib3_responses.activate - def test_skip_genotype_filter(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, _ = query_variants(results_model, num_results=2, skip_genotype_filter=True) - expected_transcript_variant = deepcopy(PARSED_VARIANTS[0]) - expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] - self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), - filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], - size=3, - ) - - # test with inheritance override - search_model.search['inheritance'] = {'mode': 'any_affected'} - search_model.search['locus'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, skip_genotype_filter=True) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.utils.LIFTOVERS', {'37': None}) - @mock.patch('seqr.utils.search.utils.LiftOver') - @urllib3_responses.activate - def test_get_lifted_grch38_variants(self, mock_liftover): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - Project.objects.filter(guid='R0003_test').update(genome_version='38') - Sample.objects.filter(elasticsearch_index=SECOND_INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - - mock_liftover.side_effect = Exception() - expected_no_lift_grch38_variant = deepcopy(PARSED_HG38_VARIANT) - expected_no_lift_grch38_variant.update({ - 'liftedOverGenomeVersion': None, - 'liftedOverChrom': None, - 'liftedOverPos': None, - }) - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, expected_no_lift_grch38_variant]) - self.assertIsNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - mock_liftover.side_effect = None - mock_liftover.return_value.convert_coordinate.side_effect = lambda chrom, pos: [[chrom, pos - 10]] - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, PARSED_HG38_VARIANT]) - self.assertIsNotNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_NAME_LENGTH', 30) - @urllib3_responses.activate - def test_get_es_variants_create_index_alias(self): - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - setup_search_responses() - urllib3_responses.add_json( - '/{}/_mapping'.format(INDEX_ALIAS), {k: {'mappings': v} for k, v in CORE_INDEX_METADATA.items()}) - urllib3_responses.add_json('/_aliases', {'success': True}, method=urllib3_responses.POST) - - query_variants(results_model, num_results=2) - - self.assertExecutedSearch(index=INDEX_ALIAS, size=8) - self.assertDictEqual(urllib3_responses.call_request_json(index=0), { - 'actions': [{'add': {'indices': [INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME], 'alias': INDEX_ALIAS}}]}) - - @urllib3_responses.activate - def test_get_es_variants_search_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - setup_search_responses() - urllib3_responses.add_json('/{}/_mapping'.format(SECOND_INDEX_NAME), { - k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SUB_INDICES}) - - query_variants(results_model, num_results=2) - - expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - self.assertExecutedSearches([ - dict(index=SUB_INDICES[1], **expected_search), - dict(index=SUB_INDICES[0], **expected_search), - ]) - _set_cache('index_metadata__{}'.format(SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variants_search_multiple_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000011_11'])) - - setup_search_responses() - mappings = {k: {'mappings': INDEX_METADATA[INDEX_NAME]} for k in SUB_INDICES} - mappings.update({k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_mapping'.format(INDEX_NAME, SECOND_INDEX_NAME), mappings) - aliases = {k: {'aliases': {INDEX_NAME: {}}} for k in SUB_INDICES} - aliases.update({k: {'aliases': {SECOND_INDEX_NAME: {}, INDEX_ALIAS: {}}} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_alias'.format(INDEX_NAME, SECOND_INDEX_NAME), aliases) - - query_variants(results_model, num_results=2) - - second_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - first_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}} - ], - '_name': 'F000003_3' - }}] - } - self.assertExecutedSearches([ - dict(index=SECOND_SUB_INDICES[1], **second_alias_expected_search), - dict(index=SECOND_SUB_INDICES[0], **second_alias_expected_search), - dict(index=SUB_INDICES[1], **first_alias_expected_search), - dict(index=SUB_INDICES[0], **first_alias_expected_search), - ]) - _set_cache('index_metadata__{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000240361': PARSED_COMPOUND_HET_VARIANTS}], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 7, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000240361': {'total': 2, 'families': {'F000003_3': 2}}, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], - size=1, index=INDEX_NAME, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 4, 'loaded': 4}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 2, 'loaded': 2}, - }, - 'total_results': 13, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 4, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}} - }) - - expected_search = dict(size=1, start_index=0, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - self.assertExecutedSearches([ - dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], index=SECOND_INDEX_NAME, **expected_search), - dict(filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], index=INDEX_NAME, **expected_search), - ]) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps({'total_results': 5})) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}}, - 'ENSG00000228198': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}} - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts, 'total_results': 5}) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}}, - 'ENSG00000228198': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}} - }) - - self.assertExecutedSearches([dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ), dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - )]) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts}) - - def test_get_family_affected_status(self): - samples_by_id = { - sample_id: Sample.objects.get(sample_id=sample_id, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - for sample_id in ['HG00731', 'HG00732', 'HG00733'] - } - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - self.assertDictEqual(_get_family_affected_status(samples_by_id, {}), { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'N', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_multi_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - @urllib3_responses.activate - def test_sort(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, _ = query_variants(results_model, sort='primate_ai', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], sort=[ - {'primate_ai_score': {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}, 'xpos', 'variantId'], - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4) - self.assertEqual(variants[0]['_sort'][0], maxsize) - self.assertEqual(variants[1]['_sort'][0], -1) - - variants, _ = query_variants(results_model, sort='gnomad', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': {'field': 'gnomad_genomes_AF'}, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - self.assertEqual(variants[0]['_sort'][0], 0.00012925741614425127) - self.assertEqual(variants[1]['_sort'][0], maxsize) - - variants, _ = query_variants(results_model, sort='in_omim', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - results_model.families.set(Family.objects.filter(guid='F000001_1')) - query_variants(results_model, sort='prioritized_gene', num_results=2) - family_sample_filter = {'bool': {'_name': 'F000001_1', 'must': mock.ANY}} - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, family_sample_filter], index=INDEX_NAME, size=2, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'prioritized_ranks_by_gene': {'ENSG00000268903': 1, 'ENSG00000268904': 11} - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - @urllib3_responses.activate - def test_genotype_inheritance_filter(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - cache_key = 'search_results__{}__xpos'.format(results_model.guid) - - def _execute_inheritance_search( - mode=None, inheritance_filter=None, expected_filter=None, expected_comp_het_filter=None, - quality_filter=None, expected_quality_filter=None, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, **kwargs): - _set_cache(cache_key, None) - annotations = {'frameshift': ['frameshift_variant']} if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS \ - else {'structural': ['DEL', 'gCNV_DEL']} - search_model.search = { - 'inheritance': {'mode': mode, 'filter': inheritance_filter}, - 'annotations': annotations, - } - if quality_filter: - search_model.search['qualityFilter'] = quality_filter - search_model.save() - variants, _ = query_variants(results_model, num_results=2) - - if mode not in {'compound_het', 'recessive'}: - self.assertSetEqual( - set(variants[-1]['genotypes'].keys()), - {'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'}, - ) - - index = INDEX_NAME if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else SV_INDEX_NAME - annotation_query = {'terms': {'transcriptConsequenceTerms': next(iter(annotations.values()))}} - if expected_comp_het_filter: - self.assertExecutedSearches([ - dict(gene_aggs=True, start_index=0, size=1, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_comp_het_filter]}} - ]), - dict(start_index=0, size=2, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_filter]}}]) - ]) - else: - filters = [expected_filter] - if expected_quality_filter: - filters.append(expected_quality_filter) - self.assertExecutedSearch(index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': filters}}], **kwargs) - - # custom genotype - _execute_inheritance_search( - inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref', 'I000005_hg00732': 'ref_alt'}}, - expected_filter={ - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}} - ] - } - }) - - # de novo - _execute_inheritance_search(mode='de_novo', expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', dataset_type='SV', expected_filter={'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }, quality_filter={'affected_only': True, 'min_gq': 10}, expected_quality_filter={ - 'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - ]}} - ) - - _execute_inheritance_search( - mode='de_novo', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ], - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ]} - }] - } - }) - - recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - } - sv_recessive_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - } - custom_affected_recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - } - - # homozygous recessive - _execute_inheritance_search(mode='homozygous_recessive', expected_filter=recessive_filter) - - _execute_inheritance_search(mode='homozygous_recessive', dataset_type='SV', expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='homozygous_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_recessive_filter) - - # compound het - com_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - } - - sv_com_het_filter = {'term': {'samples': 'HG00731'}} - - custom_affected_comp_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='compound_het', expected_filter=com_het_filter, gene_aggs=True, size=1) - _execute_inheritance_search( - mode='compound_het', dataset_type='SV', gene_aggs=True, size=1, expected_filter=sv_com_het_filter) - _execute_inheritance_search( - mode='compound_het', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_comp_het_filter, gene_aggs=True, size=1) - - # x-linked recessive - x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - } - - sv_x_linked_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ], - 'must': [{'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}},], - } - } - - custom_affected_x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='x_linked_recessive', expected_filter=x_linked_filter) - - _execute_inheritance_search(mode='x_linked_recessive', dataset_type='SV', expected_filter=sv_x_linked_filter) - - _execute_inheritance_search( - mode='x_linked_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_x_linked_filter) - - # recessive - _execute_inheritance_search(mode='recessive', expected_comp_het_filter=com_het_filter, expected_filter=recessive_filter) - - _execute_inheritance_search( - mode='recessive', dataset_type='SV', expected_comp_het_filter=sv_com_het_filter, expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='recessive', inheritance_filter={'affected': custom_affected}, - expected_comp_het_filter=custom_affected_comp_het_filter, expected_filter=custom_affected_recessive_filter) - - # any affected - _execute_inheritance_search(mode='any_affected', expected_filter={ - 'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - } - }) - - _execute_inheritance_search( - mode='any_affected', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732']}}, - {'terms': {'samples': ['HG00731', 'HG00732']}}, - ] - } - }) From cb470b699b1710a2ff9700fedb1f19d65b86e8e0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:31:52 -0500 Subject: [PATCH 019/252] fix tests --- seqr/views/apis/variant_search_api_tests.py | 43 +-------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index ca1ec7f853..7a9cc23bf1 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -5,13 +5,11 @@ from django.db import transaction from django.urls.base import reverse -from elasticsearch.exceptions import ConnectionTimeout, TransportError from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, \ SV_LOOKUP_VARIANT, SV_VARIANT4, GCNV_VARIANT4 from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch from seqr.utils.search.utils import InvalidSearchException -from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException from seqr.views.apis.variant_search_api import query_variants_handler, query_single_variant_handler, vlm_lookup_handler, \ export_variants_handler, search_context_handler, get_saved_search_handler, create_saved_search_handler, \ update_saved_search_handler, delete_saved_search_handler, get_variant_gene_breakdown, variant_lookup_handler @@ -408,16 +406,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertEqual(response.reason_phrase, 'Invalid search: no projects/ families specified') mock_error_logger.assert_not_called() - mock_get_variants.side_effect = InvalidIndexException('Invalid index') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Invalid index') - self.assertFalse('traceback' in response.json()) - mock_error_logger.assert_called_with( - 'Invalid index', self.collaborator_user, http_request_json=mock.ANY, traceback=mock.ANY, request_body=mock.ANY, detail=None) - mock_get_variants.side_effect = InvalidSearchException('Invalid search') mock_error_logger.reset_mock() response = self.client.post(url, content_type='application/json', data=json.dumps({ @@ -427,35 +415,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertEqual(response.json()['error'], 'Invalid search') mock_error_logger.assert_not_called() - mock_get_variants.side_effect = ConnectionTimeout('', '', ValueError('Timeout')) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 504) - self.assertEqual(response.json()['error'], 'ConnectionTimeout caused by - ValueError(Timeout)') - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = TransportError('N/A', 'search_phase_execution_exception', {'error': 'Invalid'}) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], "TransportError: N/A - 'search_phase_execution_exception' - 'Invalid'") - self.assertEqual(response.json()['detail'], {'error': 'Invalid'}) - mock_error_logger.assert_not_called() - - error_info_json = {'error': {'root_cause': [{'type': 'response_handler_failure_transport_exception'}]}} - mock_get_variants.side_effect = TransportError('401', 'search_phase_execution_exception', error_info_json) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 401) - self.assertEqual( - response.json()['error'], - "TransportError: 401 - 'search_phase_execution_exception' - response_handler_failure_transport_exception") - self.assertEqual(response.json()['detail'], error_info_json) - mock_error_logger.assert_not_called() - mock_get_variants.side_effect = _get_es_variants # Test new search @@ -893,7 +852,7 @@ def test_query_single_variant(self, mock_get_variant): self.assertEqual(response.status_code, 200) self._assert_expected_single_variant_results_context(response.json(), searchedVariants=[SINGLE_FAMILY_VARIANT]) - mock_get_variant.assert_called_with(mock.ANY, '21-3343353-GAGA-G', user=self.collaborator_user) + mock_get_variant.assert_called_with(mock.ANY, '21-3343353-GAGA-G') searched_family = mock_get_variant.call_args.args[0] self.assertEqual(searched_family.guid, 'F000001_1') From 9266d90da842860e2fa8e625bd7c26883da41c5f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:39:22 -0500 Subject: [PATCH 020/252] Revert "remove utility tests" This reverts commit 9b6df59be91d459f52bf3e18816a595762c9a6e8. --- .../search/elasticsearch/es_utils_tests.py | 3796 +++++++++++++++++ 1 file changed, 3796 insertions(+) create mode 100644 seqr/utils/search/elasticsearch/es_utils_tests.py diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py new file mode 100644 index 0000000000..85651ec04f --- /dev/null +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -0,0 +1,3796 @@ +from copy import deepcopy +import mock +import jmespath +import json +import re +from collections import defaultdict +from datetime import timedelta +from django.test import TestCase +from elasticsearch.exceptions import ConnectionTimeout, TransportError +from sys import maxsize +from urllib3.exceptions import ReadTimeoutError +from urllib3_mock import Responses + +from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults +from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids +from seqr.utils.search.utils import get_single_variant, query_variants, \ + get_variant_query_gene_counts, InvalidSearchException +from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status +from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException +from seqr.utils.search.utils import _get_liftover +from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_SV_WGS_VARIANT,\ + PARSED_MITO_VARIANT, TRANSCRIPT_2, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT + + +# The responses library for mocking requests does not work with urllib3 (which is used by elasticsearch) +# The urllib3_mock library works for those requests, but it has limited functionality, so this extension adds helper +# methods for easier usage +class Urllib3Responses(Responses): + def add_json(self, url, json_response, method=None, match_querystring=True, **kwargs): + if not method: + method = self.GET + body = json.dumps(json_response) + self.add(method, url, match_querystring=match_querystring, content_type='application/json', body=body, **kwargs) + + def replace_json(self, url, *args, **kwargs): + existing_index = next(i for i, match in enumerate(self._urls) if match['url'] == url) + self.add_json(url, *args, **kwargs) + self._urls[existing_index] = self._urls.pop() + + def call_request_json(self, index=-1): + return json.loads(self.calls[index].request.body) + + def activate(self, func): + def wrapper(*args, **kwargs): + with self: + return func(*args, **kwargs) + return wrapper + +urllib3_responses = Urllib3Responses() + + +INDEX_NAME = 'test_index' +SECOND_INDEX_NAME = 'test_index_second' +HG38_INDEX_NAME = 'test_index_hg38' +SV_INDEX_NAME = 'test_index_sv' +SV_WGS_INDEX_NAME = 'test_index_sv_wgs' +MITO_WGS_INDEX_NAME = 'test_index_mito_wgs' +INDEX_ALIAS = '377e97bd791cf92a78296bc184e0976a' +ALIAS_MAP = {INDEX_ALIAS: ','.join([INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME, MITO_WGS_INDEX_NAME])} +SUB_INDICES = ['sub_index_1', 'sub_index_2'] +SECOND_SUB_INDICES = ['sub_index_a', 'sub_index_b'] + +ES_VARIANTS = [ + { + '_source': { + 'homozygote_count': 3, + 'gnomad_exomes_Hemi': None, + 'originalAltAlleles': [ + '1-248367227-TC-T' + ], + 'hgmd_accession': None, + 'g1k_AF': None, + 'gnomad_genomes_Hom': 0, + 'cadd_PHRED': '25.9', + 'splice_ai_delta_score': 0.75, + 'exac_AC_Hemi': None, + 'g1k_AC': None, + 'topmed_AN': 125568, + 'g1k_AN': None, + 'topmed_AF': 0.00016724, + 'dbnsfp_MutationTaster_pred': None, + 'ref': 'TC', + 'exac_AC_Hom': 0, + 'topmed_AC': 21, + 'dbnsfp_REVEL_score': None, + 'dbnsfp_VEST4_score': '0.335;0.341;0.38', + 'primate_ai_score': None, + 'variantId': '1-248367227-TC-T', + 'sortedTranscriptConsequences': [ + { + 'amino_acids': 'LL/L', + 'biotype': 'nonsense_mediated_decay', + 'lof': None, + 'lof_flags': None, + 'major_consequence_rank': 10, + 'codons': 'ctTCTc/ctc', + 'gene_symbol': 'MFSD9', + 'domains': [ + 'Transmembrane_helices:TMhelix', + 'Gene3D:1', + ], + 'canonical': None, + 'transcript_rank': 1, + 'cdna_end': 143, + 'lof_filter': None, + 'hgvs': 'ENSP00000413641.1:p.Leu48del', + 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', + 'cdna_start': 141, + 'transcript_id': 'ENST00000428085', + 'protein_id': 'ENSP00000413641', + 'category': 'missense', + 'gene_id': 'ENSG00000135953', + 'hgvsp': 'ENSP00000413641.1:p.Leu48del', + 'major_consequence': 'frameshift_variant', + 'consequence_terms': [ + 'frameshift_variant', + 'inframe_deletion', + 'NMD_transcript_variant' + ] + }, + { + 'amino_acids': 'P/X', + 'biotype': 'protein_coding', + 'lof': None, + 'lof_flags': None, + 'major_consequence_rank': 4, + 'codons': 'Ccc/cc', + 'gene_symbol': 'OR2M3', + 'domains': [ + 'Transmembrane_helices:TMhelix', + 'Prints_domain:PR00237', + ], + 'canonical': 1, + 'transcript_rank': 0, + 'cdna_end': 897, + 'lof_filter': None, + 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', + 'hgvsc': 'ENST00000456743.1:c.862delC', + 'cdna_start': 897, + 'transcript_id': 'ENST00000456743', + 'protein_id': 'ENSP00000389625', + 'category': 'lof', + 'gene_id': 'ENSG00000228198', + 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', + 'major_consequence': 'frameshift_variant', + 'consequence_terms': [ + 'frameshift_variant' + ] + } + ], + 'screen_region_type' : [ + 'dELS', + 'CTCF-bound' + ], + 'gnomad_non_coding_constraint_z_score': 1.01272, + 'hgmd_class': 'DM', + 'AC': 2, + 'exac_AN_Adj': 121308, + 'mpc_MPC': None, + 'AF': 0.063, + 'alt': 'T', + 'clinvar_clinical_significance': 'Pathogenic/Likely_pathogenic', + 'rsid': None, + 'dbnsfp_DANN_score': None, + 'AN': 32, + 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': 0.0004590314436538903, + 'gnomad_genomes_FAF_AF': 0.000437, + 'exac_AF': 0.00006589, + 'dbnsfp_GERP_RS': None, + 'dbnsfp_SIFT_pred': None, + 'exac_AC_Adj': 8, + 'g1k_POPMAX_AF': None, + 'topmed_Hom': 0, + 'gnomad_genomes_AN': 30946, + 'dbnsfp_MetaSVM_pred': None, + 'dbnsfp_Polyphen2_HVAR_pred': None, + 'clinvar_allele_id': None, + 'gnomad_exomes_Hom': 0, + 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.0009151523074911753, + 'gnomad_genomes_Hemi': None, + 'xpos': 1248367227, + 'start': 248367227, + 'filters': [], + 'dbnsfp_phastCons100way_vertebrate': None, + 'gnomad_exomes_AN': 245930, + 'contig': '1', + 'clinvar_gold_stars': None, + 'eigen_Eigen_phred': None, + 'exac_AF_POPMAX': 0.0006726888333653661, + 'gnomad_exomes_AC': 16, + 'dbnsfp_FATHMM_pred': 'T', + 'dbnsfp_fathmm_MKL_coding_pred': 'D', + 'gnomad_exomes_AF': 0.00006505916317651364, + 'gnomad_genomes_AF': 0.00012925741614425127, + 'gnomad_genomes_AC': 4, + 'genotypes': [ + { + 'num_alt': 2, + 'ab': 1, + 'dp': 74, + 'gq': 99, + 'sample_id': 'NA20870', + }, + { + 'num_alt': 0, + 'ab': 0, + 'dp': 88, + 'gq': 99, + 'sample_id': 'HG00731', + }, + { + 'num_alt': 1, + 'ab': 0.631, + 'dp': 50, + 'gq': 99, + 'sample_id': 'NA20885', + }, + ], + 'samples_num_alt_1': ['NA20885'], + 'samples_num_alt_2': ['NA20870'], + }, + 'matched_queries': {INDEX_NAME: ['F000003_3'], SECOND_INDEX_NAME: ['F000011_11']}, + }, + { + '_source': { + 'gnomad_exomes_Hemi': None, + 'originalAltAlleles': [ + '2-103343353-GAGA-G' + ], + 'hgmd_accession': None, + 'g1k_AF': None, + 'gnomad_genomes_Hom': None, + 'cadd_PHRED': None, + 'exac_AC_Hemi': None, + 'g1k_AC': None, + 'topmed_AN': None, + 'g1k_AN': None, + 'topmed_AF': None, + 'dbnsfp_MutationTaster_pred': None, + 'ref': 'GAGA', + 'exac_AC_Hom': 0, + 'topmed_AC': None, + 'dbnsfp_REVEL_score': None, + 'primate_ai_score': 1, + 'variantId': '2-103343353-GAGA-G', + 'sortedTranscriptConsequences': [ + { + 'amino_acids': 'LL/L', + 'biotype': 'protein_coding', + 'lof': None, + 'lof_flags': None, + 'major_consequence_rank': 10, + 'codons': 'ctTCTc/ctc', + 'gene_symbol': 'MFSD9', + 'domains': [ + 'Transmembrane_helices:TMhelix', + 'PROSITE_profiles:PS50850', + ], + 'canonical': 1, + 'transcript_rank': 0, + 'cdna_end': 421, + 'lof_filter': None, + 'hgvs': 'ENSP00000258436.5:p.Leu126del', + 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', + 'cdna_start': 419, + 'transcript_id': 'ENST00000258436', + 'protein_id': 'ENSP00000258436', + 'category': 'missense', + 'gene_id': 'ENSG00000135953', + 'hgvsp': 'ENSP00000258436.5:p.Leu126del', + 'major_consequence': 'inframe_deletion', + 'consequence_terms': [ + 'inframe_deletion' + ] + }, + { + 'amino_acids': 'P/X', + 'biotype': 'protein_coding', + 'lof': None, + 'lof_flags': None, + 'major_consequence_rank': 4, + 'codons': 'Ccc/cc', + 'gene_symbol': 'OR2M3', + 'domains': [ + 'Transmembrane_helices:TMhelix', + 'Prints_domain:PR00237', + ], + 'canonical': 1, + 'transcript_rank': 0, + 'cdna_end': 897, + 'lof_filter': None, + 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', + 'hgvsc': 'ENST00000456743.1:c.862delC', + 'cdna_start': 897, + 'transcript_id': 'ENST00000456743', + 'protein_id': 'ENSP00000389625', + 'category': 'lof', + 'gene_id': 'ENSG00000228198', + 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', + 'major_consequence': 'frameshift_variant', + 'consequence_terms': [ + 'frameshift_variant' + ] + } + ], + 'hgmd_class': None, + 'AC': 1, + 'exac_AN_Adj': 121336, + 'mpc_MPC': None, + 'AF': 0.031, + 'alt': 'G', + 'clinvar_clinical_significance': None, + 'rsid': None, + 'dbnsfp_DANN_score': None, + 'AN': 32, + 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': None, + 'gnomad_genomes_FAF_AF': None, + 'exac_AF': 0.00004942, + 'dbnsfp_GERP_RS': None, + 'dbnsfp_SIFT_pred': None, + 'exac_AC_Adj': 6, + 'g1k_POPMAX_AF': None, + 'topmed_Hom': None, + 'gnomad_genomes_AN': None, + 'dbnsfp_MetaSVM_pred': None, + 'dbnsfp_Polyphen2_HVAR_pred': None, + 'dbnsfp_VEST4_score': '.;.;.;.', + 'clinvar_allele_id': None, + 'gnomad_exomes_Hom': 0, + 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.00016269686320447742, + 'gnomad_genomes_Hemi': None, + 'xpos': 2103343353, + 'start': 103343353, + 'filters': [], + 'dbnsfp_phastCons100way_vertebrate': None, + 'gnomad_exomes_AN': 245714, + 'contig': '2', + 'clinvar_gold_stars': None, + 'eigen_Eigen_phred': None, + 'exac_AF_POPMAX': 0.000242306760358614, + 'gnomad_exomes_AC': 6, + 'dbnsfp_FATHMM_pred': None, + 'gnomad_exomes_AF': 0.000024418633044922146, + 'gnomad_genomes_AF': None, + 'gnomad_genomes_AC': None, + 'genotypes': [ + { + 'num_alt': 1, + 'ab': 0.70212764, + 'dp': 50, + 'gq': 46, + 'sample_id': 'NA20870', + }, + { + 'num_alt': 1, + 'ab': 0.631, + 'dp': 50, + 'gq': 99, + 'sample_id': 'NA20885', + }, + { + 'num_alt': 2, + 'ab': 0, + 'dp': 67, + 'gq': 99, + 'sample_id': 'HG00731', + }, + { + 'num_alt': 1, + 'ab': 0, + 'dp': 42, + 'gq': 96, + 'sample_id': 'HG00732', + }, + { + 'num_alt': 0, + 'ab': 0, + 'dp': 42, + 'gq': 96, + 'sample_id': 'HG00733', + } + ], + 'samples_num_alt_1': ['NA20870', 'HG00733', 'NA20885'], + 'samples_num_alt_2': ['HG00732'], + }, + 'matched_queries': {INDEX_NAME: ['F000003_3', 'F000002_2'], SECOND_INDEX_NAME: ['F000011_11']}, + }, +] +BUILD_38_NO_LIFTOVER_ES_VARIANT = deepcopy(ES_VARIANTS[1]) +BUILD_38_NO_LIFTOVER_ES_VARIANT['_source'].update({ + 'start': 103343363, + 'xpos': 2103343363, + 'variantId': '2-103343363-GAGA-G' +}) +BUILD_38_ES_VARIANT = deepcopy(BUILD_38_NO_LIFTOVER_ES_VARIANT) +BUILD_38_ES_VARIANT['_source']['rg37_locus'] = {'contig': '2', 'position': 103343353} + +ES_SV_VARIANT = { + '_source': { + 'genotypes': [ + { + 'qs': 33, + 'cn': 1, + 'defragged': False, + 'sample_id': 'HG00731', + 'num_exon': 2, + 'start': 49045487, + 'end': 49045899, + 'geneIds': ['ENSG00000228198'], + 'prev_call': False, + 'prev_overlap': False, + 'new_call': True, + }, + { + 'qs': 80, + 'cn': 2, + 'defragged': False, + 'sample_id': 'HG00733', + 'num_exon': 1, + 'start': 49045987, + 'end': 49045890, + 'geneIds': ['ENSG00000228198', 'ENSG00000135953'], + 'prev_call': False, + 'prev_overlap': True, + 'new_call': False, + } + ], + 'xpos': 1049045387, + 'end': 49045898, + 'start': 49045387, + 'xstart': 1049045587, + 'num_exon': 1, + 'pos': 49045487, + 'StrVCTVRE_score': 0.374, + 'svType': 'INS', + 'xstop': 9049045898, + 'variantId': 'prefix_19107_DEL', + 'samples': ['HG00731'], + 'sc': 7, + 'contig': '1', + 'bothsides_support': True, + 'sortedTranscriptConsequences': [ + { + 'gene_id': 'ENSG00000228198' + }, + { + 'gene_id': 'ENSG00000135953' + }, + { + 'gene_id': 'ENSG00000186092' + }, + ], + 'geneIds': ['ENSG00000228198'], + 'sf': 0.000693825, + 'sn': 10088 + }, + 'matched_queries': {SV_INDEX_NAME: ['F000002_2']}, + } + +ES_SV_WGS_VARIANT = { + '_source': { + 'genotypes': [ + { + 'gq': 33, + 'sample_id': 'NA21234', + 'num_alt': 1, + 'prev_num_alt': 2, + } + ], + 'xpos': 2049045387, + 'end': 49045898, + 'start': 49045387, + 'xstart': 2049045387, + 'pos': 49045387, + 'svType': 'CPX', + 'xstop': 20012345678, + 'variantId': 'prefix_19107_CPX', + 'algorithms': ['wham', 'manta'], + 'sc': 7, + 'contig': '2', + 'rg37_locus': {'contig': '2', 'position': 49272526}, + 'rg37_locus_end': {'contig': '20', 'position': 12326326}, + 'sortedTranscriptConsequences': [ + { + 'gene_symbol': 'OR4F5', + 'major_consequence': 'DUP_PARTIAL', + 'gene_id': 'ENSG00000228198' + }, + { + 'gene_symbol': 'FBXO28', + 'major_consequence': 'MSV_EXON_OVR', + 'gene_id': 'ENSG00000228199' + }, + { + 'gene_symbol': 'FAM131C', + 'major_consequence': 'DUP_LOF', + 'gene_id': 'ENSG00000228201' + }, + { + "gene_symbol": "H3-2", + "gene_id": None, + "major_consequence": "NEAREST_TSS" + }, + ], + 'cpx_intervals': [{'type': 'DUP', 'chrom': '2', 'start': 1000, 'end': 3000}, + {'type': 'INV', 'chrom': '20', 'start': 11000, 'end': 13000}], + 'sv_type_detail': 'dupINV', + 'gnomad_svs_ID': 'gnomAD-SV_v2.1_BND_1_1', + 'gnomad_svs_AF': 0.00679, + 'gnomad_svs_AC': 22, + 'gnomad_svs_AN': 3240, + 'geneIds': ['ENSG00000228198'], + 'sf': 0.000693825, + 'sn': 10088 + }, + 'matched_queries': {SV_WGS_INDEX_NAME: ['F000014_14']}, + } + +ES_MITO_WGS_VARIANT = { + "_source" : { + "genotypes" : [ + { + "num_alt": 2, + "gq": 60.0, + "hl": 1.0, + "mito_cn": 319.03225806451616, + "contamination": 0.0, + "dp": 5139.0, + "sample_id": "HG00731" + }, + ], + "samples_gq_60_to_65" : [ + "HG00731" + ], + "samples_num_alt_2" : [ "HG00731" ], + "AC" : 0, + "AC_het" : 1, + "AF" : 0.0, + "AF_het" : 3.968253968253968E-4, + "alt" : "A", + "AN" : 2520, + "clinvar_allele_id" : None, + "clinvar_clinical_significance" : "Likely_pathogenic", + "clinvar_gold_stars" : None, + "codingGeneIds" : [ + "ENSG00000198840" + ], + "common_low_heteroplasmy" : False, + "contig" : "M", + "dbnsfp_SIFT_pred" : "D", + "dbnsfp_MutationTaster_pred" : "N", + "dbnsfp_FATHMM_pred" : "T", + "dbnsfp_GERP_RS" : "5.07", + "dbnsfp_phastCons100way_vertebrate" : "0.958000", + "docId" : "M-10195-C-A", + "domains" : [ + "ENSP_mappings:5xtc", + "ENSP_mappings:5xtd", + "Gene3D:1", + "PANTHER:PTHR11058", + "Pfam:PF00507" + ], + "end" : 10195, + "filters" : [ ], + "geneIds" : [ + "ENSG00000198840" + ], + "gnomad_mito_AN": 56421, + "gnomad_mito_AC": 1368, + "gnomad_mito_AC_het": 3, + "gnomad_mito_AF": 0.024246292, + "gnomad_mito_AF_het": 5.317169E-5, + "gnomad_mito_max_hl": 1.0, + "hap_defining_variant" : False, + "helix_AC": 1312, + "helix_AC_het": 5, + "helix_AF": 0.0033268193, + "helix_AF_het": 4.081987E-5, + "helix_max_hl": 0.90441, + "high_constraint_region" : True, + "hmtvar_hmtVar" : 0.71, + "mainTranscript_biotype" : "protein_coding", + "mainTranscript_canonical" : 1, + "mainTranscript_category" : "missense", + "mainTranscript_cdna_start" : 137, + "mainTranscript_cdna_end" : 137, + "mainTranscript_codons" : "cCc/cAc", + "mainTranscript_gene_id" : "ENSG00000198840", + "mainTranscript_gene_symbol" : "MT-ND3", + "mainTranscript_hgvs" : "p.Pro46His", + "mainTranscript_hgvsc" : "ENST00000361227.2:c.137C>A", + "mainTranscript_major_consequence" : "missense_variant", + "mainTranscript_major_consequence_rank" : 11, + "mainTranscript_transcript_id" : "ENST00000361227", + "mainTranscript_amino_acids" : "P/H", + "mainTranscript_domains" : "Gene3D:1,ENSP_mappings:5xtc,PANTHER:PTHR11058,ENSP_mappings:5xtd,Pfam:PF00507", + "mainTranscript_hgvsp" : "ENSP00000355206.2:p.Pro46His", + "mainTranscript_polyphen_prediction" : "probably_damaging", + "mainTranscript_protein_id" : "ENSP00000355206", + "mainTranscript_sift_prediction" : "deleterious_low_confidence", + "mitimpact_apogee" : 0.42, + "mitomap_pathogenic" : True, + "pos" : 10195, + "ref" : "C", + "rg37_locus" : { + "contig" : "MT", + "position" : 10195 + }, + "rsid" : None, + "sortedTranscriptConsequences" : [ + { + "biotype" : "protein_coding", + "canonical" : 1, + "cdna_start" : 137, + "cdna_end" : 137, + "codons" : "cCc/cAc", + "gene_id" : "ENSG00000198840", + "gene_symbol" : "MT-ND3", + "hgvsc" : "ENST00000361227.2:c.137C>A", + "hgvsp" : "ENSP00000355206.2:p.Pro46His", + "transcript_id" : "ENST00000361227", + "amino_acids" : "P/H", + "polyphen_prediction" : "probably_damaging", + "protein_id" : "ENSP00000355206", + "protein_start" : 46, + "sift_prediction" : "deleterious_low_confidence", + "consequence_terms" : [ + "missense_variant" + ], + "domains" : [ + "Gene3D:1", + "ENSP_mappings:5xtc", + "ENSP_mappings:5xtd", + "Pfam:PF00507", + "PANTHER:PTHR11058", + "PANTHER:PTHR11058" + ], + "major_consequence" : "missense_variant", + "category" : "missense", + "hgvs" : "p.Pro46His", + "major_consequence_rank" : 11, + "transcript_rank" : 0 + } + ], + "start" : 10195, + "transcriptConsequenceTerms" : [ + "missense_variant" + ], + "transcriptIds" : [ + "ENST00000361227" + ], + "variantId" : "M-10195-C-A", + "xpos" : 25000010195, + "xstart" : 25000010195, + "xstop" : 25000010195 + }, + 'matched_queries': {MITO_WGS_INDEX_NAME: ['F000002_2']}, +} + +OR2M3_COMPOUND_HET_ES_VARIANTS = deepcopy(ES_VARIANTS) +transcripts = OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] +transcripts[0]['major_consequence'] = 'frameshift_variant' +OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] = [transcripts[1], transcripts[0]] +OR2M3_COMPOUND_HET_ES_VARIANTS[0]['_source']['rg37_locus'] = {'contig': '1', 'position': 248367217} +OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['rg37_locus'] = {'contig': '2', 'position': 103343343} +MFSD9_COMPOUND_HET_ES_VARIANTS = deepcopy(OR2M3_COMPOUND_HET_ES_VARIANTS) +for var in MFSD9_COMPOUND_HET_ES_VARIANTS: + var['_source']['variantId'] = '{}-het'.format(var['_source']['variantId']) +EXTRA_FAMILY_ES_VARIANTS = deepcopy(ES_VARIANTS) + [deepcopy(ES_VARIANTS[0])] +EXTRA_FAMILY_ES_VARIANTS[2]['matched_queries'][INDEX_NAME] = ['F000005_5'] +MISSING_SAMPLE_ES_VARIANTS = deepcopy(ES_VARIANTS) +MISSING_SAMPLE_ES_VARIANTS[1]['_source']['samples_num_alt_1'] = [] + +ES_SV_COMP_HET_VARIANT = deepcopy(ES_SV_VARIANT) +ES_SV_COMP_HET_VARIANT['_source']['xpos'] = 2101343374 +ES_SV_COMP_HET_VARIANT['_source']['start'] = 101343374 +ES_SV_COMP_HET_VARIANT['_source']['xstop'] = 1104943628 +ES_SV_COMP_HET_VARIANT['_source']['end'] = 104943628 +ES_SV_COMP_HET_VARIANT['_source']['num_exon'] = 2 +ES_SV_COMP_HET_VARIANT['_source']['variantId'] = 'prefix_191011_DEL' +ES_SV_COMP_HET_VARIANT['_source']['svType'] = 'DEL' +for gen in ES_SV_COMP_HET_VARIANT['_source']['genotypes']: + gen.update({'start': None, 'end': None, 'num_exon': None}) + gen.pop('geneIds') + +COMPOUND_HET_INDEX_VARIANTS = { + INDEX_NAME: { + 'ENSG00000135953': EXTRA_FAMILY_ES_VARIANTS, + 'ENSG00000228198': EXTRA_FAMILY_ES_VARIANTS, + 'ENSG00000228199': EXTRA_FAMILY_ES_VARIANTS, + }, + SECOND_INDEX_NAME: { + 'ENSG00000135953': MFSD9_COMPOUND_HET_ES_VARIANTS, 'ENSG00000228198': OR2M3_COMPOUND_HET_ES_VARIANTS, + }, + '{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME): {'ENSG00000135953': MISSING_SAMPLE_ES_VARIANTS}, + SV_INDEX_NAME: {'ENSG00000228198': [ES_SV_VARIANT], 'ENSG00000135953': []}, + '{},{}'.format(INDEX_NAME, SV_INDEX_NAME): {'ENSG00000228198': [ES_SV_VARIANT, ES_SV_COMP_HET_VARIANT, ES_VARIANTS[1]], 'ENSG00000135953': []}, +} + +INDEX_ES_VARIANTS = { + INDEX_NAME: ES_VARIANTS, + SECOND_INDEX_NAME: [ES_VARIANTS[1]], + SV_INDEX_NAME: [ES_SV_VARIANT], + SV_WGS_INDEX_NAME: [ES_SV_WGS_VARIANT], + HG38_INDEX_NAME: [BUILD_38_ES_VARIANT, BUILD_38_NO_LIFTOVER_ES_VARIANT], + MITO_WGS_INDEX_NAME: [ES_MITO_WGS_VARIANT], +} +INDEX_ES_VARIANTS.update({k: ES_VARIANTS for k in SUB_INDICES + SECOND_SUB_INDICES}) + +PARSED_ANY_AFFECTED_VARIANTS = deepcopy(PARSED_VARIANTS) +PARSED_ANY_AFFECTED_VARIANTS[1]['familyGuids'] = ['F000003_3'] +PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes'] = {'I000007_na20870': PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes']['I000007_na20870']} + +PARSED_COMPOUND_HET_VARIANTS = deepcopy(PARSED_VARIANTS) +PARSED_COMPOUND_HET_VARIANTS[0]['_sort'] = [1248367327] +PARSED_COMPOUND_HET_VARIANTS[1]['_sort'] = [2103343453] +PARSED_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000003_3'] + +PARSED_SV_COMPOUND_HET_VARIANTS = [deepcopy(PARSED_SV_VARIANT), deepcopy(PARSED_COMPOUND_HET_VARIANTS[1])] +PARSED_SV_COMPOUND_HET_VARIANTS[0].update({ + '_sort': [2101343474], + 'xpos': 2101343374, + 'pos': 101343374, + 'end': 104943628, + 'variantId': 'prefix_191011_DEL', + 'svType': 'DEL', +}) +del PARSED_SV_COMPOUND_HET_VARIANTS[0]['svSourceDetail'] +PARSED_SV_COMPOUND_HET_VARIANTS[0]['transcripts']['ENSG00000186092'] = [{'geneId': 'ENSG00000186092'}] +for gen in PARSED_SV_COMPOUND_HET_VARIANTS[0]['genotypes'].values(): + gen.update({'start': None, 'end': None, 'numExon': None, 'geneIds': None}) +PARSED_SV_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000002_2'] + +PARSED_COMPOUND_HET_VARIANTS_PROJECT_2 = deepcopy(PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT) +for variant in PARSED_COMPOUND_HET_VARIANTS_PROJECT_2: + variant.update({ + 'variantId': '{}-het'.format(variant['variantId']), + 'familyGuids': ['F000011_11'], + 'genotypes': { + 'I000015_na20885': variant['genotypes']['I000015_na20885'], + }, + 'genomeVersion': '37', + 'selectedMainTranscriptId': None, + }) + variant['clinvar']['version'] = None + +PARSED_NO_CONSEQUENCE_FILTER_VARIANTS = deepcopy(PARSED_VARIANTS) +PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[1]['selectedMainTranscriptId'] = None + +PARSED_CADD_VARIANTS = deepcopy(PARSED_NO_CONSEQUENCE_FILTER_VARIANTS) +PARSED_CADD_VARIANTS[0]['_sort'][0] = -25.9 +PARSED_CADD_VARIANTS[1]['_sort'][0] = maxsize + + +PARSED_MULTI_INDEX_VARIANT = deepcopy(PARSED_VARIANTS[1]) +PARSED_MULTI_INDEX_VARIANT['familyGuids'].append('F000011_11') +PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'] = { + 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, 'sampleType': 'WES', +} + +PARSED_HG38_VARIANT = deepcopy(PARSED_VARIANTS[1]) +PARSED_HG38_VARIANT.update({ + 'familyGuids': ['F000011_11'], + 'genotypes': { + 'I000015_na20885': PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'], + }, + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverPos': PARSED_MULTI_INDEX_VARIANT['pos'], + 'liftedOverChrom': PARSED_MULTI_INDEX_VARIANT['chrom'], + 'pos': PARSED_MULTI_INDEX_VARIANT['pos'] + 10, + 'xpos': PARSED_MULTI_INDEX_VARIANT['xpos'] + 10, + 'variantId': PARSED_MULTI_INDEX_VARIANT['variantId'].replace( + str(PARSED_MULTI_INDEX_VARIANT['pos']), str(PARSED_MULTI_INDEX_VARIANT['pos'] + 10) + ), + '_sort': [PARSED_MULTI_INDEX_VARIANT['_sort'][0] + 10], +}) +PARSED_HG38_VARIANT['clinvar']['version'] = None + +PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) +for guid, genotype in PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'].items(): + PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) + +PARSED_MULTI_SAMPLE_VARIANT = deepcopy(PARSED_VARIANTS[1]) +for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT['genotypes'].items(): + PARSED_MULTI_SAMPLE_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) + +PARSED_MULTI_SAMPLE_VARIANT_0 = deepcopy(PARSED_VARIANTS[0]) +for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'].items(): + PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'][guid] = dict(otherSample=genotype, **genotype) + +PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS = deepcopy(PARSED_COMPOUND_HET_VARIANTS) +for variant in PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS: + for guid, genotype in variant['genotypes'].items(): + variant['genotypes'][guid] = dict(otherSample=genotype, **genotype) + + +PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) +PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT.update({ + 'familyGuids': ['F000003_3', 'F000011_11'], + 'genotypes': { + ind_guid: PARSED_MULTI_INDEX_VARIANT['genotypes'][ind_guid] + for ind_guid in ['I000007_na20870', 'I000015_na20885'] + }, +}) +MAPPING_FIELDS = [ + 'start', + 'end', + 'rsid', + 'originalAltAlleles', + 'filters', + 'xpos', + 'alt', + 'ref', + 'contig', + 'variantId', + 'dbnsfp_MutationTaster_pred', + 'mpc_MPC', + 'dbnsfp_DANN_score', + 'eigen_Eigen_phred', + 'dbnsfp_REVEL_score', + 'splice_ai_delta_score', + 'splice_ai_splice_consequence', + 'dbnsfp_FATHMM_pred', + 'dbnsfp_fathmm_MKL_coding_pred', + 'dbnsfp_MutPred_score', + 'dbnsfp_VEST4_score', + 'primate_ai_score', + 'dbnsfp_SIFT_pred', + 'dbnsfp_Polyphen2_HVAR_pred', + 'cadd_PHRED', + 'gnomad_non_coding_constraint_z_score', + 'sortedTranscriptConsequences', + 'screen_region_type', + 'genotypes', + 'samples_no_call', + 'samples_num_alt_1', + 'samples_num_alt_2', + 'clinvar_clinical_significance', + 'clinvar_allele_id', + 'clinvar_variation_id', + 'clinvar_gold_stars', + 'hgmd_accession', + 'hgmd_class', + 'AC', + 'AF', + 'AN', + 'gnomad_genomes_AC', + 'gnomad_genomes_Hom', + 'gnomad_genomes_Hemi', + 'gnomad_genomes_AF', + 'gnomad_genomes_AF_POPMAX_OR_GLOBAL', + 'gnomad_genomes_AN', + 'gnomad_genomes_Het', + 'gnomad_genomes_ID', + 'gnomad_exomes_AC', + 'gnomad_exomes_Hom', + 'gnomad_exomes_Hemi', + 'gnomad_exomes_AF', + 'gnomad_exomes_AF_POPMAX_OR_GLOBAL', + 'gnomad_exomes_AN', + 'gnomad_exomes_Het', + 'gnomad_exomes_ID', + 'exac_AC_Adj', + 'exac_AC_Hom', + 'exac_AC_Hemi', + 'exac_AF_POPMAX', + 'exac_AF', + 'exac_AN_Adj', + 'exac_Het', + 'exac_ID', + 'topmed_AC', + 'topmed_Hom', + 'topmed_Hemi', + 'topmed_Het', + 'topmed_AF', + 'topmed_AN', + 'topmed_ID', + 'gnomad_genomes_FAF_AF', + 'rg37_locus', + 'rg37_locus_end', + 'xstop', + 'bothsides_support', +] + +SV_MAPPING_FIELDS = [ + 'start', + 'end', + 'xpos', + 'contig', + 'variantId', + 'sortedTranscriptConsequences', + 'genotypes', + 'samples', + 'samples_cn_0', + 'samples_cn_1', + 'samples_cn_2', + 'samples_cn_3', + 'samples_cn_gte_4', + 'sf', + 'sc', + 'sn', + 'num_exon', + 'svType', + 'StrVCTVRE_score', + 'sv_type_detail', + 'cpx_intervals', + 'gnomad_svs_AC', + 'gnomad_svs_Hom', + 'gnomad_svs_Hemi', + 'gnomad_svs_AF', + 'gnomad_svs_AN', + 'gnomad_svs_filter_AF', + 'gnomad_svs_Het', + 'gnomad_svs_ID', + 'bothsides_support', +] + +MITO_MAPPING_FIELDS = [ + "AC", + "AC_het", + "AF", + "AF_het", + "AN", + "alt", + "clinvar_allele_id", + "clinvar_clinical_significance", + "clinvar_gold_stars", + "common_low_heteroplasmy", + "contig", + "dbnsfp_FATHMM_pred", + "dbnsfp_MutationTaster_pred", + "dbnsfp_Polyphen2_HVAR_pred", + "dbnsfp_REVEL_score", + "dbnsfp_SIFT_pred", + "end", + "filters", + "genotypes", + "gnomad_mito_AC", + "gnomad_mito_AC_het", + "gnomad_mito_AF", + "gnomad_mito_AF_het", + "gnomad_mito_AN", + "gnomad_mito_max_hl", + "hap_defining_variant", + "helix_AC", + "helix_AC_het", + "helix_AF", + "helix_AF_het", + "helix_max_hl", + "high_constraint_region", + "hmtvar_hmtVar", + "mitimpact_apogee", + "mitomap_pathogenic", + "mitotip_mitoTIP", + "ref", + "rg37_locus", + "rsid", + "samples_num_alt_1", + "samples_num_alt_2", + "sortedTranscriptConsequences", + "start", + "variantId", + "xpos", + "xstop", +] + +MITO_SOURCE_ONLY_FIELDS = [ + 'callset_max_hl', + 'exac_max_hl', + 'gnomad_exomes_max_hl', + 'gnomad_genomes_max_hl', + 'gnomad_svs_max_hl', + 'sv_callset_max_hl', + 'topmed_max_hl', + 'helix_ID', + 'helix_Hemi', + 'helix_AN', + 'helix_filter_AF', + 'helix_Het', + 'helix_Hom', + 'gnomad_mito_filter_AF', + 'gnomad_mito_Hom', + 'gnomad_mito_ID', + 'gnomad_mito_Het', + 'gnomad_mito_Hemi', + 'callset_heteroplasmy_Het', + 'callset_heteroplasmy_ID', + 'callset_heteroplasmy_Hemi', + 'callset_heteroplasmy_filter_AF', + 'callset_heteroplasmy_Hom', + 'callset_heteroplasmy_max_hl', + 'gnomad_mito_heteroplasmy_filter_AF', + 'gnomad_mito_heteroplasmy_Hemi', + 'gnomad_mito_heteroplasmy_Hom', + 'gnomad_mito_heteroplasmy_ID', + 'gnomad_mito_heteroplasmy_Het', + 'helix_heteroplasmy_Het', + 'helix_heteroplasmy_ID', + 'helix_heteroplasmy_Hemi', + 'helix_heteroplasmy_filter_AF', + 'helix_heteroplasmy_Hom', +] + +SOURCE_FIELDS = { + 'homozygote_count', 'callset_Hemi', 'callset_Het', 'callset_ID', 'sv_callset_Hemi', + 'sv_callset_Hom', 'sv_callset_Het', 'sv_callset_ID', 'algorithms', +} +SOURCE_FIELDS.update(MAPPING_FIELDS) +SOURCE_FIELDS.update(SV_MAPPING_FIELDS) +SOURCE_FIELDS.update(MITO_MAPPING_FIELDS) +SOURCE_FIELDS.update(MITO_SOURCE_ONLY_FIELDS) +SOURCE_FIELDS -= { + 'samples_no_call', 'samples_cn_0', 'samples_cn_1', 'samples_cn_2', 'samples_cn_3', 'samples_cn_gte_4', 'topmed_Het', + 'gnomad_genomes_FAF_AF', +} + +FIELD_TYPE_MAP = { + 'cadd_PHRED': {'type': 'keyword'}, + 'primate_ai_score': {'type': 'float'}, + 'rg37_locus': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}}, + 'rg37_locus_end': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}} +} +MAPPING_PROPERTIES = {field: FIELD_TYPE_MAP.get(field, {'type': 'keyword'}) for field in MAPPING_FIELDS} + +CORE_INDEX_METADATA = { + INDEX_NAME: { + '_meta': {'genomeVersion': '37', 'clinvar_version': '2023-03-05'}, + 'properties': MAPPING_PROPERTIES, + }, + SECOND_INDEX_NAME: { + '_meta': {'genomeVersion': '37', 'datasetType': 'VARIANTS'}, + 'properties': MAPPING_PROPERTIES, + }, + SV_INDEX_NAME: { + '_meta': {'genomeVersion': '37', 'datasetType': 'SV'}, + 'properties': {field: {'type': 'keyword'} for field in SV_MAPPING_FIELDS}, + }, + MITO_WGS_INDEX_NAME: { + '_meta': {'genomeVersion': '37', 'datasetType': 'MITO'}, + 'properties': {field: {'type': 'keyword'} for field in MITO_MAPPING_FIELDS}, + }, +} +INDEX_METADATA = { + HG38_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SECOND_INDEX_NAME]), + SV_WGS_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SV_INDEX_NAME]), +} +for meta in INDEX_METADATA.values(): + meta['_meta']['genomeVersion'] = '38' +INDEX_METADATA.update(CORE_INDEX_METADATA) + +ALL_INHERITANCE_QUERY = { + 'bool': { + 'should': [ + {'bool': { + 'must': [ + {'bool': {'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732', 'HG00733']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732', 'HG00733']}}, + {'terms': {'samples': ['HG00731', 'HG00732', 'HG00733']}}, + ]}} + ], + '_name': 'F000002_2' + }}, + {'bool': { + 'must': [ + {'bool': {'should': [ + {'terms': {'samples_num_alt_1': ['NA20870']}}, + {'terms': {'samples_num_alt_2': ['NA20870']}}, + {'terms': {'samples': ['NA20870']}}, + ]}} + ], + '_name': 'F000003_3' + }}, + {'bool': { + 'must': [ + {'bool': {'should': [ + {'terms': {'samples_num_alt_1': ['NA20874']}}, + {'terms': {'samples_num_alt_2': ['NA20874']}}, + {'terms': {'samples': ['NA20874']}}, + ]}} + ], + '_name': 'F000005_5' + }} + ] + } +} + +COMPOUND_HET_INHERITANCE_QUERY = { + 'bool': { + 'should': [ + {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] + }}, + {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00731'}}, + {'term': {'samples_gq_5_to_10': 'HG00731'}}, + {'term': {'samples_gq_0_to_5': 'HG00732'}}, + {'term': {'samples_gq_5_to_10': 'HG00732'}}, + {'term': {'samples_gq_0_to_5': 'HG00733'}}, + {'term': {'samples_gq_5_to_10': 'HG00733'}}, + ]}}, + ] + }}, + {'bool': { + '_name': 'F000003_3', + 'must': [ + {'term': {'samples_num_alt_1': 'NA20870'}}, + {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20870'}}, + {'term': {'samples_gq_5_to_10': 'NA20870'}} + ]}} + ] + }}, + ] + } +} + +COMPOUND_HET_PATH_INHERITANCE_QUERY = deepcopy(COMPOUND_HET_INHERITANCE_QUERY) +for fam_q in COMPOUND_HET_PATH_INHERITANCE_QUERY['bool']['should']: + fam_quality_q = fam_q['bool']['must'][1] + fam_quality_q['bool'] = {'should': [ + deepcopy(fam_quality_q), + {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}} + ]} + +RECESSIVE_INHERITANCE_QUERY = { + 'bool': { + 'should': [ + {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] + }}, + {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00731'}}, + {'term': {'samples_gq_5_to_10': 'HG00731'}}, + {'term': {'samples_gq_0_to_5': 'HG00732'}}, + {'term': {'samples_gq_5_to_10': 'HG00732'}}, + {'term': {'samples_gq_0_to_5': 'HG00733'}}, + {'term': {'samples_gq_5_to_10': 'HG00733'}}, + ]}}, + ] + }}, + {'bool': { + '_name': 'F000003_3', + 'must': [ + {'term': {'samples_num_alt_2': 'NA20870'}}, + {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20870'}}, + {'term': {'samples_gq_5_to_10': 'NA20870'}} + ]}} + ] + }}, + ] + } +} + +ANNOTATION_QUERY = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}} + +REDIS_CACHE = {} +def _set_cache(k, v): + REDIS_CACHE[k] = v +MOCK_REDIS = mock.MagicMock() +MOCK_REDIS.get.side_effect = REDIS_CACHE.get +MOCK_REDIS.set.side_effect =_set_cache + +def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=None, index=INDEX_NAME): + parsed_hits = deepcopy(hits) + for hit in parsed_hits: + hit.update({ + '_index': index, + '_id': hit['_source']['variantId'], + '_type': 'structural_variant' if SV_INDEX_NAME in index or SV_WGS_INDEX_NAME in index else 'variant', + }) + matched_queries = hit.pop('matched_queries') + if include_matched_queries: + hit['matched_queries'] = [] + for subindex in index.split(','): + if subindex in matched_queries: + hit['_index'] = subindex + hit['matched_queries'] += matched_queries[subindex] + + if sort or increment_sort: + sort_key = sort[0] if sort else 'xpos' + if isinstance(sort_key, dict): + if '_script' in sort_key: + sort_key = sort_key['_script']['script']['params'].get('field', 'xpos') # pylint: disable=invalid-sequence-index + else: + sort_key = next(iter(sort_key.keys())) + sort_value = jmespath.search(sort_key, hit['_source']) + if sort_value is None: + sort_value = 'Infinity' + if increment_sort: + sort_value += 100 + hit['_sort'] = [sort_value] + return parsed_hits + + +def create_mock_response(search, index=INDEX_NAME): + index = ALIAS_MAP.get(index, index) + indices = index.split(',') + include_matched_queries = False + variant_id_filters = None + gene_ids_filters = set() + if 'query' in search: + for search_filter in search['query']['bool']['filter']: + if not variant_id_filters: + variant_id_filters = search_filter.get('terms', {}).get('variantId') + if not gene_ids_filters and not search.get('aggs'): + gene_ids_filters.update(search_filter.get('terms', {}).get('geneIds') or []) + possible_inheritance_filters = search_filter.get('bool', {}).get('should', []) + [search_filter] + if any('_name' in possible_filter.get('bool', {}) for possible_filter in possible_inheritance_filters): + include_matched_queries = True + break + + response_dict = { + 'took': 1, + 'hits': {'total': {'value': 5}, 'hits': []} + } + for index_name in sorted(indices): + index_hits = mock_hits( + INDEX_ES_VARIANTS[index_name], include_matched_queries=include_matched_queries, sort=search.get('sort'), + index=index_name) + if variant_id_filters: + index_hits = [hit for hit in index_hits if hit['_id'] in variant_id_filters] + elif gene_ids_filters: + index_hits = [hit for hit in index_hits if any( + gene_ids_filters.intersection({t['gene_id'] for t in hit['_source']['sortedTranscriptConsequences']}) + )] + response_dict['hits']['hits'] += index_hits + + try: + response_dict['hits']['hits'] = sorted(response_dict['hits']['hits'], key=lambda v: v['_sort']) + except (KeyError, TypeError): + pass + + if search.get('aggs'): + index_vars = COMPOUND_HET_INDEX_VARIANTS.get(index, {}) + buckets = [{'key': gene_id, 'doc_count': 3} for gene_id in ['ENSG00000135953', 'ENSG00000228198']] + if search['aggs']['genes']['aggs'].get('vars_by_gene'): + for bucket in buckets: + bucket['vars_by_gene'] = { + 'hits': { + 'hits': mock_hits(index_vars.get(bucket['key'], ES_VARIANTS), increment_sort=True, index=index) + }} + else: + for bucket in buckets: + doc_count = 0 + for sample_field in ['samples', 'samples_num_alt_1', 'samples_num_alt_2']: + gene_samples = defaultdict(int) + for var in index_vars.get(bucket['key'], ES_VARIANTS): + for sample in var['_source'].get(sample_field, []): + gene_samples[sample] += 1 + bucket[sample_field] = {'buckets': [{'key': k, 'doc_count': v} for k, v in gene_samples.items()]} + doc_count += sum(gene_samples.values()) + bucket['doc_count'] = doc_count + + response_dict['aggregations'] = {'genes': {'buckets': buckets}} + + if len(response_dict['hits']['hits']) == 0: + response_dict['hits']['total']['value'] = 0 + elif gene_ids_filters == {'ENSG00000186092'}: + response_dict['hits']['total']['value'] = 1 + + + return response_dict + +def get_indices_from_url(url): + return url.split('/')[1] + +def get_metadata_callback(request): + indices = get_indices_from_url(request.url).split(',') + response = {index: {'mappings': INDEX_METADATA[index]} for index in indices} + return 200, {}, json.dumps(response) + +def get_search_callback(request): + body = json.loads(request.body) + response = create_mock_response(body, get_indices_from_url(request.url)) + return 200, {}, json.dumps(response) + +def parse_msearch_body(body): + return [json.loads(row) for row in body.decode().split('\n') if row] + +def get_msearch_callback(request): + body = parse_msearch_body(request.body) + response = { + 'responses': [ + create_mock_response(exec_search, index=','.join(body[i-1]['index'])) + for i, exec_search in enumerate(body) if not exec_search.get('index')] + } + return 200, {}, json.dumps(response) + +def setup_search_responses(): + urllib3_responses.add_callback( + urllib3_responses.POST, '/_msearch', callback=get_msearch_callback, + content_type='application/json', match_querystring=True) + urllib3_responses.add_callback( + urllib3_responses.POST, re.compile('^/[,\w]+/_search$'), callback=get_search_callback, + content_type='application/json', match_querystring=True) + +def setup_responses(): + urllib3_responses.add_callback( + urllib3_responses.GET, re.compile('^/[,\w]+/_mapping$'), callback=get_metadata_callback, + content_type='application/json', match_querystring=True) + setup_search_responses() + + +@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') +@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) +class EsUtilsTest(TestCase): + databases = ['default', 'reference_data'] + fixtures = ['users', '1kg_project', 'reference_data'] + + def setUp(self): + Sample.objects.filter(sample_id='NA19678').update(is_active=False) + Sample.objects.filter(sample_id='NA20885').update(sample_type='WES') + self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) + + def assertExecutedSearch(self, filters=None, start_index=0, size=2, index=INDEX_NAME, expected_source_fields=SOURCE_FIELDS, call_index=-1, **kwargs): + executed_search = urllib3_responses.call_request_json(index=call_index) + searched_indices = get_indices_from_url(urllib3_responses.calls[call_index].request.url) + self.assertListEqual(sorted(searched_indices.split(',')), sorted(index.split(','))) + self.assertSameSearch( + executed_search, + dict(filters=filters, start_index=start_index, size=size, **kwargs), + expected_source_fields=expected_source_fields, + ) + + def assertExecutedSearches(self, searches): + executed_search = parse_msearch_body(urllib3_responses.calls[-1].request.body) + self.assertEqual(len(executed_search), len(searches) * 2) + for i, expected_search in enumerate(searches): + self.assertDictEqual(executed_search[i * 2], {'index': expected_search.get('index', INDEX_NAME).split(',')}) + self.assertSameSearch(executed_search[(i * 2) + 1], expected_search) + + def assertSameSearch(self, executed_search, expected_search_params, expected_source_fields=SOURCE_FIELDS): + expected_search = { + 'from': expected_search_params['start_index'], + 'size': expected_search_params['size'] + } + + if expected_search_params['filters']: + expected_search['query'] = { + 'bool': { + 'filter': expected_search_params['filters'] + } + } + + if expected_search_params.get('query'): + expected_search['query']['bool']['must'] = expected_search_params['query'] + + if not expected_search_params.get('unsorted'): + expected_search['sort'] = expected_search_params.get('sort') or ['xpos', 'variantId'] + + if expected_search_params.get('gene_aggs'): + expected_search['aggs'] = { + 'genes': {'terms': {'field': 'geneIds', 'min_doc_count': 2, 'size': 1001}, 'aggs': { + 'vars_by_gene': { + 'top_hits': {'sort': expected_search['sort'], '_source': mock.ANY, 'size': 100} + } + }}} + elif expected_search_params.get('gene_count_aggs'): + expected_search['aggs'] = {'genes': { + 'terms': {'field': 'mainTranscript_gene_id', 'size': 1001}, + 'aggs': expected_search_params['gene_count_aggs'] + }} + del expected_search['sort'] + else: + expected_search['_source'] = mock.ANY + self.assertDictEqual(executed_search, expected_search) + + if not expected_search_params.get('gene_count_aggs'): + source = executed_search['aggs']['genes']['aggs']['vars_by_gene']['top_hits']['_source'] \ + if expected_search_params.get('gene_aggs') else executed_search['_source'] + self.assertSetEqual(expected_source_fields, set(source)) + + def assertCachedResults(self, results_model, expected_results, sort='xpos'): + cache_key = 'search_results__{}__{}'.format(results_model.guid, sort) + self.assertIn(cache_key, REDIS_CACHE.keys()) + self.assertDictEqual(json.loads(REDIS_CACHE[cache_key]), expected_results) + MOCK_REDIS.expire.assert_called_with(cache_key, timedelta(weeks=2)) + + @urllib3_responses.activate + def test_get_es_variants_for_variant_ids(self): + setup_responses() + samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') + get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) + self.assertExecutedSearch( + filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], + size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), + ) + + @urllib3_responses.activate + def test_get_single_es_variant(self): + setup_responses() + family = self.families.filter(guid='F000003_3').first() + variant = get_single_variant(family, '2-103343353-GAGA-G') + self.assertDictEqual(variant, {**PARSED_ANY_AFFECTED_VARIANTS[1], 'selectedMainTranscriptId': None}) + self.assertExecutedSearch( + filters=[{'terms': {'variantId': ['2-103343353-GAGA-G']}}], + size=1, index=INDEX_NAME + ) + + family = self.families.filter(guid='F000002_2').first() + variant = get_single_variant(family, 'prefix_19107_DEL') + self.assertDictEqual(variant, PARSED_SV_VARIANT) + self.assertExecutedSearch( + filters=[{'terms': {'variantId': ['prefix_19107_DEL']}}], size=1, index=SV_INDEX_NAME, + ) + + variant = get_single_variant(family, 'M-10195-C-A') + self.assertDictEqual(variant, PARSED_MITO_VARIANT) + self.assertExecutedSearch( + filters=[{'terms': {'variantId': ['M-10195-C-A']}}], size=1, index=MITO_WGS_INDEX_NAME, + ) + + with self.assertRaises(InvalidSearchException) as cm: + get_single_variant(family, '10-10334333-A-G') + self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') + + @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_COMPOUND_HET_GENES', 1) + @mock.patch('seqr.utils.search.elasticsearch.es_gene_agg_search.MAX_COMPOUND_HET_GENES', 1) + @mock.patch('seqr.utils.search.elasticsearch.es_search.logger') + @urllib3_responses.activate + def test_invalid_get_es_variants(self, mock_logger): + setup_responses() + mito_sample = Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME) + mito_sample.individual_id = 6 + mito_sample.save() + search_model = VariantSearch.objects.create(search={}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + + results_model.families.set(self.families) + Sample.objects.filter(elasticsearch_index=INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) + with self.assertRaises(InvalidSearchException) as cm: + query_variants(results_model) + self.assertEqual( + str(cm.exception), 'The following indices do not have the expected genome version 37: test_index_hg38 (38)', + ) + Sample.objects.filter(elasticsearch_index=HG38_INDEX_NAME).update(elasticsearch_index=INDEX_NAME) + + results_model.families.set(self.families) + search_model.search = { + 'inheritance': {'mode': 'compound_het'}, + 'locus': {'rawItems': 'WASH7P'}, + 'annotations': {'frameshift': ['frameshift_variant']}, + } + search_model.save() + with self.assertRaises(InvalidSearchException) as cm: + query_variants(results_model) + self.assertEqual( + str(cm.exception), + 'This search returned too many compound heterozygous variants. Please add stricter filters') + + with self.assertRaises(InvalidSearchException) as cm: + get_variant_query_gene_counts(results_model, None) + self.assertEqual(str(cm.exception), 'This search returned too many genes') + + search_model.search = {'qualityFilter': {'min_gq': 7}} + search_model.save() + with self.assertRaises(Exception) as cm: + query_variants(results_model) + self.assertEqual(str(cm.exception), 'Invalid gq filter 7') + + search_model.search = {} + search_model.save() + urllib3_responses.reset() + urllib3_responses.add( + urllib3_responses.POST, '/_msearch', body=ReadTimeoutError('', '', 'timeout')) + urllib3_responses.add_json('/_tasks?actions=*search&group_by=parents', {'tasks': { + 123: {'running_time_in_nanos': 10}, + 456: {'running_time_in_nanos': 10 ** 12}, + }}) + with self.assertRaises(ConnectionTimeout): + query_variants(results_model) + self.assertListEqual( + [call.request.url for call in urllib3_responses.calls], + ['/_msearch', '/_tasks?actions=%2Asearch&group_by=parents']) + mock_logger.error.assert_called_with('ES Query Timeout: Found 1 long running searches', None, detail=[ + {'task': {'running_time_in_nanos': 10 ** 12}, 'parent_task_id': '456'}, + ]) + + urllib3_responses.reset() + urllib3_responses.add_json('/_msearch', {'responses': [ + {'error': {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}} + ]}, method=urllib3_responses.POST) + + with self.assertRaises(TransportError) as cm: + query_variants(results_model) + self.assertDictEqual( + cm.exception.info, + {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}) + + _set_cache(f'index_metadata__{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}', None) + urllib3_responses.add( + urllib3_responses.GET, f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', body=Exception('Connection error')) + with self.assertRaises(InvalidIndexException) as cm: + query_variants(results_model) + self.assertEqual(str(cm.exception), 'test_index,test_index_mito_wgs,test_index_sv - Error accessing index: Connection error') + + urllib3_responses.replace_json(f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', {}) + with self.assertRaises(InvalidIndexException) as cm: + query_variants(results_model) + self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') + + @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) + @urllib3_responses.activate + def test_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={'annotations': {'frameshift': ['frameshift_variant']}}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 2) + self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) + self.assertDictEqual(variants[1], PARSED_VARIANTS[1]) + self.assertEqual(total_results, 5) + + self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) + + self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY]) + + # does not save non-consecutive pages + variants, total_results = query_variants(results_model, page=3, num_results=2) + self.assertEqual(total_results, 5) + self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=2) + + # test pagination + variants, total_results = query_variants(results_model, page=2, num_results=2) + self.assertEqual(len(variants), 2) + self.assertEqual(total_results, 5) + self.assertCachedResults(results_model, {'all_results': PARSED_VARIANTS + PARSED_VARIANTS, 'total_results': 5}) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=2, size=2) + + # test does not re-fetch page + urllib3_responses.reset() + variants, total_results = query_variants(results_model, page=1, num_results=3) + self.assertEqual(len(variants), 3) + self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS[:1]) + self.assertEqual(total_results, 5) + + # test load_all + setup_responses() + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) + self.assertEqual(len(variants), 5) + self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) + + # test does not re-fetch once all loaded + urllib3_responses.reset() + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + self.assertEqual(len(variants), 5) + self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) + + @urllib3_responses.activate + def test_filtered_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'pathogenicity': { + 'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting'], + 'hgmd': ['disease_causing', 'likely_disease_causing'], + }, + 'annotations': { + 'in_frame': ['inframe_insertion', 'inframe_deletion'], + 'other': ['5_prime_UTR_variant', 'intergenic_variant'], + 'SCREEN': ['dELS', 'DNase-only'], + 'splice_ai': '0.8', + }, + 'freqs': { + 'callset': {'af': 0.1}, + 'exac': {'ac': 2, 'af': None}, + 'g1k': {'ac': None, 'af': 0.001}, + 'gnomad_exomes': {'af': 0.01, 'ac': 3, 'hh': 3}, + 'gnomad_genomes': {'af': 0.01, 'hh': 3}, + 'topmed': {'ac': 2, 'af': None}, + }, + 'qualityFilter': {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass', 'affected_only': True}, + 'in_silico': {'cadd': '11.5', 'sift': 'D', 'fathmm': 'D'}, + 'inheritance': {'mode': 'de_novo'}, + 'customQuery': {'term': {'customFlag': 'flagVal'}}, + 'exclude': {'rawItems': 'WASH7P, chr2:1234-5678, chr7:100-10100%10'}, + }) + + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + variants, total_results = query_variants(results_model, sort='cadd', num_results=2) + + self.assertListEqual(variants, PARSED_CADD_VARIANTS) + self.assertEqual(total_results, 5) + + self.assertExecutedSearch(filters=[ + {'term': {'customFlag': 'flagVal'}}, + { + 'bool': { + 'must_not': [ + {'range': {'xpos': {'gte': 2000001234, 'lte': 2000005678}}}, + {'range': {'xstop': {'gte': 2000001234, 'lte': 2000005678}}}, + {'bool': {'must': [ + {'range': {'xpos': {'lte': 2000001234}}}, + {'range': {'xstop': {'gte': 2000005678}}}, + {'range': {'xpos': {'gte': 2000000001}}}, + {'range': {'xstop': {'lte': 2300000000}}}, + ]}}, + {'terms': {'geneIds': ['ENSG00000227232']}}, + {'bool': {'must': [ + {'range': {'xpos': {'gte': 7000000001, 'lte': 7000001100}}}, + {'range': {'xstop': {'gte': 7000009100, 'lte': 7000011100}}}]}}, + ] + } + }, + {'bool': {'should': [ + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, + {'range': {'AF': {'lte': 0.1}}} + ], + 'must': [ + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'exac_AC_Adj'}}]}}, + {'range': {'exac_AC_Adj': {'lte': 2}}} + ]} + }, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, + {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hom'}}]}}, + {'range': {'gnomad_exomes_Hom': {'lte': 3}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hemi'}}]}}, + {'range': {'gnomad_exomes_Hemi': {'lte': 3}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, + {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hom'}}]}}, + {'range': {'gnomad_genomes_Hom': {'lte': 3}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hemi'}}]}}, + {'range': {'gnomad_genomes_Hemi': {'lte': 3}}} + ]} + }, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'topmed_AC'}}]}}, + {'range': {'topmed_AC': {'lte': 2}}} + ]} + } + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, + {'range': {'AF': {'lte': 0.1}}} + ], + 'must': [ + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, + {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} + ] + }}, + {'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, + {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} + ] + }}, + {'regexp': { + 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', + }} + ] + }}, + + ]}}, + {'bool': {'should': [ + {'bool': {'must_not': [{'exists': {'field': 'cadd_PHRED'}}]}}, + {'range': {'cadd_PHRED': {'gte': 11.5}}}, + {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_SIFT_pred'}}]}}, + {'prefix': {'dbnsfp_SIFT_pred': 'D'}}, + {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_fathmm_MKL_coding_pred'}}]}}, + {'prefix': {'dbnsfp_fathmm_MKL_coding_pred': 'D'}}, + ]}}, + {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}}, + {'bool': { + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'transcriptConsequenceTerms'}}]}}, + {'terms': { + 'transcriptConsequenceTerms': [ + '5_prime_UTR_variant', + 'inframe_deletion', + 'inframe_insertion', + 'intergenic_variant', + ] + }}, + {'regexp': { + 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*|Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))', + }}, + {'terms': {'hgmd_class': ['DM', 'DM?']}}, + {'range': {'splice_ai_delta_score': {'gte': 0.8}}}, + {'terms': {'screen_region_type': ['dELS', 'DNase-only']}}, + ] + } + }, + {'bool': { + 'should': [ + {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}} + ] + }}, + {'bool': {'should': [{'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': { + 'must_not': [ + {'term': {'samples_ab_0_to_5': 'HG00731'}}, + {'term': {'samples_ab_5_to_10': 'HG00731'}}, + ] + }}, + {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'HG00731'}}]}} + ], + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00731'}}, + {'term': {'samples_gq_5_to_10': 'HG00731'}}, + {'term': {'samples_gq_10_to_15': 'HG00731'}}, + ], + }}, {'regexp': { + 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*' + }}]}} + ], + }}, + {'bool': { + 'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20870'}}, + {'term': {'samples_num_alt_2': 'NA20870'}} + ]}}, + {'bool': {'should': [{'bool': { + 'minimum_should_match': 1, + 'should': [ + {'bool': { + 'must_not': [ + {'term': {'samples_ab_0_to_5': 'NA20870'}}, + {'term': {'samples_ab_5_to_10': 'NA20870'}}, + ] + }}, + {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'NA20870'}}]}} + ], + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20870'}}, + {'term': {'samples_gq_5_to_10': 'NA20870'}}, + {'term': {'samples_gq_10_to_15': 'NA20870'}}, + ] + }}, {'regexp': { + 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', + }}]}} + ], + '_name': 'F000003_3' + }}, + ] + }} + ], sort=[{'cadd_PHRED': {'order': 'desc', 'unmapped_type': 'keyword'}}, 'xpos', 'variantId']) + + # Test sort does not error on pagination + query_variants(results_model, sort='cadd', num_results=2, page=2) + + @urllib3_responses.activate + def test_sv_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'new_structural_variants': ['NEW']}, + 'freqs': {'sv_callset': {'af': 0.1}}, + 'in_silico': {'strvctvre': '3.1', 'requireScore': True}, + 'qualityFilter': {'min_qs': 20}, + 'inheritance': {'mode': 'de_novo'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, _ = query_variants(results_model, num_results=2) + + self.assertListEqual(variants, [PARSED_SV_VARIANT]) + self.assertExecutedSearch(filters=[ + {'bool': { + 'should': [ + {'bool': {'must_not': [{'exists': {'field': 'sf'}}]}}, + {'range': {'sf': {'lte': 0.1}}} + ] + }}, + {'range': {'StrVCTVRE_score': {'gte': 3.1}}}, + {'bool': { + 'must': [ + {'bool': { + 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], + 'must': [{'term': {'samples': 'HG00731'}}], + }}, + {'bool': { + 'must_not': [ + {'term': {'samples_qs_0_to_10': 'HG00731'}}, + {'term': {'samples_qs_10_to_20': 'HG00731'}}, + {'term': {'samples_qs_0_to_10': 'HG00732'}}, + {'term': {'samples_qs_10_to_20': 'HG00732'}}, + {'term': {'samples_qs_0_to_10': 'HG00733'}}, + {'term': {'samples_qs_10_to_20': 'HG00733'}}, + ], + 'must': [{'terms': {'samples_new_call': ['HG00731', 'HG00732', 'HG00733']}}], + }}, + ], + '_name': 'F000002_2' + }} + ], index=SV_INDEX_NAME) + + @urllib3_responses.activate + def test_sv_wgs_get_es_variants(self): + self.families = Family.objects.filter(guid='F000014_14') + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'structural': ['DUP', 'CPX']}, + 'qualityFilter': {'min_gq_sv': 20}, + 'inheritance': {'mode': 'de_novo'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, _ = query_variants(results_model, num_results=2) + self.assertListEqual(variants, [PARSED_SV_WGS_VARIANT]) + + self.assertExecutedSearch(filters=[ + {'terms': {'transcriptConsequenceTerms': ['CPX', 'DUP']}}, + {'bool': { + 'must': [{'bool': {'must': [{'term': {'samples': 'NA21234'}}, {'term': {'samples': 'NA21987'}}], + 'must_not': [{'term': {'samples': 'NA21654'}}]}}, + {'bool': { + 'must_not': [ + {'term': {'samples_gq_sv_0_to_5': 'NA21234'}}, + {'term': {'samples_gq_sv_5_to_10': 'NA21234'}}, + {'term': {'samples_gq_sv_10_to_15': 'NA21234'}}, + {'term': {'samples_gq_sv_15_to_20': 'NA21234'}}, + {'term': {'samples_gq_sv_0_to_5': 'NA21654'}}, + {'term': {'samples_gq_sv_5_to_10': 'NA21654'}}, + {'term': {'samples_gq_sv_10_to_15': 'NA21654'}}, + {'term': {'samples_gq_sv_15_to_20': 'NA21654'}}, + {'term': {'samples_gq_sv_0_to_5': 'NA21987'}}, + {'term': {'samples_gq_sv_5_to_10': 'NA21987'}}, + {'term': {'samples_gq_sv_10_to_15': 'NA21987'}}, + {'term': {'samples_gq_sv_15_to_20': 'NA21987'}}, + ], + }} + ], + '_name': 'F000014_14' + }} + ], index=SV_WGS_INDEX_NAME) + + @urllib3_responses.activate + def test_multi_dataset_get_es_variants(self): + setup_responses() + + search_model = VariantSearch.objects.create(search={}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, _ = query_variants(results_model, num_results=5) + self.assertListEqual(variants, [PARSED_SV_VARIANT] + PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) + self.assertExecutedSearches([ + dict(filters=[], start_index=0, size=5, index=SV_INDEX_NAME), + dict(filters=[], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), + dict(filters=[ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), + ]) + + search_model.search['pathogenicity'] = {'clinvar': ['pathogenic']} + search_model.save() + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + variants, _ = query_variants(results_model, num_results=5) + self.assertListEqual(variants, PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) + path_filter = {'regexp': { + 'clinvar_clinical_significance': '.*Pathogenic.*' + }} + self.assertExecutedSearches([ + dict(filters=[path_filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), + dict(filters=[path_filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), + ]) + + # test with dataset filtering applied + search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} + search_model.save() + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + + query_variants(results_model, num_results=5) + filter = {'bool': {'should': [{'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}}, path_filter]}} + self.assertExecutedSearches([ + dict(filters=[filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), + dict(filters=[filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), + ]) + + search_model.search['annotations'] = { + 'structural': ['DEL'], 'structural_consequence': ['MSV_EXON_OVERLAP', 'INTRAGENIC_EXON_DUP']} + search_model.search['pathogenicity'] = None + search_model.save() + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + + query_variants(results_model, num_results=5) + self.assertExecutedSearch( + filters=[{'terms': { + 'transcriptConsequenceTerms': ['DEL', 'DUP_LOF', 'INTRAGENIC_EXON_DUP', 'MSV_EXON_OVERLAP', 'MSV_EXON_OVR'] + }}], + start_index=0, size=5, index=SV_INDEX_NAME) + + @urllib3_responses.activate + def test_multi_dataset_no_affected_inheritance_get_es_variants(self): + setup_responses() + # The family has multiple data types loaded but only one loaded in an affected individual + Sample.objects.get(individual_id=4, elasticsearch_index=SV_INDEX_NAME).delete() + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + + search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid='F000002_2')) + + query_variants(results_model, num_results=2) + self.assertExecutedSearch(filters=[{'bool': { + 'must': [{'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ], + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}}, + ], + }}], '_name': 'F000002_2', + }}]) + + @urllib3_responses.activate + def test_compound_het_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'qualityFilter': {'min_gq': 10}, + 'annotations': {'frameshift': ['frameshift_variant']}, + 'inheritance': {'mode': 'compound_het'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 1) + self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) + self.assertEqual(total_results, 1) + + self.assertCachedResults(results_model, { + 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], + 'total_results': 1, + }) + + self.assertExecutedSearch( + filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], + gene_aggs=True, + start_index=0, + size=1 + ) + + # test pagination does not fetch + urllib3_responses.reset() + query_variants(results_model, page=2, num_results=2) + + @urllib3_responses.activate + def test_compound_het_get_es_variants_secondary_annotation(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'qualityFilter': {'min_gq': 10}, + 'annotations': {'frameshift': ['frameshift_variant'], 'splice_ai': '0.5'}, + 'inheritance': {'mode': 'compound_het'}, + 'annotations_secondary': {'other': ['intron']}, + 'pathogenicity': {'clinvar': ['pathogenic'], 'hgmd': ['disease_causing']}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 1) + self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) + self.assertEqual(total_results, 1) + + self.assertCachedResults(results_model, { + 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], + 'total_results': 1, + }) + + annotation_query = {'bool': {'should': [ + {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}}, + {'terms': {'hgmd_class': ['DM']}}, + {'range': {'splice_ai_delta_score': {'gte': 0.5}}}, + {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}}, + ]}} + + self.assertExecutedSearch( + filters=[annotation_query, COMPOUND_HET_PATH_INHERITANCE_QUERY], + gene_aggs=True, + start_index=0, + size=1 + ) + + # test pagination does not fetch + urllib3_responses.reset() + query_variants(results_model, page=2, num_results=2) + + # variants require both primary and secondary annotations + setup_responses() + del search_model.search['pathogenicity'] + del search_model.search['annotations']['splice_ai'] + search_model.save() + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertIsNone(variants) + self.assertEqual(total_results, 0) + + annotation_query = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}} + + self.assertExecutedSearch( + filters=[annotation_query, COMPOUND_HET_INHERITANCE_QUERY], + gene_aggs=True, + start_index=0, + size=1 + ) + + @urllib3_responses.activate + def test_recessive_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'qualityFilter': {'min_gq': 10, 'vcf_filter': 'pass'}, + 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 2) + self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) + self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS[0]) + self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS[1]) + self.assertEqual(total_results, 6) + + self.assertCachedResults(results_model, { + 'compound_het_results': [], + 'variant_results': [PARSED_VARIANTS[1]], + 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], + 'duplicate_doc_count': 0, + 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 2, 'total': 5}}, + 'total_results': 6, + }) + + pass_filter_query = {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}} + + self.assertExecutedSearches([ + dict( + filters=[pass_filter_query, ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], + gene_aggs=True, + start_index=0, + size=1 + ), + dict( + filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=0, size=2, + ), + ]) + + # test pagination + + variants, total_results = query_variants(results_model, page=3, num_results=2) + self.assertEqual(len(variants), 2) + self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) + self.assertDictEqual(variants[1], PARSED_MULTI_SAMPLE_VARIANT) + self.assertEqual(total_results, 5) + + self.assertCachedResults(results_model, { + 'compound_het_results': [], + 'variant_results': [], + 'grouped_results': [ + {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}, + {'null': [PARSED_VARIANTS[0]]}, {'null': [PARSED_MULTI_SAMPLE_VARIANT]}], + 'duplicate_doc_count': 1, + 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 4, 'total': 5}}, + 'total_results': 5, + }) + + self.assertExecutedSearches([dict(filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=2, size=4)]) + + urllib3_responses.reset() + query_variants(results_model, page=2, num_results=2) + + @urllib3_responses.activate + def test_multi_datatype_recessive_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'inheritance': {'mode': 'recessive'}, + 'annotations': {'frameshift': ['frameshift_variant'], 'structural': ['DEL']} + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, _ = query_variants(results_model, num_results=10) + self.assertEqual(len(variants), 4) + self.assertDictEqual(variants[0], PARSED_SV_VARIANT) + self.assertDictEqual(variants[1], PARSED_VARIANTS[0]) + self.assertDictEqual(variants[2][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) + self.assertDictEqual(variants[2][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) + self.assertDictEqual(variants[3], PARSED_VARIANTS[1]) + + annotations_q = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} + self.assertExecutedSearches([ + dict( + filters=[ + annotations_q, + {'bool': { + '_name': 'F000002_2', + 'must': [{ + 'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_cn_0': 'HG00731'}}, + {'term': {'samples_cn_2': 'HG00731'}}, + {'term': {'samples_cn_gte_4': 'HG00731'}}, + ], + 'must_not': [ + {'term': {'samples_cn_0': 'HG00732'}}, + {'term': {'samples_cn_gte_4': 'HG00732'}}, + {'term': {'samples_cn_0': 'HG00733'}}, + {'term': {'samples_cn_gte_4': 'HG00733'}}, + ] + } + }] + }} + ], start_index=0, size=10, index=SV_INDEX_NAME, + ), + dict( + filters=[ + annotations_q, + {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'should': [ + {'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ] + + }}, + {'term': {'samples': 'HG00731'}}, + ] + }}, + ] + }}, + ], + gene_aggs=True, + start_index=0, + size=1, + index=','.join([INDEX_NAME, SV_INDEX_NAME]), + ), + dict( + filters=[ + annotations_q, + {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, + ], + gene_aggs=True, + start_index=0, + size=1 + ), + dict( + filters=[ + annotations_q, + { + 'bool': { + 'should': [ + {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] + }}, + ] + }}, + {'bool': { + '_name': 'F000003_3', + 'must': [ + {'term': {'samples_num_alt_2': 'NA20870'}}, + ] + }}, + ] + } + } + ], start_index=0, size=10, + ), + ]) + + @urllib3_responses.activate + def test_multi_datatype_secondary_annotations_recessive_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'structural': ['DEL']}, + 'annotations_secondary': {'frameshift': ['frameshift_variant']}, + 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, _ = query_variants(results_model, num_results=10) + self.assertEqual(len(variants), 2) + self.assertDictEqual(variants[0], PARSED_SV_VARIANT) + self.assertDictEqual(variants[1][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) + self.assertDictEqual(variants[1][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) + + annotation_secondary_query = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} + + self.assertExecutedSearches([ + dict( + filters=[ + {'terms': {'transcriptConsequenceTerms': ['DEL']}}, + {'bool': { + '_name': 'F000002_2', + 'must': [{ + 'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_cn_0': 'HG00731'}}, + {'term': {'samples_cn_2': 'HG00731'}}, + {'term': {'samples_cn_gte_4': 'HG00731'}}, + ], + 'must_not': [ + {'term': {'samples_cn_0': 'HG00732'}}, + {'term': {'samples_cn_gte_4': 'HG00732'}}, + {'term': {'samples_cn_0': 'HG00733'}}, + {'term': {'samples_cn_gte_4': 'HG00733'}}, + ] + } + }] + }} + ], start_index=0, size=10, index=SV_INDEX_NAME, + ), + dict( + filters=[annotation_secondary_query, {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'should': [ + {'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ] + }}, + {'term': {'samples': 'HG00731'}}, + ] + }}, + ] + }}, + ], + gene_aggs=True, + start_index=0, + size=1, + index=','.join([INDEX_NAME, SV_INDEX_NAME]), + ), + dict( + filters=[ + annotation_secondary_query, + {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, + ], + gene_aggs=True, + start_index=0, + size=1 + ), + ]) + + @urllib3_responses.activate + def test_multi_datatype_secondary_annotations_comp_het_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'structural': ['DEL'], 'SCREEN': ['dELS']}, + 'annotations_secondary': {'structural_consequence': ['LOF']}, + 'inheritance': {'mode': 'compound_het'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + query_variants(results_model, num_results=10) + + annotation_secondary_query = {'bool': {'should': [ + {'terms': {'transcriptConsequenceTerms': ['DEL', 'LOF']}}, + {'terms': {'screen_region_type': ['dELS']}}]}} + + self.assertExecutedSearches([ + dict( + filters=[annotation_secondary_query, {'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'should': [ + {'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ] + }}, + {'term': {'samples': 'HG00731'}}, + ] + }}, + ] + }}, + ], + gene_aggs=True, + start_index=0, + size=1, + index=','.join([INDEX_NAME, SV_INDEX_NAME]), + ), + dict( + filters=[ + annotation_secondary_query, + {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, + ], + gene_aggs=True, + start_index=0, + size=1 + ), + ]) + + @urllib3_responses.activate + def test_all_samples_all_inheritance_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'locus': {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertListEqual(variants, PARSED_VARIANTS) + self.assertEqual(total_results, 5) + + self.assertExecutedSearch(index=INDEX_NAME, size=2, filters=[ + {'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G']}}, ANNOTATION_QUERY]) + + @urllib3_responses.activate + def test_all_samples_any_affected_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertListEqual(variants, PARSED_ANY_AFFECTED_VARIANTS) + self.assertEqual(total_results, 10) + + self.assertExecutedSearches([dict(filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731']}}, + {'terms': {'samples_num_alt_2': ['HG00731']}}, + {'terms': {'samples': ['HG00731']}}, + ] + }} + ], index=MITO_WGS_INDEX_NAME, start_index=0, size=2), dict(filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, + ] + }} + ], index=INDEX_NAME, start_index=0, size=2)]) + + @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_SEARCH_CLAUSES', 1) + @urllib3_responses.activate + def test_many_family_inheitance_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(self.families) + + variants, total_results = query_variants(results_model, num_results=2) + + self.assertEqual(len(variants), 2) + self.assertEqual(total_results, 14) + self.assertDictEqual(variants[0], PARSED_MULTI_SAMPLE_VARIANT_0) + self.assertListEqual(variants[1], PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS) + + self.assertCachedResults(results_model, { + 'compound_het_results': [], + 'variant_results': [PARSED_MULTI_SAMPLE_VARIANT, PARSED_MITO_VARIANT], + 'grouped_results': [{'null': [PARSED_MULTI_SAMPLE_VARIANT_0]}, {'ENSG00000228198': PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS}], + 'duplicate_doc_count': 3, + 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, + MITO_WGS_INDEX_NAME: {'loaded': 1, 'total': 5}, + INDEX_NAME: {'loaded': 4, 'total': 10}}, + 'total_results': 14, + }) + + self.assertExecutedSearches([ + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000002_2', + 'must': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + ] + } + }, + ], + index=MITO_WGS_INDEX_NAME, + gene_aggs=True, + start_index=0, + size=1 + ), + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000002_2', + 'must': [ + {'term': {'samples_num_alt_2': 'HG00731'}}, + ] + } + }, + ], + index=MITO_WGS_INDEX_NAME, + start_index=0, + size=2, + ), + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] + }}, + ] + }, + }, + ], + gene_aggs=True, + start_index=0, + size=1 + ), + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000003_3', + 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}], + }, + }, + ], + gene_aggs=True, + start_index=0, + size=1 + ), + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000002_2', + 'must': [ + {'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] + }}, + ] + } + }, + ], + start_index=0, + size=2, + ), + dict( + filters=[ + ANNOTATION_QUERY, + { + 'bool': { + '_name': 'F000003_3', + 'must': [ + {'term': {'samples_num_alt_2': 'NA20870'}}, + ] + } + }, + ], + start_index=0, + size=2, + ), + ]) + + @urllib3_responses.activate + def test_multi_project_get_es_variants(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'qualityFilter': {'min_gq': 10}, + 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2'])) + + variants, total_results = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 2) + self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) + self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[0]) + self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[1]) + self.assertEqual(total_results, 11) + + self.assertCachedResults(results_model, { + 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], + 'variant_results': [PARSED_MULTI_INDEX_VARIANT], + 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], + 'duplicate_doc_count': 2, + 'loaded_variant_counts': { + SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, + '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, + INDEX_NAME: {'loaded': 2, 'total': 5}, + '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, + }, + 'total_results': 11, + }) + self.assertTrue('index_metadata__{}'.format(INDEX_NAME) in REDIS_CACHE) + self.assertTrue('index_metadata__{}'.format(SECOND_INDEX_NAME) in REDIS_CACHE) + + project_2_search = dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'must': [ + {'term': {'samples_num_alt_2': 'NA20885'}}, + {'bool': { + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20885'}}, + {'term': {'samples_gq_5_to_10': 'NA20885'}}, + ] + }} + ], + '_name': 'F000011_11' + }} + ], start_index=0, size=2, index=SECOND_INDEX_NAME) + project_1_search = dict( + filters=[ + ANNOTATION_QUERY, + RECESSIVE_INHERITANCE_QUERY, + ], start_index=0, size=2, index=INDEX_NAME) + self.assertExecutedSearches([ + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + '_name': 'F000011_11', + 'must': [ + {'term': {'samples_num_alt_1': 'NA20885'}}, + {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20885'}}, + {'term': {'samples_gq_5_to_10': 'NA20885'}} + ]}} + ] + }} + ], + gene_aggs=True, start_index=0, size=1, index=SECOND_INDEX_NAME, + ), + project_2_search, + dict( + filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], + gene_aggs=True, start_index=0, size=1, index=INDEX_NAME, + ), + project_1_search, + ]) + + # test pagination + variants, total_results = query_variants(results_model, num_results=2, page=2) + self.assertEqual(len(variants), 2) + self.assertListEqual(variants, [PARSED_VARIANTS[0], PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) + self.assertEqual(total_results, 9) + + cache_results = { + 'compound_het_results': [], + 'variant_results': [PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT], + 'grouped_results': [ + {'null': [PARSED_VARIANTS[0]]}, + {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}, + {'null': [PARSED_VARIANTS[0]]}, + {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT} + ], + 'duplicate_doc_count': 4, + 'loaded_variant_counts': { + SECOND_INDEX_NAME: {'loaded': 2, 'total': 5}, + '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, + INDEX_NAME: {'loaded': 4, 'total': 5}, + '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, + }, + 'total_results': 9, + } + self.assertCachedResults(results_model, cache_results) + + project_2_search['start_index'] = 1 + project_2_search['size'] = 3 + project_1_search['start_index'] = 2 + self.assertExecutedSearches([project_2_search, project_1_search]) + + # If one project is fully loaded, only query the second project + cache_results['loaded_variant_counts'][INDEX_NAME]['total'] = 4 + _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(cache_results)) + query_variants(results_model, num_results=2, page=3) + project_2_search['start_index'] = 2 + project_2_search['size'] = 4 + self.assertExecutedSearches([project_2_search]) + + @urllib3_responses.activate + def test_multi_project_all_samples_all_inheritance_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + + variants, total_results = query_variants(results_model, num_results=2) + expected_variants = [PARSED_VARIANTS[0], PARSED_MULTI_INDEX_VARIANT] + self.assertListEqual(variants, expected_variants) + self.assertEqual(total_results, 4) + + cached_variants = expected_variants + [PARSED_MITO_VARIANT] + self.assertCachedResults(results_model, { + 'all_results': cached_variants, + 'duplicate_doc_count': 1, + 'total_results': 4, + }) + + self.assertExecutedSearch( + index=','.join([INDEX_NAME, SECOND_INDEX_NAME, MITO_WGS_INDEX_NAME]), + filters=[ANNOTATION_QUERY], + size=6, + ) + + # test pagination + variants, total_results = query_variants(results_model, num_results=2, page=2) + expected_variants = [PARSED_MITO_VARIANT, PARSED_VARIANTS[0]] + self.assertListEqual(variants, expected_variants) + self.assertEqual(total_results, 3) + + self.assertCachedResults(results_model, { + 'all_results': cached_variants + cached_variants, + 'duplicate_doc_count': 2, + 'total_results': 3, + }) + + self.assertExecutedSearch( + index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), + filters=[ANNOTATION_QUERY], + size=8, + start_index=4, + ) + + # test skipping page fetches all consecutively + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + query_variants(results_model, num_results=2, page=2) + self.assertExecutedSearch( + index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), + filters=[ANNOTATION_QUERY], + size=12, + ) + + @urllib3_responses.activate + def test_multi_project_all_samples_any_affected_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, + }, + ) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + + variants, total_results = query_variants(results_model, num_results=2) + expected_variants = [PARSED_VARIANTS[0], PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT] + self.assertListEqual(variants, expected_variants) + self.assertEqual(total_results, 14) + + self.assertExecutedSearches([ + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['NA20885']}}, + {'terms': {'samples_num_alt_2': ['NA20885']}}, + {'terms': {'samples': ['NA20885']}}, + ] + }} + ], start_index=0, size=2, index=SECOND_INDEX_NAME), + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731']}}, + {'terms': {'samples_num_alt_2': ['HG00731']}}, + {'terms': {'samples': ['HG00731']}}, + ] + }} + ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, + ] + }}, + ], start_index=0, size=2, index=INDEX_NAME) + ]) + + @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_SEARCHES', 1) + @urllib3_responses.activate + def test_multi_project_prefilter_indices_get_es_variants(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'inheritance': {'mode': 'de_novo'}, + 'qualityFilter': {'min_gq': 10}, + 'locus': {'rawItems': 'ENSG00000228198'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + + _, total_results = query_variants(results_model, num_results=2) + self.assertEqual(total_results, 14) + self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) + + gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} + prefilter_search = dict( + filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', + size=200, expected_source_fields=set(), + ) + sv_search = dict( + filters=[ + gene_filter, + {'bool': { + 'must': [ + {'bool': { + 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], + 'must': [{'term': {'samples': 'HG00731'}}], + }}, + {'bool': { + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00731'}}, + {'term': {'samples_gq_5_to_10': 'HG00731'}}, + {'term': {'samples_gq_0_to_5': 'HG00732'}}, + {'term': {'samples_gq_5_to_10': 'HG00732'}}, + {'term': {'samples_gq_0_to_5': 'HG00733'}}, + {'term': {'samples_gq_5_to_10': 'HG00733'}}, + ], + }}, + ], + '_name': 'F000002_2' + }} + ], start_index=0, size=2, index=SV_INDEX_NAME) + num_calls = 3 + self.assertEqual(len(urllib3_responses.calls), num_calls) + self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) + # Search total is greater than returned hits, so proceed with regular multi-search + self.assertExecutedSearches([ + sv_search, + dict(filters=[ + gene_filter, + { + 'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20885'}}, + {'term': {'samples_num_alt_2': 'NA20885'}}, + ]}}, {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20885'}}, + {'term': {'samples_gq_5_to_10': 'NA20885'}}, + ]}} + ], + '_name': 'F000011_11' + }} + ], start_index=0, size=2, index=SECOND_INDEX_NAME), + dict(filters=[ + gene_filter, + {'bool': {'should': [ + {'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA19675'}}, + {'term': {'samples_num_alt_2': 'NA19675'}}, + ]}}, {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA19675'}}, + {'term': {'samples_gq_5_to_10': 'NA19675'}}, + ]}}], '_name': 'F000001_1'}}, + {'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ], 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}}, + ], 'minimum_should_match': 1}}, + {'bool': { + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00731'}}, + {'term': {'samples_gq_5_to_10': 'HG00731'}}, + {'term': {'samples_gq_0_to_5': 'HG00732'}}, + {'term': {'samples_gq_5_to_10': 'HG00732'}}, + {'term': {'samples_gq_0_to_5': 'HG00733'}}, + {'term': {'samples_gq_5_to_10': 'HG00733'}}, + ], + }}, + ], + '_name': 'F000002_2' + }}, + {'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20870'}}, + {'term': {'samples_num_alt_2': 'NA20870'}}, + ]}}, {'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20870'}}, + {'term': {'samples_gq_5_to_10': 'NA20870'}}, + ]}}], '_name': 'F000003_3'}}, + ]}}], start_index=0, size=2, index=INDEX_NAME), + ]) + + # Test successful prefilter + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + gene_filter['terms']['geneIds'] = ['ENSG00000186092'] + search_model.search['locus']['rawItems'] = 'ENSG00000186092' + search_model.save() + + _, total_results = query_variants(results_model, num_results=2) + self.assertEqual(total_results, 1) + num_calls += 2 + self.assertEqual(len(urllib3_responses.calls), num_calls) + self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) + sv_search['query'] = [{'ids': {'values': ['prefix_19107_DEL']}}] + self.assertExecutedSearches([sv_search]) + + # Test no results in prefilter + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + gene_filter['terms']['geneIds'] = ['ENSG00000269732'] + search_model.search['locus']['rawItems'] = 'ENSG00000269732' + search_model.save() + + _, total_results = query_variants(results_model, num_results=2) + self.assertEqual(total_results, 0) + # Only the prefliter search is run, no multi-search + self.assertEqual(len(urllib3_responses.calls), num_calls + 1) + self.assertExecutedSearch(**prefilter_search) + + + @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_VARIANTS', 3) + @urllib3_responses.activate + def test_skip_genotype_filter(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'locus': {'rawItems': 'ENSG00000228198'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + + variants, _ = query_variants(results_model, num_results=2, skip_genotype_filter=True) + expected_transcript_variant = deepcopy(PARSED_VARIANTS[0]) + expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] + self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) + self.assertExecutedSearch( + index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), + filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], + size=3, + ) + + # test with inheritance override + search_model.search['inheritance'] = {'mode': 'any_affected'} + search_model.search['locus'] = None + search_model.save() + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + query_variants(results_model, num_results=2, skip_genotype_filter=True) + + self.assertExecutedSearches([ + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['NA20885']}}, + {'terms': {'samples_num_alt_2': ['NA20885']}}, + {'terms': {'samples': ['NA20885']}}, + ] + }} + ], start_index=0, size=2, index=SECOND_INDEX_NAME), + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731']}}, + {'terms': {'samples_num_alt_2': ['HG00731']}}, + {'terms': {'samples': ['HG00731']}}, + ] + }} + ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), + dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, + ] + }}, + ], start_index=0, size=2, index=INDEX_NAME) + ]) + + @mock.patch('seqr.utils.search.utils.LIFTOVERS', {'37': None}) + @mock.patch('seqr.utils.search.utils.LiftOver') + @urllib3_responses.activate + def test_get_lifted_grch38_variants(self, mock_liftover): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) + + Project.objects.filter(guid='R0003_test').update(genome_version='38') + Sample.objects.filter(elasticsearch_index=SECOND_INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) + + mock_liftover.side_effect = Exception() + expected_no_lift_grch38_variant = deepcopy(PARSED_HG38_VARIANT) + expected_no_lift_grch38_variant.update({ + 'liftedOverGenomeVersion': None, + 'liftedOverChrom': None, + 'liftedOverPos': None, + }) + variants, _ = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 2) + self.assertListEqual(variants, [PARSED_HG38_VARIANT, expected_no_lift_grch38_variant]) + self.assertIsNone(_get_liftover('37')) + mock_liftover.assert_called_with('hg38', 'hg19') + + _set_cache('search_results__{}__xpos'.format(results_model.guid), None) + mock_liftover.side_effect = None + mock_liftover.return_value.convert_coordinate.side_effect = lambda chrom, pos: [[chrom, pos - 10]] + variants, _ = query_variants(results_model, num_results=2) + self.assertEqual(len(variants), 2) + self.assertListEqual(variants, [PARSED_HG38_VARIANT, PARSED_HG38_VARIANT]) + self.assertIsNotNone(_get_liftover('37')) + mock_liftover.assert_called_with('hg38', 'hg19') + + @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_NAME_LENGTH', 30) + @urllib3_responses.activate + def test_get_es_variants_create_index_alias(self): + search_model = VariantSearch.objects.create(search={}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + + setup_search_responses() + urllib3_responses.add_json( + '/{}/_mapping'.format(INDEX_ALIAS), {k: {'mappings': v} for k, v in CORE_INDEX_METADATA.items()}) + urllib3_responses.add_json('/_aliases', {'success': True}, method=urllib3_responses.POST) + + query_variants(results_model, num_results=2) + + self.assertExecutedSearch(index=INDEX_ALIAS, size=8) + self.assertDictEqual(urllib3_responses.call_request_json(index=0), { + 'actions': [{'add': {'indices': [INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME], 'alias': INDEX_ALIAS}}]}) + + @urllib3_responses.activate + def test_get_es_variants_search_index_alias(self): + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'inheritance': {'mode': 'de_novo'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) + + setup_search_responses() + urllib3_responses.add_json('/{}/_mapping'.format(SECOND_INDEX_NAME), { + k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SUB_INDICES}) + + query_variants(results_model, num_results=2) + + expected_search = { + 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { + 'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20885'}}, + {'term': {'samples_num_alt_2': 'NA20885'}}, + ]}} + ], + '_name': 'F000011_11' + }}] + } + self.assertExecutedSearches([ + dict(index=SUB_INDICES[1], **expected_search), + dict(index=SUB_INDICES[0], **expected_search), + ]) + _set_cache('index_metadata__{}'.format(SECOND_INDEX_NAME), None) + + @urllib3_responses.activate + def test_get_es_variants_search_multiple_index_alias(self): + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'inheritance': {'mode': 'de_novo'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000011_11'])) + + setup_search_responses() + mappings = {k: {'mappings': INDEX_METADATA[INDEX_NAME]} for k in SUB_INDICES} + mappings.update({k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SECOND_SUB_INDICES}) + urllib3_responses.add_json('/{},{}/_mapping'.format(INDEX_NAME, SECOND_INDEX_NAME), mappings) + aliases = {k: {'aliases': {INDEX_NAME: {}}} for k in SUB_INDICES} + aliases.update({k: {'aliases': {SECOND_INDEX_NAME: {}, INDEX_ALIAS: {}}} for k in SECOND_SUB_INDICES}) + urllib3_responses.add_json('/{},{}/_alias'.format(INDEX_NAME, SECOND_INDEX_NAME), aliases) + + query_variants(results_model, num_results=2) + + second_alias_expected_search = { + 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { + 'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20885'}}, + {'term': {'samples_num_alt_2': 'NA20885'}}, + ]}} + ], + '_name': 'F000011_11' + }}] + } + first_alias_expected_search = { + 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { + 'bool': {'must': [ + {'bool': {'should': [ + {'term': {'samples_num_alt_1': 'NA20870'}}, + {'term': {'samples_num_alt_2': 'NA20870'}}, + ]}} + ], + '_name': 'F000003_3' + }}] + } + self.assertExecutedSearches([ + dict(index=SECOND_SUB_INDICES[1], **second_alias_expected_search), + dict(index=SECOND_SUB_INDICES[0], **second_alias_expected_search), + dict(index=SUB_INDICES[1], **first_alias_expected_search), + dict(index=SUB_INDICES[0], **first_alias_expected_search), + ]) + _set_cache('index_metadata__{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME), None) + + @urllib3_responses.activate + def test_get_es_variant_gene_counts(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'qualityFilter': {'min_gq': 10}, + 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5'])) + + initial_cached_results = { + 'compound_het_results': [{'ENSG00000240361': PARSED_COMPOUND_HET_VARIANTS}], + 'variant_results': [PARSED_VARIANTS[1]], + 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], + 'duplicate_doc_count': 0, + 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, INDEX_NAME: {'loaded': 2, 'total': 5}}, + 'total_results': 7, + } + _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) + + # Test gene counts + gene_counts = get_variant_query_gene_counts(results_model, None) + self.assertDictEqual(gene_counts, { + 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1}}, + 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1}}, + 'ENSG00000240361': {'total': 2, 'families': {'F000003_3': 2}}, + }) + + self.assertExecutedSearch( + filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], + size=1, index=INDEX_NAME, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) + + expected_cached_results = {'gene_aggs': gene_counts} + expected_cached_results.update(initial_cached_results) + self.assertCachedResults(results_model, expected_cached_results) + + @urllib3_responses.activate + def test_multi_project_get_es_variant_gene_counts(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + 'qualityFilter': {'min_gq': 10}, + 'inheritance': {'mode': 'recessive'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2', 'F000005_5'])) + + initial_cached_results = { + 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], + 'variant_results': [PARSED_MULTI_INDEX_VARIANT], + 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], + 'duplicate_doc_count': 3, + 'loaded_variant_counts': { + SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, + '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 4, 'loaded': 4}, + INDEX_NAME: {'loaded': 2, 'total': 5}, + '{}_compound_het'.format(INDEX_NAME): {'total': 2, 'loaded': 2}, + }, + 'total_results': 13, + } + _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) + + # Test gene counts + gene_counts = get_variant_query_gene_counts(results_model, None) + self.assertDictEqual(gene_counts, { + 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}}, + 'ENSG00000228198': {'total': 4, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}} + }) + + expected_search = dict(size=1, start_index=0, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) + self.assertExecutedSearches([ + dict(filters=[ + ANNOTATION_QUERY, + {'bool': { + 'must': [ + {'term': {'samples_num_alt_2': 'NA20885'}}, + {'bool': { + 'must_not': [ + {'term': {'samples_gq_0_to_5': 'NA20885'}}, + {'term': {'samples_gq_5_to_10': 'NA20885'}}, + ] + }} + ], + '_name': 'F000011_11' + }} + ], index=SECOND_INDEX_NAME, **expected_search), + dict(filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], index=INDEX_NAME, **expected_search), + ]) + + expected_cached_results = {'gene_aggs': gene_counts} + expected_cached_results.update(initial_cached_results) + self.assertCachedResults(results_model, expected_cached_results) + + @urllib3_responses.activate + def test_multi_project_all_samples_all_inheritance_get_es_variant_gene_counts(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) + _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps({'total_results': 5})) + gene_counts = get_variant_query_gene_counts(results_model, None) + + self.assertDictEqual(gene_counts, { + 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}}, + 'ENSG00000228198': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}} + }) + + self.assertExecutedSearch( + index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), + filters=[ANNOTATION_QUERY], + size=1, + gene_count_aggs={ + 'samples': {'terms': {'field': 'samples', 'size': 10000}}, + 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, + 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} + } + ) + + self.assertCachedResults(results_model, {'gene_aggs': gene_counts, 'total_results': 5}) + + @urllib3_responses.activate + def test_all_samples_any_affected_get_es_variant_gene_counts(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) + gene_counts = get_variant_query_gene_counts(results_model, None) + + self.assertDictEqual(gene_counts, { + 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}}, + 'ENSG00000228198': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}} + }) + + self.assertExecutedSearches([dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731']}}, + {'terms': {'samples_num_alt_2': ['HG00731']}}, + {'terms': {'samples': ['HG00731']}}, + ] + }} + ], + index=MITO_WGS_INDEX_NAME, + start_index=0, + size=1, + gene_count_aggs={ + 'samples': {'terms': {'field': 'samples', 'size': 10000}}, + 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, + 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} + } + ), dict( + filters=[ + ANNOTATION_QUERY, + {'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, + {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, + ] + }} + ], + start_index=0, + size=1, + gene_count_aggs={ + 'samples': {'terms': {'field': 'samples', 'size': 10000}}, + 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, + 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} + } + )]) + + self.assertCachedResults(results_model, {'gene_aggs': gene_counts}) + + def test_get_family_affected_status(self): + samples_by_id = { + sample_id: Sample.objects.get(sample_id=sample_id, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + for sample_id in ['HG00731', 'HG00732', 'HG00733'] + } + custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} + custom_multi_affected = {'I000005_hg00732': 'A'} + + self.assertDictEqual(_get_family_affected_status(samples_by_id, {}), { + 'I000004_hg00731': 'A', 'I000005_hg00732': 'N', 'I000006_hg00733': 'N'}) + + custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_affected}) + self.assertDictEqual(custom_affected_status, { + 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) + + custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_multi_affected}) + self.assertDictEqual(custom_affected_status, { + 'I000004_hg00731': 'A', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) + + @urllib3_responses.activate + def test_sort(self): + setup_responses() + search_model = VariantSearch.objects.create(search={ + 'annotations': {'frameshift': ['frameshift_variant']}, + }) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) + + variants, _ = query_variants(results_model, sort='primate_ai', num_results=2) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY], sort=[ + {'primate_ai_score': {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}, 'xpos', 'variantId'], + index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4) + self.assertEqual(variants[0]['_sort'][0], maxsize) + self.assertEqual(variants[1]['_sort'][0], -1) + + variants, _ = query_variants(results_model, sort='gnomad', num_results=2) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ + { + '_script': { + 'type': 'number', + 'script': { + 'params': {'field': 'gnomad_genomes_AF'}, + 'source': mock.ANY, + } + } + }, 'xpos', 'variantId']) + self.assertEqual(variants[0]['_sort'][0], 0.00012925741614425127) + self.assertEqual(variants[1]['_sort'][0], maxsize) + + variants, _ = query_variants(results_model, sort='in_omim', num_results=2) + self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ + { + '_script': { + 'type': 'number', + 'script': { + 'params': { + 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] + }, + 'source': mock.ANY, + } + } + }, { + '_script': { + 'type': 'number', + 'script': { + 'params': { + 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] + }, + 'source': mock.ANY, + } + } + }, 'xpos', 'variantId']) + + results_model.families.set(Family.objects.filter(guid='F000001_1')) + query_variants(results_model, sort='prioritized_gene', num_results=2) + family_sample_filter = {'bool': {'_name': 'F000001_1', 'must': mock.ANY}} + self.assertExecutedSearch(filters=[ANNOTATION_QUERY, family_sample_filter], index=INDEX_NAME, size=2, sort=[ + { + '_script': { + 'type': 'number', + 'script': { + 'params': { + 'prioritized_ranks_by_gene': {'ENSG00000268903': 1, 'ENSG00000268904': 11} + }, + 'source': mock.ANY, + } + } + }, 'xpos', 'variantId']) + + @urllib3_responses.activate + def test_genotype_inheritance_filter(self): + setup_responses() + # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test + Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() + custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} + custom_multi_affected = {'I000005_hg00732': 'A'} + + search_model = VariantSearch.objects.create(search={}) + results_model = VariantSearchResults.objects.create(variant_search=search_model) + results_model.families.set(Family.objects.filter(guid='F000002_2')) + cache_key = 'search_results__{}__xpos'.format(results_model.guid) + + def _execute_inheritance_search( + mode=None, inheritance_filter=None, expected_filter=None, expected_comp_het_filter=None, + quality_filter=None, expected_quality_filter=None, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, **kwargs): + _set_cache(cache_key, None) + annotations = {'frameshift': ['frameshift_variant']} if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS \ + else {'structural': ['DEL', 'gCNV_DEL']} + search_model.search = { + 'inheritance': {'mode': mode, 'filter': inheritance_filter}, + 'annotations': annotations, + } + if quality_filter: + search_model.search['qualityFilter'] = quality_filter + search_model.save() + variants, _ = query_variants(results_model, num_results=2) + + if mode not in {'compound_het', 'recessive'}: + self.assertSetEqual( + set(variants[-1]['genotypes'].keys()), + {'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'}, + ) + + index = INDEX_NAME if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else SV_INDEX_NAME + annotation_query = {'terms': {'transcriptConsequenceTerms': next(iter(annotations.values()))}} + if expected_comp_het_filter: + self.assertExecutedSearches([ + dict(gene_aggs=True, start_index=0, size=1, index=index, filters=[ + annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_comp_het_filter]}} + ]), + dict(start_index=0, size=2, index=index, filters=[ + annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_filter]}}]) + ]) + else: + filters = [expected_filter] + if expected_quality_filter: + filters.append(expected_quality_filter) + self.assertExecutedSearch(index=index, filters=[ + annotation_query, {'bool': {'_name': 'F000002_2', 'must': filters}}], **kwargs) + + # custom genotype + _execute_inheritance_search( + inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref', 'I000005_hg00732': 'ref_alt'}}, + expected_filter={ + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ], + 'must': [ + {'term': {'samples_num_alt_1': 'HG00732'}} + ] + } + }) + + # de novo + _execute_inheritance_search(mode='de_novo', expected_filter={ + 'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}} + ] + } + }) + + _execute_inheritance_search(mode='de_novo', dataset_type='SV', expected_filter={'bool': { + 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], + 'must': [{'term': {'samples': 'HG00731'}}], + } + }) + + _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ + 'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}} + ] + } + }) + + _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ + 'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}} + ] + } + }, quality_filter={'affected_only': True, 'min_gq': 10}, expected_quality_filter={ + 'bool': {'must_not': [ + {'term': {'samples_gq_0_to_5': 'HG00732'}}, + {'term': {'samples_gq_5_to_10': 'HG00732'}}, + ]}} + ) + + _execute_inheritance_search( + mode='de_novo', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { + 'minimum_should_match': 1, + 'must_not': [ + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_1': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'should': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}} + ], + 'must': [{ + 'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}} + ]} + }] + } + }) + + recessive_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'term': {'samples_num_alt_2': 'HG00731'}} + ] + } + } + sv_recessive_filter = { + 'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_cn_0': 'HG00731'}}, + {'term': {'samples_cn_2': 'HG00731'}}, + {'term': {'samples_cn_gte_4': 'HG00731'}}, + ], + 'must_not': [ + {'term': {'samples_cn_0': 'HG00732'}}, + {'term': {'samples_cn_gte_4': 'HG00732'}}, + {'term': {'samples_cn_0': 'HG00733'}}, + {'term': {'samples_cn_gte_4': 'HG00733'}}, + ] + } + } + custom_affected_recessive_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'term': {'samples_num_alt_2': 'HG00732'}} + ] + } + } + + # homozygous recessive + _execute_inheritance_search(mode='homozygous_recessive', expected_filter=recessive_filter) + + _execute_inheritance_search(mode='homozygous_recessive', dataset_type='SV', expected_filter=sv_recessive_filter) + + _execute_inheritance_search( + mode='homozygous_recessive', inheritance_filter={'affected': custom_affected}, + expected_filter=custom_affected_recessive_filter) + + # compound het + com_het_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'term': {'samples_num_alt_1': 'HG00731'}}, + ] + } + } + + sv_com_het_filter = {'term': {'samples': 'HG00731'}} + + custom_affected_comp_het_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'term': {'samples_num_alt_1': 'HG00732'}}, + ] + } + } + + _execute_inheritance_search(mode='compound_het', expected_filter=com_het_filter, gene_aggs=True, size=1) + _execute_inheritance_search( + mode='compound_het', dataset_type='SV', gene_aggs=True, size=1, expected_filter=sv_com_het_filter) + _execute_inheritance_search( + mode='compound_het', inheritance_filter={'affected': custom_affected}, + expected_filter=custom_affected_comp_het_filter, gene_aggs=True, size=1) + + # x-linked recessive + x_linked_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00732'}}, + {'term': {'samples_num_alt_1': 'HG00732'}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + ] + } + } + + sv_x_linked_filter = { + 'bool': { + 'minimum_should_match': 1, + 'should': [ + {'term': {'samples_cn_0': 'HG00731'}}, + {'term': {'samples_cn_2': 'HG00731'}}, + {'term': {'samples_cn_gte_4': 'HG00731'}}, + ], + 'must_not': [ + {'term': {'samples': 'HG00732'}}, + {'term': {'samples_cn_0': 'HG00733'}}, + {'term': {'samples_cn_gte_4': 'HG00733'}}, + ], + 'must': [{'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}},], + } + } + + custom_affected_x_linked_filter = { + 'bool': { + 'must_not': [ + {'term': {'samples_no_call': 'HG00731'}}, + {'term': {'samples_num_alt_2': 'HG00731'}}, + {'term': {'samples_no_call': 'HG00733'}}, + {'term': {'samples_num_alt_2': 'HG00733'}} + ], + 'must': [ + {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, + {'term': {'samples_num_alt_2': 'HG00732'}}, + ] + } + } + + _execute_inheritance_search(mode='x_linked_recessive', expected_filter=x_linked_filter) + + _execute_inheritance_search(mode='x_linked_recessive', dataset_type='SV', expected_filter=sv_x_linked_filter) + + _execute_inheritance_search( + mode='x_linked_recessive', inheritance_filter={'affected': custom_affected}, + expected_filter=custom_affected_x_linked_filter) + + # recessive + _execute_inheritance_search(mode='recessive', expected_comp_het_filter=com_het_filter, expected_filter=recessive_filter) + + _execute_inheritance_search( + mode='recessive', dataset_type='SV', expected_comp_het_filter=sv_com_het_filter, expected_filter=sv_recessive_filter) + + _execute_inheritance_search( + mode='recessive', inheritance_filter={'affected': custom_affected}, + expected_comp_het_filter=custom_affected_comp_het_filter, expected_filter=custom_affected_recessive_filter) + + # any affected + _execute_inheritance_search(mode='any_affected', expected_filter={ + 'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731']}}, + {'terms': {'samples_num_alt_2': ['HG00731']}}, + {'terms': {'samples': ['HG00731']}}, + ] + } + }) + + _execute_inheritance_search( + mode='any_affected', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { + 'should': [ + {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732']}}, + {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732']}}, + {'terms': {'samples': ['HG00731', 'HG00732']}}, + ] + } + }) From 4fc3160572b1e2110f2427eeccb62863e8c28885 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:45:06 -0500 Subject: [PATCH 021/252] clean up es tests --- .../search/elasticsearch/es_utils_tests.py | 2380 +---------------- 1 file changed, 2 insertions(+), 2378 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 85651ec04f..0a3a348d27 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -6,20 +6,13 @@ from collections import defaultdict from datetime import timedelta from django.test import TestCase -from elasticsearch.exceptions import ConnectionTimeout, TransportError from sys import maxsize -from urllib3.exceptions import ReadTimeoutError from urllib3_mock import Responses -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults +from seqr.models import Family, Sample from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, InvalidSearchException from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status -from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException -from seqr.utils.search.utils import _get_liftover -from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_SV_WGS_VARIANT,\ - PARSED_MITO_VARIANT, TRANSCRIPT_2, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT +from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT # The responses library for mocking requests does not work with urllib3 (which is used by elasticsearch) @@ -1407,1971 +1400,6 @@ def test_get_es_variants_for_variant_ids(self): size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), ) - @urllib3_responses.activate - def test_get_single_es_variant(self): - setup_responses() - family = self.families.filter(guid='F000003_3').first() - variant = get_single_variant(family, '2-103343353-GAGA-G') - self.assertDictEqual(variant, {**PARSED_ANY_AFFECTED_VARIANTS[1], 'selectedMainTranscriptId': None}) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['2-103343353-GAGA-G']}}], - size=1, index=INDEX_NAME - ) - - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, 'prefix_19107_DEL') - self.assertDictEqual(variant, PARSED_SV_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['prefix_19107_DEL']}}], size=1, index=SV_INDEX_NAME, - ) - - variant = get_single_variant(family, 'M-10195-C-A') - self.assertDictEqual(variant, PARSED_MITO_VARIANT) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['M-10195-C-A']}}], size=1, index=MITO_WGS_INDEX_NAME, - ) - - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G') - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_gene_agg_search.MAX_COMPOUND_HET_GENES', 1) - @mock.patch('seqr.utils.search.elasticsearch.es_search.logger') - @urllib3_responses.activate - def test_invalid_get_es_variants(self, mock_logger): - setup_responses() - mito_sample = Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME) - mito_sample.individual_id = 6 - mito_sample.save() - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - - results_model.families.set(self.families) - Sample.objects.filter(elasticsearch_index=INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), 'The following indices do not have the expected genome version 37: test_index_hg38 (38)', - ) - Sample.objects.filter(elasticsearch_index=HG38_INDEX_NAME).update(elasticsearch_index=INDEX_NAME) - - results_model.families.set(self.families) - search_model.search = { - 'inheritance': {'mode': 'compound_het'}, - 'locus': {'rawItems': 'WASH7P'}, - 'annotations': {'frameshift': ['frameshift_variant']}, - } - search_model.save() - with self.assertRaises(InvalidSearchException) as cm: - query_variants(results_model) - self.assertEqual( - str(cm.exception), - 'This search returned too many compound heterozygous variants. Please add stricter filters') - - with self.assertRaises(InvalidSearchException) as cm: - get_variant_query_gene_counts(results_model, None) - self.assertEqual(str(cm.exception), 'This search returned too many genes') - - search_model.search = {'qualityFilter': {'min_gq': 7}} - search_model.save() - with self.assertRaises(Exception) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Invalid gq filter 7') - - search_model.search = {} - search_model.save() - urllib3_responses.reset() - urllib3_responses.add( - urllib3_responses.POST, '/_msearch', body=ReadTimeoutError('', '', 'timeout')) - urllib3_responses.add_json('/_tasks?actions=*search&group_by=parents', {'tasks': { - 123: {'running_time_in_nanos': 10}, - 456: {'running_time_in_nanos': 10 ** 12}, - }}) - with self.assertRaises(ConnectionTimeout): - query_variants(results_model) - self.assertListEqual( - [call.request.url for call in urllib3_responses.calls], - ['/_msearch', '/_tasks?actions=%2Asearch&group_by=parents']) - mock_logger.error.assert_called_with('ES Query Timeout: Found 1 long running searches', None, detail=[ - {'task': {'running_time_in_nanos': 10 ** 12}, 'parent_task_id': '456'}, - ]) - - urllib3_responses.reset() - urllib3_responses.add_json('/_msearch', {'responses': [ - {'error': {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}} - ]}, method=urllib3_responses.POST) - - with self.assertRaises(TransportError) as cm: - query_variants(results_model) - self.assertDictEqual( - cm.exception.info, - {'type': 'search_phase_execution_exception', 'root_cause': [{'type': 'too_many_clauses'}]}) - - _set_cache(f'index_metadata__{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}', None) - urllib3_responses.add( - urllib3_responses.GET, f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', body=Exception('Connection error')) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'test_index,test_index_mito_wgs,test_index_sv - Error accessing index: Connection error') - - urllib3_responses.replace_json(f'/{INDEX_NAME},{MITO_WGS_INDEX_NAME},{SV_INDEX_NAME}/_mapping', {}) - with self.assertRaises(InvalidIndexException) as cm: - query_variants(results_model) - self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') - - @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) - @urllib3_responses.activate - def test_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={'annotations': {'frameshift': ['frameshift_variant']}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_VARIANTS[1]) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY]) - - # does not save non-consecutive pages - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': variants, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=2) - - # test pagination - variants, total_results = query_variants(results_model, page=2, num_results=2) - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 5) - self.assertCachedResults(results_model, {'all_results': PARSED_VARIANTS + PARSED_VARIANTS, 'total_results': 5}) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=2, size=2) - - # test does not re-fetch page - urllib3_responses.reset() - variants, total_results = query_variants(results_model, page=1, num_results=3) - self.assertEqual(len(variants), 3) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS[:1]) - self.assertEqual(total_results, 5) - - # test load_all - setup_responses() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - # test does not re-fetch once all loaded - urllib3_responses.reset() - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) - self.assertEqual(len(variants), 5) - self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) - - @urllib3_responses.activate - def test_filtered_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'pathogenicity': { - 'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting'], - 'hgmd': ['disease_causing', 'likely_disease_causing'], - }, - 'annotations': { - 'in_frame': ['inframe_insertion', 'inframe_deletion'], - 'other': ['5_prime_UTR_variant', 'intergenic_variant'], - 'SCREEN': ['dELS', 'DNase-only'], - 'splice_ai': '0.8', - }, - 'freqs': { - 'callset': {'af': 0.1}, - 'exac': {'ac': 2, 'af': None}, - 'g1k': {'ac': None, 'af': 0.001}, - 'gnomad_exomes': {'af': 0.01, 'ac': 3, 'hh': 3}, - 'gnomad_genomes': {'af': 0.01, 'hh': 3}, - 'topmed': {'ac': 2, 'af': None}, - }, - 'qualityFilter': {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass', 'affected_only': True}, - 'in_silico': {'cadd': '11.5', 'sift': 'D', 'fathmm': 'D'}, - 'inheritance': {'mode': 'de_novo'}, - 'customQuery': {'term': {'customFlag': 'flagVal'}}, - 'exclude': {'rawItems': 'WASH7P, chr2:1234-5678, chr7:100-10100%10'}, - }) - - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - variants, total_results = query_variants(results_model, sort='cadd', num_results=2) - - self.assertListEqual(variants, PARSED_CADD_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(filters=[ - {'term': {'customFlag': 'flagVal'}}, - { - 'bool': { - 'must_not': [ - {'range': {'xpos': {'gte': 2000001234, 'lte': 2000005678}}}, - {'range': {'xstop': {'gte': 2000001234, 'lte': 2000005678}}}, - {'bool': {'must': [ - {'range': {'xpos': {'lte': 2000001234}}}, - {'range': {'xstop': {'gte': 2000005678}}}, - {'range': {'xpos': {'gte': 2000000001}}}, - {'range': {'xstop': {'lte': 2300000000}}}, - ]}}, - {'terms': {'geneIds': ['ENSG00000227232']}}, - {'bool': {'must': [ - {'range': {'xpos': {'gte': 7000000001, 'lte': 7000001100}}}, - {'range': {'xstop': {'gte': 7000009100, 'lte': 7000011100}}}]}}, - ] - } - }, - {'bool': {'should': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'exac_AC_Adj'}}]}}, - {'range': {'exac_AC_Adj': {'lte': 2}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hom'}}]}}, - {'range': {'gnomad_exomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_Hemi'}}]}}, - {'range': {'gnomad_exomes_Hemi': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.01}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hom'}}]}}, - {'range': {'gnomad_genomes_Hom': {'lte': 3}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_Hemi'}}]}}, - {'range': {'gnomad_genomes_Hemi': {'lte': 3}}} - ]} - }, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'topmed_AC'}}]}}, - {'range': {'topmed_AC': {'lte': 2}}} - ]} - } - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'AF'}}]}}, - {'range': {'AF': {'lte': 0.1}}} - ], - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_exomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_exomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'gnomad_genomes_AF_POPMAX_OR_GLOBAL'}}]}}, - {'range': {'gnomad_genomes_AF_POPMAX_OR_GLOBAL': {'lte': 0.05}}} - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }} - ] - }}, - - ]}}, - {'bool': {'should': [ - {'bool': {'must_not': [{'exists': {'field': 'cadd_PHRED'}}]}}, - {'range': {'cadd_PHRED': {'gte': 11.5}}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_SIFT_pred'}}]}}, - {'prefix': {'dbnsfp_SIFT_pred': 'D'}}, - {'bool': {'must_not': [{'exists': {'field': 'dbnsfp_fathmm_MKL_coding_pred'}}]}}, - {'prefix': {'dbnsfp_fathmm_MKL_coding_pred': 'D'}}, - ]}}, - {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}}, - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'transcriptConsequenceTerms'}}]}}, - {'terms': { - 'transcriptConsequenceTerms': [ - '5_prime_UTR_variant', - 'inframe_deletion', - 'inframe_insertion', - 'intergenic_variant', - ] - }}, - {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*|Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))', - }}, - {'terms': {'hgmd_class': ['DM', 'DM?']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.8}}}, - {'terms': {'screen_region_type': ['dELS', 'DNase-only']}}, - ] - } - }, - {'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - }}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'HG00731'}}, - {'term': {'samples_ab_5_to_10': 'HG00731'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'HG00731'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_10_to_15': 'HG00731'}}, - ], - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*' - }}]}} - ], - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}} - ]}}, - {'bool': {'should': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'bool': { - 'must_not': [ - {'term': {'samples_ab_0_to_5': 'NA20870'}}, - {'term': {'samples_ab_5_to_10': 'NA20870'}}, - ] - }}, - {'bool': {'must_not': [{'term': {'samples_num_alt_1': 'NA20870'}}]}} - ], - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - {'term': {'samples_gq_10_to_15': 'NA20870'}}, - ] - }}, {'regexp': { - 'clinvar_clinical_significance': '.*Likely_pathogenic.*|.*Pathogenic.*', - }}]}} - ], - '_name': 'F000003_3' - }}, - ] - }} - ], sort=[{'cadd_PHRED': {'order': 'desc', 'unmapped_type': 'keyword'}}, 'xpos', 'variantId']) - - # Test sort does not error on pagination - query_variants(results_model, sort='cadd', num_results=2, page=2) - - @urllib3_responses.activate - def test_sv_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'new_structural_variants': ['NEW']}, - 'freqs': {'sv_callset': {'af': 0.1}}, - 'in_silico': {'strvctvre': '3.1', 'requireScore': True}, - 'qualityFilter': {'min_qs': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - - self.assertListEqual(variants, [PARSED_SV_VARIANT]) - self.assertExecutedSearch(filters=[ - {'bool': { - 'should': [ - {'bool': {'must_not': [{'exists': {'field': 'sf'}}]}}, - {'range': {'sf': {'lte': 0.1}}} - ] - }}, - {'range': {'StrVCTVRE_score': {'gte': 3.1}}}, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_qs_0_to_10': 'HG00731'}}, - {'term': {'samples_qs_10_to_20': 'HG00731'}}, - {'term': {'samples_qs_0_to_10': 'HG00732'}}, - {'term': {'samples_qs_10_to_20': 'HG00732'}}, - {'term': {'samples_qs_0_to_10': 'HG00733'}}, - {'term': {'samples_qs_10_to_20': 'HG00733'}}, - ], - 'must': [{'terms': {'samples_new_call': ['HG00731', 'HG00732', 'HG00733']}}], - }}, - ], - '_name': 'F000002_2' - }} - ], index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_sv_wgs_get_es_variants(self): - self.families = Family.objects.filter(guid='F000014_14') - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DUP', 'CPX']}, - 'qualityFilter': {'min_gq_sv': 20}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=2) - self.assertListEqual(variants, [PARSED_SV_WGS_VARIANT]) - - self.assertExecutedSearch(filters=[ - {'terms': {'transcriptConsequenceTerms': ['CPX', 'DUP']}}, - {'bool': { - 'must': [{'bool': {'must': [{'term': {'samples': 'NA21234'}}, {'term': {'samples': 'NA21987'}}], - 'must_not': [{'term': {'samples': 'NA21654'}}]}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_sv_0_to_5': 'NA21234'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21234'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21234'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21234'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21654'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21654'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21654'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21654'}}, - {'term': {'samples_gq_sv_0_to_5': 'NA21987'}}, - {'term': {'samples_gq_sv_5_to_10': 'NA21987'}}, - {'term': {'samples_gq_sv_10_to_15': 'NA21987'}}, - {'term': {'samples_gq_sv_15_to_20': 'NA21987'}}, - ], - }} - ], - '_name': 'F000014_14' - }} - ], index=SV_WGS_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_get_es_variants(self): - setup_responses() - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, [PARSED_SV_VARIANT] + PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - self.assertExecutedSearches([ - dict(filters=[], start_index=0, size=5, index=SV_INDEX_NAME), - dict(filters=[], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['pathogenicity'] = {'clinvar': ['pathogenic']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - variants, _ = query_variants(results_model, num_results=5) - self.assertListEqual(variants, PARSED_NO_CONSEQUENCE_FILTER_VARIANTS + [PARSED_MITO_VARIANT]) - path_filter = {'regexp': { - 'clinvar_clinical_significance': '.*Pathogenic.*' - }} - self.assertExecutedSearches([ - dict(filters=[path_filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[path_filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - # test with dataset filtering applied - search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - filter = {'bool': {'should': [{'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}}, path_filter]}} - self.assertExecutedSearches([ - dict(filters=[filter], start_index=0, size=5, index=MITO_WGS_INDEX_NAME), - dict(filters=[filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, index=INDEX_NAME), - ]) - - search_model.search['annotations'] = { - 'structural': ['DEL'], 'structural_consequence': ['MSV_EXON_OVERLAP', 'INTRAGENIC_EXON_DUP']} - search_model.search['pathogenicity'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - query_variants(results_model, num_results=5) - self.assertExecutedSearch( - filters=[{'terms': { - 'transcriptConsequenceTerms': ['DEL', 'DUP_LOF', 'INTRAGENIC_EXON_DUP', 'MSV_EXON_OVERLAP', 'MSV_EXON_OVR'] - }}], - start_index=0, size=5, index=SV_INDEX_NAME) - - @urllib3_responses.activate - def test_multi_dataset_no_affected_inheritance_get_es_variants(self): - setup_responses() - # The family has multiple data types loaded but only one loaded in an affected individual - Sample.objects.get(individual_id=4, elasticsearch_index=SV_INDEX_NAME).delete() - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - - search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - - query_variants(results_model, num_results=2) - self.assertExecutedSearch(filters=[{'bool': { - 'must': [{'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], - }}], '_name': 'F000002_2', - }}]) - - @urllib3_responses.activate - def test_compound_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_compound_het_get_es_variants_secondary_annotation(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'qualityFilter': {'min_gq': 10}, - 'annotations': {'frameshift': ['frameshift_variant'], 'splice_ai': '0.5'}, - 'inheritance': {'mode': 'compound_het'}, - 'annotations_secondary': {'other': ['intron']}, - 'pathogenicity': {'clinvar': ['pathogenic'], 'hgmd': ['disease_causing']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 1) - self.assertListEqual(variants, [PARSED_COMPOUND_HET_VARIANTS]) - self.assertEqual(total_results, 1) - - self.assertCachedResults(results_model, { - 'grouped_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'total_results': 1, - }) - - annotation_query = {'bool': {'should': [ - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}}, - {'terms': {'hgmd_class': ['DM']}}, - {'range': {'splice_ai_delta_score': {'gte': 0.5}}}, - {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}}, - ]}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_PATH_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - # test pagination does not fetch - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - # variants require both primary and secondary annotations - setup_responses() - del search_model.search['pathogenicity'] - del search_model.search['annotations']['splice_ai'] - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertIsNone(variants) - self.assertEqual(total_results, 0) - - annotation_query = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant', 'intron']}} - - self.assertExecutedSearch( - filters=[annotation_query, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ) - - @urllib3_responses.activate - def test_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10, 'vcf_filter': 'pass'}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS[1]) - self.assertEqual(total_results, 6) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 6, - }) - - pass_filter_query = {'bool': {'must_not': [{'exists': {'field': 'filters'}}]}} - - self.assertExecutedSearches([ - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=0, size=2, - ), - ]) - - # test pagination - - variants, total_results = query_variants(results_model, page=3, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1], PARSED_MULTI_SAMPLE_VARIANT) - self.assertEqual(total_results, 5) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}, - {'null': [PARSED_VARIANTS[0]]}, {'null': [PARSED_MULTI_SAMPLE_VARIANT]}], - 'duplicate_doc_count': 1, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 1, 'loaded': 1}, INDEX_NAME: {'loaded': 4, 'total': 5}}, - 'total_results': 5, - }) - - self.assertExecutedSearches([dict(filters=[pass_filter_query, ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], start_index=2, size=4)]) - - urllib3_responses.reset() - query_variants(results_model, page=2, num_results=2) - - @urllib3_responses.activate - def test_multi_datatype_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'recessive'}, - 'annotations': {'frameshift': ['frameshift_variant'], 'structural': ['DEL']} - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 4) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[2][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[2][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - self.assertDictEqual(variants[3], PARSED_VARIANTS[1]) - - annotations_q = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - self.assertExecutedSearches([ - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[ - annotations_q, - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotations_q, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - annotations_q, - { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - }}, - ] - } - } - ], start_index=0, size=10, - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_recessive_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL']}, - 'annotations_secondary': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, _ = query_variants(results_model, num_results=10) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_SV_VARIANT) - self.assertDictEqual(variants[1][0], PARSED_SV_COMPOUND_HET_VARIANTS[0]) - self.assertDictEqual(variants[1][1], PARSED_SV_COMPOUND_HET_VARIANTS[1]) - - annotation_secondary_query = {'terms': {'transcriptConsequenceTerms': ['DEL', 'frameshift_variant']}} - - self.assertExecutedSearches([ - dict( - filters=[ - {'terms': {'transcriptConsequenceTerms': ['DEL']}}, - {'bool': { - '_name': 'F000002_2', - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - }] - }} - ], start_index=0, size=10, index=SV_INDEX_NAME, - ), - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_multi_datatype_secondary_annotations_comp_het_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'structural': ['DEL'], 'SCREEN': ['dELS']}, - 'annotations_secondary': {'structural_consequence': ['LOF']}, - 'inheritance': {'mode': 'compound_het'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - query_variants(results_model, num_results=10) - - annotation_secondary_query = {'bool': {'should': [ - {'terms': {'transcriptConsequenceTerms': ['DEL', 'LOF']}}, - {'terms': {'screen_region_type': ['dELS']}}]}} - - self.assertExecutedSearches([ - dict( - filters=[annotation_secondary_query, {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'should': [ - {'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - }}, - {'term': {'samples': 'HG00731'}}, - ] - }}, - ] - }}, - ], - gene_aggs=True, - start_index=0, - size=1, - index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ), - dict( - filters=[ - annotation_secondary_query, - {'bool': {'_name': 'F000003_3', 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}]}}, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - ]) - - @urllib3_responses.activate - def test_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_VARIANTS) - self.assertEqual(total_results, 5) - - self.assertExecutedSearch(index=INDEX_NAME, size=2, filters=[ - {'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G']}}, ANNOTATION_QUERY]) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertListEqual(variants, PARSED_ANY_AFFECTED_VARIANTS) - self.assertEqual(total_results, 10) - - self.assertExecutedSearches([dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], index=MITO_WGS_INDEX_NAME, start_index=0, size=2), dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], index=INDEX_NAME, start_index=0, size=2)]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_SEARCH_CLAUSES', 1) - @urllib3_responses.activate - def test_many_family_inheitance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(self.families) - - variants, total_results = query_variants(results_model, num_results=2) - - self.assertEqual(len(variants), 2) - self.assertEqual(total_results, 14) - self.assertDictEqual(variants[0], PARSED_MULTI_SAMPLE_VARIANT_0) - self.assertListEqual(variants[1], PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS) - - self.assertCachedResults(results_model, { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_VARIANT, PARSED_MITO_VARIANT], - 'grouped_results': [{'null': [PARSED_MULTI_SAMPLE_VARIANT_0]}, {'ENSG00000228198': PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, - MITO_WGS_INDEX_NAME: {'loaded': 1, 'total': 5}, - INDEX_NAME: {'loaded': 4, 'total': 10}}, - 'total_results': 14, - }) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - }, - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - ] - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [{'term': {'samples_num_alt_1': 'NA20870'}}], - }, - }, - ], - gene_aggs=True, - start_index=0, - size=1 - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - ] - } - }, - ], - start_index=0, - size=2, - ), - dict( - filters=[ - ANNOTATION_QUERY, - { - 'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - ] - } - }, - ], - start_index=0, - size=2, - ), - ]) - - @urllib3_responses.activate - def test_multi_project_get_es_variants(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2'])) - - variants, total_results = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertDictEqual(variants[0], PARSED_VARIANTS[0]) - self.assertDictEqual(variants[1][0], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[0]) - self.assertDictEqual(variants[1][1], PARSED_COMPOUND_HET_VARIANTS_PROJECT_2[1]) - self.assertEqual(total_results, 11) - - self.assertCachedResults(results_model, { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 2, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 11, - }) - self.assertTrue('index_metadata__{}'.format(INDEX_NAME) in REDIS_CACHE) - self.assertTrue('index_metadata__{}'.format(SECOND_INDEX_NAME) in REDIS_CACHE) - - project_2_search = dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME) - project_1_search = dict( - filters=[ - ANNOTATION_QUERY, - RECESSIVE_INHERITANCE_QUERY, - ], start_index=0, size=2, index=INDEX_NAME) - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - '_name': 'F000011_11', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}} - ]}} - ] - }} - ], - gene_aggs=True, start_index=0, size=1, index=SECOND_INDEX_NAME, - ), - project_2_search, - dict( - filters=[ANNOTATION_QUERY, COMPOUND_HET_INHERITANCE_QUERY], - gene_aggs=True, start_index=0, size=1, index=INDEX_NAME, - ), - project_1_search, - ]) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_VARIANTS[0], PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT]) - self.assertEqual(total_results, 9) - - cache_results = { - 'compound_het_results': [], - 'variant_results': [PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT], - 'grouped_results': [ - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}, - {'null': [PARSED_VARIANTS[0]]}, - {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT} - ], - 'duplicate_doc_count': 4, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 2, 'loaded': 2}, - INDEX_NAME: {'loaded': 4, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 1, 'loaded': 1}, - }, - 'total_results': 9, - } - self.assertCachedResults(results_model, cache_results) - - project_2_search['start_index'] = 1 - project_2_search['size'] = 3 - project_1_search['start_index'] = 2 - self.assertExecutedSearches([project_2_search, project_1_search]) - - # If one project is fully loaded, only query the second project - cache_results['loaded_variant_counts'][INDEX_NAME]['total'] = 4 - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(cache_results)) - query_variants(results_model, num_results=2, page=3) - project_2_search['start_index'] = 2 - project_2_search['size'] = 4 - self.assertExecutedSearches([project_2_search]) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_MULTI_INDEX_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 4) - - cached_variants = expected_variants + [PARSED_MITO_VARIANT] - self.assertCachedResults(results_model, { - 'all_results': cached_variants, - 'duplicate_doc_count': 1, - 'total_results': 4, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME, MITO_WGS_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=6, - ) - - # test pagination - variants, total_results = query_variants(results_model, num_results=2, page=2) - expected_variants = [PARSED_MITO_VARIANT, PARSED_VARIANTS[0]] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 3) - - self.assertCachedResults(results_model, { - 'all_results': cached_variants + cached_variants, - 'duplicate_doc_count': 2, - 'total_results': 3, - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=8, - start_index=4, - ) - - # test skipping page fetches all consecutively - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, page=2) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=12, - ) - - @urllib3_responses.activate - def test_multi_project_all_samples_any_affected_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }, - ) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, total_results = query_variants(results_model, num_results=2) - expected_variants = [PARSED_VARIANTS[0], PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT] - self.assertListEqual(variants, expected_variants) - self.assertEqual(total_results, 14) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_SEARCHES', 1) - @urllib3_responses.activate - def test_multi_project_prefilter_indices_get_es_variants(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'inheritance': {'mode': 'de_novo'}, - 'qualityFilter': {'min_gq': 10}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 14) - self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) - - gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} - prefilter_search = dict( - filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', - size=200, expected_source_fields=set(), - ) - sv_search = dict( - filters=[ - gene_filter, - {'bool': { - 'must': [ - {'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - }}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }} - ], start_index=0, size=2, index=SV_INDEX_NAME) - num_calls = 3 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - # Search total is greater than returned hits, so proceed with regular multi-search - self.assertExecutedSearches([ - sv_search, - dict(filters=[ - gene_filter, - { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict(filters=[ - gene_filter, - {'bool': {'should': [ - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA19675'}}, - {'term': {'samples_num_alt_2': 'NA19675'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA19675'}}, - {'term': {'samples_gq_5_to_10': 'NA19675'}}, - ]}}], '_name': 'F000001_1'}}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}}, - ], 'minimum_should_match': 1}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ], - }}, - ], - '_name': 'F000002_2' - }}, - {'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}}, - ]}}], '_name': 'F000003_3'}}, - ]}}], start_index=0, size=2, index=INDEX_NAME), - ]) - - # Test successful prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000186092'] - search_model.search['locus']['rawItems'] = 'ENSG00000186092' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 1) - num_calls += 2 - self.assertEqual(len(urllib3_responses.calls), num_calls) - self.assertExecutedSearch(call_index=num_calls-2, **prefilter_search) - sv_search['query'] = [{'ids': {'values': ['prefix_19107_DEL']}}] - self.assertExecutedSearches([sv_search]) - - # Test no results in prefilter - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - gene_filter['terms']['geneIds'] = ['ENSG00000269732'] - search_model.search['locus']['rawItems'] = 'ENSG00000269732' - search_model.save() - - _, total_results = query_variants(results_model, num_results=2) - self.assertEqual(total_results, 0) - # Only the prefliter search is run, no multi-search - self.assertEqual(len(urllib3_responses.calls), num_calls + 1) - self.assertExecutedSearch(**prefilter_search) - - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_VARIANTS', 3) - @urllib3_responses.activate - def test_skip_genotype_filter(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'locus': {'rawItems': 'ENSG00000228198'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - variants, _ = query_variants(results_model, num_results=2, skip_genotype_filter=True) - expected_transcript_variant = deepcopy(PARSED_VARIANTS[0]) - expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] - self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) - self.assertExecutedSearch( - index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), - filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], - size=3, - ) - - # test with inheritance override - search_model.search['inheritance'] = {'mode': 'any_affected'} - search_model.search['locus'] = None - search_model.save() - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - query_variants(results_model, num_results=2, skip_genotype_filter=True) - - self.assertExecutedSearches([ - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['NA20885']}}, - {'terms': {'samples_num_alt_2': ['NA20885']}}, - {'terms': {'samples': ['NA20885']}}, - ] - }} - ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), - dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }}, - ], start_index=0, size=2, index=INDEX_NAME) - ]) - - @mock.patch('seqr.utils.search.utils.LIFTOVERS', {'37': None}) - @mock.patch('seqr.utils.search.utils.LiftOver') - @urllib3_responses.activate - def test_get_lifted_grch38_variants(self, mock_liftover): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - Project.objects.filter(guid='R0003_test').update(genome_version='38') - Sample.objects.filter(elasticsearch_index=SECOND_INDEX_NAME).update(elasticsearch_index=HG38_INDEX_NAME) - - mock_liftover.side_effect = Exception() - expected_no_lift_grch38_variant = deepcopy(PARSED_HG38_VARIANT) - expected_no_lift_grch38_variant.update({ - 'liftedOverGenomeVersion': None, - 'liftedOverChrom': None, - 'liftedOverPos': None, - }) - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, expected_no_lift_grch38_variant]) - self.assertIsNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - _set_cache('search_results__{}__xpos'.format(results_model.guid), None) - mock_liftover.side_effect = None - mock_liftover.return_value.convert_coordinate.side_effect = lambda chrom, pos: [[chrom, pos - 10]] - variants, _ = query_variants(results_model, num_results=2) - self.assertEqual(len(variants), 2) - self.assertListEqual(variants, [PARSED_HG38_VARIANT, PARSED_HG38_VARIANT]) - self.assertIsNotNone(_get_liftover('37')) - mock_liftover.assert_called_with('hg38', 'hg19') - - @mock.patch('seqr.utils.search.elasticsearch.es_search.MAX_INDEX_NAME_LENGTH', 30) - @urllib3_responses.activate - def test_get_es_variants_create_index_alias(self): - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - - setup_search_responses() - urllib3_responses.add_json( - '/{}/_mapping'.format(INDEX_ALIAS), {k: {'mappings': v} for k, v in CORE_INDEX_METADATA.items()}) - urllib3_responses.add_json('/_aliases', {'success': True}, method=urllib3_responses.POST) - - query_variants(results_model, num_results=2) - - self.assertExecutedSearch(index=INDEX_ALIAS, size=8) - self.assertDictEqual(urllib3_responses.call_request_json(index=0), { - 'actions': [{'add': {'indices': [INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME], 'alias': INDEX_ALIAS}}]}) - - @urllib3_responses.activate - def test_get_es_variants_search_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11'])) - - setup_search_responses() - urllib3_responses.add_json('/{}/_mapping'.format(SECOND_INDEX_NAME), { - k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SUB_INDICES}) - - query_variants(results_model, num_results=2) - - expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - self.assertExecutedSearches([ - dict(index=SUB_INDICES[1], **expected_search), - dict(index=SUB_INDICES[0], **expected_search), - ]) - _set_cache('index_metadata__{}'.format(SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variants_search_multiple_index_alias(self): - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'inheritance': {'mode': 'de_novo'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000011_11'])) - - setup_search_responses() - mappings = {k: {'mappings': INDEX_METADATA[INDEX_NAME]} for k in SUB_INDICES} - mappings.update({k: {'mappings': INDEX_METADATA[SECOND_INDEX_NAME]} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_mapping'.format(INDEX_NAME, SECOND_INDEX_NAME), mappings) - aliases = {k: {'aliases': {INDEX_NAME: {}}} for k in SUB_INDICES} - aliases.update({k: {'aliases': {SECOND_INDEX_NAME: {}, INDEX_ALIAS: {}}} for k in SECOND_SUB_INDICES}) - urllib3_responses.add_json('/{},{}/_alias'.format(INDEX_NAME, SECOND_INDEX_NAME), aliases) - - query_variants(results_model, num_results=2) - - second_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20885'}}, - {'term': {'samples_num_alt_2': 'NA20885'}}, - ]}} - ], - '_name': 'F000011_11' - }}] - } - first_alias_expected_search = { - 'start_index': 0, 'size': 2, 'filters': [ANNOTATION_QUERY, { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'term': {'samples_num_alt_2': 'NA20870'}}, - ]}} - ], - '_name': 'F000003_3' - }}] - } - self.assertExecutedSearches([ - dict(index=SECOND_SUB_INDICES[1], **second_alias_expected_search), - dict(index=SECOND_SUB_INDICES[0], **second_alias_expected_search), - dict(index=SUB_INDICES[1], **first_alias_expected_search), - dict(index=SUB_INDICES[0], **first_alias_expected_search), - ]) - _set_cache('index_metadata__{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME), None) - - @urllib3_responses.activate - def test_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000240361': PARSED_COMPOUND_HET_VARIANTS}], - 'variant_results': [PARSED_VARIANTS[1]], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS}], - 'duplicate_doc_count': 0, - 'loaded_variant_counts': {'test_index_compound_het': {'total': 2, 'loaded': 2}, INDEX_NAME: {'loaded': 2, 'total': 5}}, - 'total_results': 7, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1}}, - 'ENSG00000240361': {'total': 2, 'families': {'F000003_3': 2}}, - }) - - self.assertExecutedSearch( - filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], - size=1, index=INDEX_NAME, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_get_es_variant_gene_counts(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - 'qualityFilter': {'min_gq': 10}, - 'inheritance': {'mode': 'recessive'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid__in=['F000011_11', 'F000003_3', 'F000002_2', 'F000005_5'])) - - initial_cached_results = { - 'compound_het_results': [{'ENSG00000228198': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT}], - 'variant_results': [PARSED_MULTI_INDEX_VARIANT], - 'grouped_results': [{'null': [PARSED_VARIANTS[0]]}, {'ENSG00000135953': PARSED_COMPOUND_HET_VARIANTS_PROJECT_2}], - 'duplicate_doc_count': 3, - 'loaded_variant_counts': { - SECOND_INDEX_NAME: {'loaded': 1, 'total': 5}, - '{}_compound_het'.format(SECOND_INDEX_NAME): {'total': 4, 'loaded': 4}, - INDEX_NAME: {'loaded': 2, 'total': 5}, - '{}_compound_het'.format(INDEX_NAME): {'total': 2, 'loaded': 2}, - }, - 'total_results': 13, - } - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps(initial_cached_results)) - - # Test gene counts - gene_counts = get_variant_query_gene_counts(results_model, None) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 4, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000005_5': 1, 'F000011_11': 4}} - }) - - expected_search = dict(size=1, start_index=0, gene_count_aggs={'vars_by_gene': {'top_hits': {'_source': 'none', 'size': 100}}}) - self.assertExecutedSearches([ - dict(filters=[ - ANNOTATION_QUERY, - {'bool': { - 'must': [ - {'term': {'samples_num_alt_2': 'NA20885'}}, - {'bool': { - 'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20885'}}, - {'term': {'samples_gq_5_to_10': 'NA20885'}}, - ] - }} - ], - '_name': 'F000011_11' - }} - ], index=SECOND_INDEX_NAME, **expected_search), - dict(filters=[ANNOTATION_QUERY, RECESSIVE_INHERITANCE_QUERY], index=INDEX_NAME, **expected_search), - ]) - - expected_cached_results = {'gene_aggs': gene_counts} - expected_cached_results.update(initial_cached_results) - self.assertCachedResults(results_model, expected_cached_results) - - @urllib3_responses.activate - def test_multi_project_all_samples_all_inheritance_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__id__in=[1, 3])) - _set_cache('search_results__{}__xpos'.format(results_model.guid), json.dumps({'total_results': 5})) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}}, - 'ENSG00000228198': {'total': 6, 'families': {'F000003_3': 2, 'F000002_2': 2, 'F000011_11': 2}} - }) - - self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), - filters=[ANNOTATION_QUERY], - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts, 'total_results': 5}) - - @urllib3_responses.activate - def test_all_samples_any_affected_get_es_variant_gene_counts(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - gene_counts = get_variant_query_gene_counts(results_model, None) - - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}}, - 'ENSG00000228198': {'total': 3, 'families': {'F000003_3': 3, 'F000002_2': 2}} - }) - - self.assertExecutedSearches([dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - }} - ], - index=MITO_WGS_INDEX_NAME, - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - ), dict( - filters=[ - ANNOTATION_QUERY, - {'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}}, - {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}}, - ] - }} - ], - start_index=0, - size=1, - gene_count_aggs={ - 'samples': {'terms': {'field': 'samples', 'size': 10000}}, - 'samples_num_alt_1': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}, - 'samples_num_alt_2': {'terms': {'field': 'samples_num_alt_2', 'size': 10000}} - } - )]) - - self.assertCachedResults(results_model, {'gene_aggs': gene_counts}) - def test_get_family_affected_status(self): samples_by_id = { sample_id: Sample.objects.get(sample_id=sample_id, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) @@ -3390,407 +1418,3 @@ def test_get_family_affected_status(self): custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_multi_affected}) self.assertDictEqual(custom_affected_status, { 'I000004_hg00731': 'A', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - @urllib3_responses.activate - def test_sort(self): - setup_responses() - search_model = VariantSearch.objects.create(search={ - 'annotations': {'frameshift': ['frameshift_variant']}, - }) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(project__guid='R0001_1kg')) - - variants, _ = query_variants(results_model, sort='primate_ai', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], sort=[ - {'primate_ai_score': {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}, 'xpos', 'variantId'], - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4) - self.assertEqual(variants[0]['_sort'][0], maxsize) - self.assertEqual(variants[1]['_sort'][0], -1) - - variants, _ = query_variants(results_model, sort='gnomad', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': {'field': 'gnomad_genomes_AF'}, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - self.assertEqual(variants[0]['_sort'][0], 0.00012925741614425127) - self.assertEqual(variants[1]['_sort'][0], maxsize) - - variants, _ = query_variants(results_model, sort='in_omim', num_results=2) - self.assertExecutedSearch(filters=[ANNOTATION_QUERY], index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME]), size=4, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': ['ENSG00000240361', 'ENSG00000135953'] - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - results_model.families.set(Family.objects.filter(guid='F000001_1')) - query_variants(results_model, sort='prioritized_gene', num_results=2) - family_sample_filter = {'bool': {'_name': 'F000001_1', 'must': mock.ANY}} - self.assertExecutedSearch(filters=[ANNOTATION_QUERY, family_sample_filter], index=INDEX_NAME, size=2, sort=[ - { - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'prioritized_ranks_by_gene': {'ENSG00000268903': 1, 'ENSG00000268904': 11} - }, - 'source': mock.ANY, - } - } - }, 'xpos', 'variantId']) - - @urllib3_responses.activate - def test_genotype_inheritance_filter(self): - setup_responses() - # Testing mito indices is done in other tests, it is helpful to have a strightforward single datatype test - Sample.objects.get(elasticsearch_index=MITO_WGS_INDEX_NAME).delete() - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - search_model = VariantSearch.objects.create(search={}) - results_model = VariantSearchResults.objects.create(variant_search=search_model) - results_model.families.set(Family.objects.filter(guid='F000002_2')) - cache_key = 'search_results__{}__xpos'.format(results_model.guid) - - def _execute_inheritance_search( - mode=None, inheritance_filter=None, expected_filter=None, expected_comp_het_filter=None, - quality_filter=None, expected_quality_filter=None, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, **kwargs): - _set_cache(cache_key, None) - annotations = {'frameshift': ['frameshift_variant']} if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS \ - else {'structural': ['DEL', 'gCNV_DEL']} - search_model.search = { - 'inheritance': {'mode': mode, 'filter': inheritance_filter}, - 'annotations': annotations, - } - if quality_filter: - search_model.search['qualityFilter'] = quality_filter - search_model.save() - variants, _ = query_variants(results_model, num_results=2) - - if mode not in {'compound_het', 'recessive'}: - self.assertSetEqual( - set(variants[-1]['genotypes'].keys()), - {'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'}, - ) - - index = INDEX_NAME if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else SV_INDEX_NAME - annotation_query = {'terms': {'transcriptConsequenceTerms': next(iter(annotations.values()))}} - if expected_comp_het_filter: - self.assertExecutedSearches([ - dict(gene_aggs=True, start_index=0, size=1, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_comp_het_filter]}} - ]), - dict(start_index=0, size=2, index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': [expected_filter]}}]) - ]) - else: - filters = [expected_filter] - if expected_quality_filter: - filters.append(expected_quality_filter) - self.assertExecutedSearch(index=index, filters=[ - annotation_query, {'bool': {'_name': 'F000002_2', 'must': filters}}], **kwargs) - - # custom genotype - _execute_inheritance_search( - inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref', 'I000005_hg00732': 'ref_alt'}}, - expected_filter={ - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}} - ] - } - }) - - # de novo - _execute_inheritance_search(mode='de_novo', expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', dataset_type='SV', expected_filter={'bool': { - 'must_not': [{'term': {'samples': 'HG00732'}}, {'term': {'samples': 'HG00733'}}], - 'must': [{'term': {'samples': 'HG00731'}}], - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }) - - _execute_inheritance_search(mode='de_novo', inheritance_filter={'affected': custom_affected}, expected_filter={ - 'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - }, quality_filter={'affected_only': True, 'min_gq': 10}, expected_quality_filter={ - 'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - ]}} - ) - - _execute_inheritance_search( - mode='de_novo', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'minimum_should_match': 1, - 'must_not': [ - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_1': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}} - ], - 'must': [{ - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}} - ]} - }] - } - }) - - recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00731'}} - ] - } - } - sv_recessive_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples_cn_0': 'HG00732'}}, - {'term': {'samples_cn_gte_4': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ] - } - } - custom_affected_recessive_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_2': 'HG00732'}} - ] - } - } - - # homozygous recessive - _execute_inheritance_search(mode='homozygous_recessive', expected_filter=recessive_filter) - - _execute_inheritance_search(mode='homozygous_recessive', dataset_type='SV', expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='homozygous_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_recessive_filter) - - # compound het - com_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - ] - } - } - - sv_com_het_filter = {'term': {'samples': 'HG00731'}} - - custom_affected_comp_het_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'term': {'samples_num_alt_1': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='compound_het', expected_filter=com_het_filter, gene_aggs=True, size=1) - _execute_inheritance_search( - mode='compound_het', dataset_type='SV', gene_aggs=True, size=1, expected_filter=sv_com_het_filter) - _execute_inheritance_search( - mode='compound_het', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_comp_het_filter, gene_aggs=True, size=1) - - # x-linked recessive - x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_1': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ] - } - } - - sv_x_linked_filter = { - 'bool': { - 'minimum_should_match': 1, - 'should': [ - {'term': {'samples_cn_0': 'HG00731'}}, - {'term': {'samples_cn_2': 'HG00731'}}, - {'term': {'samples_cn_gte_4': 'HG00731'}}, - ], - 'must_not': [ - {'term': {'samples': 'HG00732'}}, - {'term': {'samples_cn_0': 'HG00733'}}, - {'term': {'samples_cn_gte_4': 'HG00733'}}, - ], - 'must': [{'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}},], - } - } - - custom_affected_x_linked_filter = { - 'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [ - {'range': {'xpos': {'gte': 23000000001, 'lte': 24000000001}}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - ] - } - } - - _execute_inheritance_search(mode='x_linked_recessive', expected_filter=x_linked_filter) - - _execute_inheritance_search(mode='x_linked_recessive', dataset_type='SV', expected_filter=sv_x_linked_filter) - - _execute_inheritance_search( - mode='x_linked_recessive', inheritance_filter={'affected': custom_affected}, - expected_filter=custom_affected_x_linked_filter) - - # recessive - _execute_inheritance_search(mode='recessive', expected_comp_het_filter=com_het_filter, expected_filter=recessive_filter) - - _execute_inheritance_search( - mode='recessive', dataset_type='SV', expected_comp_het_filter=sv_com_het_filter, expected_filter=sv_recessive_filter) - - _execute_inheritance_search( - mode='recessive', inheritance_filter={'affected': custom_affected}, - expected_comp_het_filter=custom_affected_comp_het_filter, expected_filter=custom_affected_recessive_filter) - - # any affected - _execute_inheritance_search(mode='any_affected', expected_filter={ - 'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731']}}, - {'terms': {'samples_num_alt_2': ['HG00731']}}, - {'terms': {'samples': ['HG00731']}}, - ] - } - }) - - _execute_inheritance_search( - mode='any_affected', inheritance_filter={'affected': custom_multi_affected}, expected_filter={'bool': { - 'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732']}}, - {'terms': {'samples': ['HG00731', 'HG00732']}}, - ] - } - }) From 305db90858aed4e0996039ad2518abc92f2e3645 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 16:58:15 -0500 Subject: [PATCH 022/252] fix test --- seqr/views/apis/dataset_api_tests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py index 02dccf0297..7690e12c4e 100644 --- a/seqr/views/apis/dataset_api_tests.py +++ b/seqr/views/apis/dataset_api_tests.py @@ -302,11 +302,6 @@ def _assert_expected_add_dataset_errors(self, url): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['request must contain field: "elasticsearchIndex"']}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], 'test_index - Error accessing index: Connection refused: GET /test_index/_mapping') - urllib3_responses.add_json('/{}/_mapping'.format(INDEX_NAME), {INDEX_NAME: {'mappings': {"properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) From 942155de52a5094f2e10ecf81304b02ef09c245a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Feb 2026 17:12:58 -0500 Subject: [PATCH 023/252] variant search test cleanup --- seqr/views/apis/variant_search_api_tests.py | 141 ++++---------------- 1 file changed, 28 insertions(+), 113 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index ca1ec7f853..b5961c88fd 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -15,7 +15,7 @@ from seqr.views.apis.variant_search_api import query_variants_handler, query_single_variant_handler, vlm_lookup_handler, \ export_variants_handler, search_context_handler, get_saved_search_handler, create_saved_search_handler, \ update_saved_search_handler, delete_saved_search_handler, get_variant_gene_breakdown, variant_lookup_handler -from seqr.views.utils.test_utils import AuthenticationTestCase, VARIANTS, AnvilAuthenticationTestCase,\ +from seqr.views.utils.test_utils import AuthenticationTestCase, VARIANTS,\ GENE_VARIANT_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS, FAMILY_FIELDS, \ PA_LOCUS_LIST_FIELDS, INDIVIDUAL_FIELDS, FUNCTIONAL_FIELDS, IGV_SAMPLE_FIELDS, FAMILY_NOTE_FIELDS, ANALYSIS_GROUP_FIELDS, \ VARIANT_NOTE_FIELDS, TAG_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_DETAIL_FIELDS, DYNAMIC_ANALYSIS_GROUP_FIELDS @@ -69,6 +69,7 @@ expected_pa_gene['locusListGuids'] = ['LL00049_pid_genes_autosomal_do'] expected_pa_gene['panelAppDetail'] = mock.ANY EXPECTED_SAVED_VARIANT = {k: mock.ANY for k in SAVED_VARIANT_DETAIL_FIELDS} +EXPECTED_SAVED_VARIANT.update( {'key': mock.ANY, 'mainTranscriptId': mock.ANY}) expected_detail_saved_variant = deepcopy(EXPECTED_SAVED_VARIANT) expected_detail_saved_variant['mmeSubmissions'] = [ {'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}, @@ -127,6 +128,7 @@ }, 'mmeSubmissionsByGuid': {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}, 'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}, + 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, } EXPECTED_TRANSCRIPTS_RESPONSE = { @@ -278,7 +280,8 @@ def _get_compound_het_es_variants(results_model, **kwargs): @mock.patch('seqr.views.utils.permissions_utils.safe_redis_get_json', lambda *args: None) -class VariantSearchAPITest(object): +class VariantSearchAPITest(AuthenticationTestCase): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches', 'clickhouse_saved_variants'] def _assert_expected_search_context(self, response_json): self.assertSetEqual(set(response_json), set(EXPECTED_SEARCH_CONTEXT_RESPONSE)) @@ -464,8 +467,8 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro })) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(self.EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) + self.assertSetEqual(set(response_json.keys()), set(EXPECTED_SEARCH_RESPONSE.keys())) + self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) self.assertSetEqual( set(response_json['search']['projectFamilies'][0]['familyGuids']), {'F000001_1', 'F000002_2'}) self._assert_expected_results_context(response_json) @@ -479,7 +482,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertEqual(response.status_code, 200) response_json = response.json() expected_search_response = {'projectsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['projectsByGuid']} - expected_search_response.update(self.EXPECTED_SEARCH_RESPONSE) + expected_search_response.update(EXPECTED_SEARCH_RESPONSE) self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) self.assertDictEqual(response_json, expected_search_response) self._assert_expected_results_context(response_json) @@ -490,7 +493,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertEqual(response.status_code, 200) response_json = response.json() expected_response = { - **self.EXPECTED_SEARCH_RESPONSE, + **EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT, } self.assertSetEqual(set(response_json.keys()), set(expected_response.keys())) @@ -601,10 +604,10 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro })) self.assertEqual(response.status_code, 200) response_json = response.json() - expected_search_response = deepcopy(self.EXPECTED_SEARCH_RESPONSE) + expected_search_response = deepcopy(EXPECTED_SEARCH_RESPONSE) expected_search_response.update({ 'searchedVariants': COMP_HET_VARAINTS, - 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, + 'savedVariantsByGuid': {k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, 'genesById': {'ENSG00000233653': EXPECTED_GENE}, 'variantTagsByGuid': { 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, @@ -629,7 +632,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro response = self.client.get('{}?sort=pathogenicity'.format(url)) self.assertEqual(response.status_code, 200) response_json = response.json() - expected_search_results = deepcopy(self.EXPECTED_SEARCH_RESPONSE) + expected_search_results = deepcopy(EXPECTED_SEARCH_RESPONSE) expected_search_results['searchedVariants'] = VARIANTS_WITH_DISCOVERY_TAGS expected_search_results['savedVariantsByGuid']['SV0000002_1248367227_r0390_100']['discoveryTags'] = DISCOVERY_TAGS expected_search_results['familiesByGuid'].update({'F000012_12': mock.ANY}) @@ -707,7 +710,7 @@ def _get_variants(results_model, **kwargs): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) + self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) self._assert_expected_results_context(response_json) self.assertSetEqual( set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) @@ -722,7 +725,7 @@ def _get_variants(results_model, **kwargs): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) response_json = response.json() - expected_no_access_resonse = deepcopy(self.EXPECTED_SEARCH_RESPONSE) + expected_no_access_resonse = deepcopy(EXPECTED_SEARCH_RESPONSE) expected_no_access_resonse['search']['search']['no_access_project_genome_version'] = '37' self.assertDictEqual(response_json, expected_no_access_resonse) self._assert_expected_results_context(response_json) @@ -741,8 +744,8 @@ def _get_variants(results_model, **kwargs): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(self.EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, self.EXPECTED_SEARCH_RESPONSE) + self.assertSetEqual(set(response_json.keys()), set(EXPECTED_SEARCH_RESPONSE.keys())) + self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) self._assert_expected_results_context(response_json) self.assertSetEqual( set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) @@ -753,19 +756,19 @@ def _get_variants(results_model, **kwargs): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) response_json = response.json() - expected_variants = self.EXPECTED_SEARCH_RESPONSE['searchedVariants'][:1] + self.EXPECTED_SEARCH_RESPONSE['searchedVariants'][2:] + expected_variants = EXPECTED_SEARCH_RESPONSE['searchedVariants'][:1] + EXPECTED_SEARCH_RESPONSE['searchedVariants'][2:] expected_response = { - **self.EXPECTED_SEARCH_RESPONSE, + **EXPECTED_SEARCH_RESPONSE, 'searchedVariants': expected_variants, - 'search': {**self.EXPECTED_SEARCH_RESPONSE['search'], 'totalResults': len(expected_variants)}, + 'search': {**EXPECTED_SEARCH_RESPONSE['search'], 'totalResults': len(expected_variants)}, 'mmeSubmissionsByGuid': {}, 'variantFunctionalDataByGuid': {}, 'savedVariantsByGuid': { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() + k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k != 'SV0000001_2103343353_r0390_100' }, 'variantTagsByGuid': { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'].items() + k: v for k, v in EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'].items() if k not in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'] }, } @@ -905,7 +908,7 @@ def test_query_single_variant(self, mock_get_variant): def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response): omit_fields = {'search', *(omit_fields or [])} - expected_search_response = deepcopy({**self.EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) + expected_search_response = deepcopy({**EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) expected_search_response.update(expected_response) expected_search_response.update({ k: EXPECTED_SEARCH_CONTEXT_RESPONSE[k] for k in ['projectsByGuid', 'familiesByGuid', 'locusListsByGuid'] @@ -1001,8 +1004,8 @@ def test_variant_lookup(self, mock_variant_lookup): 'genesById': {'ENSG00000227232': EXPECTED_GENE, 'ENSG00000268903': EXPECTED_GENE}, } expected_body.update(expected_gene_body) - if 'transcriptsById' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['transcriptsById'] = self.EXPECTED_SEARCH_RESPONSE['transcriptsById'] + if 'transcriptsById' in EXPECTED_SEARCH_RESPONSE: + expected_body['transcriptsById'] = EXPECTED_SEARCH_RESPONSE['transcriptsById'] response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -1051,7 +1054,7 @@ def test_variant_lookup(self, mock_variant_lookup): k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} }) expected_body['savedVariantsByGuid']= { - k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] + k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] } expected_body.update(expected_gene_body) expected_body['genesById']['ENSG00000227232'] = expected_pa_gene @@ -1073,7 +1076,7 @@ def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False if not include_context: exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) expected_body = { - **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, + **{k: {} for k in EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, 'projectsByGuid': { p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids or [] @@ -1091,8 +1094,8 @@ def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False **EXPECTED_TRANSCRIPTS_RESPONSE, 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], }) - if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + if 'totalSampleCounts' in EXPECTED_SEARCH_RESPONSE: + expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] return expected_body @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') @@ -1386,91 +1389,3 @@ def test_search_results_redirect(self): response = self.client.get('/report/custom_search/6ebb895dfca0f63c34be1ca59d950205?page=2&sort=cadd') self.assertEqual(response.status_code, 301) self.assertEqual(response.url, '/variant_search/results/6ebb895dfca0f63c34be1ca59d950205?page=2&sort=cadd') - - -# Tests for AnVIL access disabled -class LocalVariantSearchAPITest(AuthenticationTestCase, VariantSearchAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'variant_searches'] - - EXPECTED_SEARCH_RESPONSE = { - **EXPECTED_SEARCH_RESPONSE, - **EXPECTED_TRANSCRIPTS_RESPONSE, - } - - -def assert_no_list_ws_has_al(self, acl_call_count, group_call_count, workspace_name=None): - self.mock_list_workspaces.assert_not_called() - assert_ws_has_al(self, acl_call_count, group_call_count, workspace_name) - - -def assert_has_list_ws(self, has_data_manager=False): - calls = [ - mock.call(self.no_access_user), - mock.call(self.collaborator_user), - ] - if has_data_manager: - calls.insert(1, mock.call(self.data_manager_user)) - self.mock_list_workspaces.assert_has_calls(calls) - - -def assert_no_al_has_list_ws(self, group_count=1, has_data_manager=False): - assert_has_list_ws(self, has_data_manager) - self.mock_get_ws_access_level.assert_not_called() - assert_workspace_calls(self, group_count) - - -def assert_ws_has_al(self, acl_call_count, group_call_count, workspace_name=None, user=None): - if not workspace_name: - workspace_name = 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de' - self.mock_get_ws_access_level.assert_called_with(self.collaborator_user, 'my-seqr-billing', workspace_name) - self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) - assert_workspace_calls(self, group_call_count, user) - - -def assert_workspace_calls(self, group_call_count, user=None): - self.assertEqual(self.mock_get_groups.call_count, group_call_count) - self.mock_get_groups.assert_called_with(user or self.collaborator_user) - - self.mock_get_ws_acl.assert_not_called() - self.mock_get_group_members.assert_not_called() - - -# Test for permissions from AnVIL only -class AnvilVariantSearchAPITest(AnvilAuthenticationTestCase, VariantSearchAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'variant_searches', 'clickhouse_saved_variants'] - - EXPECTED_SEARCH_RESPONSE = { - **EXPECTED_SEARCH_RESPONSE, - 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, - } - EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'] = { - k: {**v, 'key': mock.ANY, 'mainTranscriptId': mock.ANY} for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() - } - - def test_query_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_variants(*args) - assert_ws_has_al(self, 1, 9, user=self.analyst_user) - assert_has_list_ws(self) - - def test_query_all_projects_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_all_projects_variants(*args) - assert_no_al_has_list_ws(self, group_count=4, has_data_manager=True) - - def test_query_all_project_families_variants(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_all_project_families_variants(*args) - assert_no_al_has_list_ws(self) - - def test_search_context(self): - super(AnvilVariantSearchAPITest, self).test_search_context() - assert_no_al_has_list_ws(self, 12) - - def test_query_single_variant(self, *args): - super(AnvilVariantSearchAPITest, self).test_query_single_variant(*args) - assert_no_list_ws_has_al(self, 4, 1) - - def test_saved_search(self): - super(AnvilVariantSearchAPITest, self).test_saved_search() - assert_workspace_calls(self, 3, user=self.no_access_user) - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_not_called() - From 0a447e632047f8b5809222610fda65fe9405ff1d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Feb 2026 10:51:35 -0500 Subject: [PATCH 024/252] unused code clean up --- .../elasticsearch/es_gene_agg_search.py | 106 -- seqr/utils/search/elasticsearch/es_search.py | 1083 ----------------- seqr/utils/search/elasticsearch/es_utils.py | 80 +- .../search/elasticsearch/es_utils_tests.py | 16 - 4 files changed, 1 insertion(+), 1284 deletions(-) delete mode 100644 seqr/utils/search/elasticsearch/es_gene_agg_search.py diff --git a/seqr/utils/search/elasticsearch/es_gene_agg_search.py b/seqr/utils/search/elasticsearch/es_gene_agg_search.py deleted file mode 100644 index d2b04fc96a..0000000000 --- a/seqr/utils/search/elasticsearch/es_gene_agg_search.py +++ /dev/null @@ -1,106 +0,0 @@ -from collections import defaultdict - -from seqr.utils.search.elasticsearch.constants import MAX_COMPOUND_HET_GENES, HAS_ALT_FIELD_KEYS -from seqr.utils.search.elasticsearch.es_search import EsSearch - - -class EsGeneAggSearch(EsSearch): - AGGREGATION_NAME = 'gene aggregation' - CACHED_COUNTS_KEY = None - - def search(self, *args, **kwargs): - self._aggregate_by_gene() - return super(EsGeneAggSearch, self).search(*args, **kwargs) - - def _aggregate_by_gene(self): - searches = [self._search] - for index_searches in self._index_searches.values(): - searches += [index_search for index_search in index_searches] - - for search in searches: - agg = search.aggs.bucket( - 'genes', 'terms', field='mainTranscript_gene_id', size=MAX_COMPOUND_HET_GENES+1 - ) - if self._no_sample_filters or self._any_affected_sample_filters: - for key in HAS_ALT_FIELD_KEYS: - agg.bucket(key, 'terms', field=key, size=10000) - else: - agg.metric( - 'vars_by_gene', 'top_hits', size=100, _source='none' - ) - - def _should_execute_single_search(self, page=1, num_results=100): - return self._is_single_search(), {} - - def _process_single_search_response(self, parsed_response, page=1, num_results=100, deduplicate=False, **kwargs): - gene_aggs = {gene_id: {k: counts[k] for k in ['total', 'families']} for gene_id, counts in parsed_response.items()} - self._add_compound_hets(gene_aggs) - - self.previous_search_results['gene_aggs'] = gene_aggs - - return gene_aggs - - def _process_multi_search_responses(self, parsed_responses, page=1, num_results=100): - gene_aggs = parsed_responses[0] if parsed_responses else {} - for response in parsed_responses[1:]: - for gene_id, count_details in response.items(): - gene_aggs[gene_id]['sample_ids'].update(count_details['sample_ids']) - gene_aggs[gene_id]['families'].update(count_details['families']) - - gene_aggs = { - gene_id: {'total': len(counts['sample_ids']), 'families': counts['families']} - for gene_id, counts in gene_aggs.items() - } - - self._add_compound_hets(gene_aggs) - self.previous_search_results['gene_aggs'] = gene_aggs - - return gene_aggs - - def _parse_response(self, response): - if len(response.aggregations.genes.buckets) > MAX_COMPOUND_HET_GENES: - from seqr.utils.search.utils import InvalidSearchException - raise InvalidSearchException('This search returned too many genes') - - gene_counts = defaultdict(lambda: {'total': 0, 'families': defaultdict(int), 'sample_ids': set()}) - for gene_agg in response.aggregations.genes.buckets: - gene_id = gene_agg['key'] - gene_counts[gene_id]['total'] += gene_agg['doc_count'] - if 'vars_by_gene' in gene_agg: - for hit in gene_agg['vars_by_gene']: - gene_counts[gene_id]['sample_ids'].add(hit.meta.id) - for family_guid in hit.meta.matched_queries: - gene_counts[gene_id]['families'][family_guid] += 1 - else: - families_by_sample = {} - for index_samples_by_family in self.samples_by_family_index.values(): - for family_guid, samples_by_id in index_samples_by_family.items(): - for sample_id in samples_by_id.keys(): - families_by_sample[sample_id] = family_guid - - for key in HAS_ALT_FIELD_KEYS: - for sample_agg in gene_agg[key]['buckets']: - family_guid = families_by_sample.get(sample_agg['key']) - if family_guid: - gene_counts[gene_id]['families'][family_guid] += sample_agg['doc_count'] - gene_counts[gene_id]['sample_ids'].add(sample_agg['key']) - else: - # samples may be returned that are not part of the searched families if they have no - # affected individuals and were removed from the "any affected" search. - gene_counts[gene_id]['total'] -= sample_agg['doc_count'] - - return gene_counts - - def _add_compound_hets(self, gene_counts): - # Compound hets are always loaded as part of the initial search and are not part of the fetched aggregation - loaded_compound_hets = self.previous_search_results.get('grouped_results', []) + \ - self.previous_search_results.get('compound_het_results', []) - for group in loaded_compound_hets: - variants = next(iter(group.values())) - gene_id = next(iter(group)) - if gene_id and gene_id != 'null': - if gene_id not in gene_counts: - gene_counts[gene_id] = {'total': 0, 'families': defaultdict(int)} - gene_counts[gene_id]['total'] += len(variants) - for family_guid in variants[0]['familyGuids']: - gene_counts[gene_id]['families'][family_guid] += len(variants) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index 47adfd192a..3f3d637bfd 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -130,38 +130,6 @@ def _set_index_metadata(self): from seqr.utils.search.elasticsearch.es_utils import get_index_metadata self.index_metadata = get_index_metadata(self.index_name, self._client, include_fields=True) - def _update_alias_metadata(self): - additional_meta_indices = set(self.index_metadata.keys()) - set(self._indices) - aliases = [ind for ind in self._indices if ind not in self.index_metadata] - alias_map = defaultdict(list) - if len(aliases) == 1: - alias_map[aliases[0]] = additional_meta_indices - else: - for index, index_aliases in self._client.indices.get_alias(index=','.join(aliases)).items(): - for alias in index_aliases['aliases']: - alias_map[alias].append(index) - self._indices = list(self.index_metadata.keys()) - for alias, alias_indices in alias_map.items(): - alias_samples = self.samples_by_family_index.pop(alias, {}) - for alias_index in alias_indices: - if not self.samples_by_family_index[alias_index]: - self.samples_by_family_index[alias_index] = {} - self.samples_by_family_index[alias_index].update(alias_samples) - - def _set_family_affected_status(self, inheritance_filter): - for family_samples in list(self.samples_by_family_index.values()): - for family_guid, samples_by_id in family_samples.items(): - individual_affected_status = _get_family_affected_status(samples_by_id, inheritance_filter) - if family_guid not in self._family_individual_affected_status: - self._family_individual_affected_status[family_guid] = {} - self._family_individual_affected_status[family_guid].update(individual_affected_status) - - def _dataset_type_indices(self, dataset_type): - new_indices = self.indices_by_dataset_type[dataset_type] - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - new_indices += self.indices_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS] - return new_indices - def _sort_variants(self, sample_data): main_sort_dict = self._sort[0] if len(self._sort) and isinstance(self._sort[0], dict) else None @@ -201,369 +169,10 @@ def _filter(self, new_filter): self._search = self._search.filter(new_filter) return self - def filter_variants(self, inheritance_mode=None, inheritance_filter=None, genes=None, intervals=None, rs_ids=None, variant_ids=None, exclude_locations=False, - frequencies=None, pathogenicity=None, in_silico=None, annotations=None, annotations_secondary=None, - quality_filter=None, custom_query=None, skip_genotype_filter=False, dataset_type=None, secondary_dataset_type=None, **kwargs): - - self._filter_custom(custom_query) - - self._filter_by_location(genes, intervals, variant_ids, rs_ids, exclude_locations) - - self._parse_annotation_overrides(annotations, pathogenicity) - - self._filter_by_frequency(frequencies) - - self._filter_by_in_silico(in_silico) - - if quality_filter and quality_filter.get('vcf_filter') is not None: - self._filter(~Q('exists', field='filters')) - - inheritance_filter = inheritance_filter or {} - - quality_affected_only = quality_filter and quality_filter.get('affected_only') - if inheritance_mode or inheritance_filter or quality_affected_only: - self._set_family_affected_status(inheritance_filter) - - quality_filters_by_family = self._get_quality_filters_by_family(quality_filter, quality_affected_only) - - comp_het_dataset_type = dataset_type - if dataset_type != secondary_dataset_type: - comp_het_dataset_type = None - if dataset_type == ALL_DATA_TYPES: - dataset_type = None - comp_het_dataset_type = None - - has_comp_het_search = inheritance_mode in {RECESSIVE, COMPOUND_HET} and not self.previous_search_results.get('grouped_results') - if has_comp_het_search: - self._filter_compound_hets(quality_filters_by_family, annotations_secondary, comp_het_dataset_type) - if inheritance_mode == COMPOUND_HET: - return - - self._filter_by_annotations() - if skip_genotype_filter and not inheritance_mode: - return - - self._filter_by_genotype(inheritance_mode, inheritance_filter, quality_filters_by_family, dataset_type) - - def _parse_annotation_overrides(self, annotations, pathogenicity): - clinvar_terms, hgmd_classes = _parse_pathogenicity_filter(pathogenicity or {}) - - annotations = {k: v for k, v in (annotations or {}).items() if v} - - # Temporary code, remove it after all the SV indices are reloaded - if annotations.get('structural_consequence'): - if 'MSV_EXON_OVERLAP' in annotations['structural_consequence']: - annotations['structural_consequence'].append('MSV_EXON_OVR') - if 'INTRAGENIC_EXON_DUP' in annotations['structural_consequence']: - annotations['structural_consequence'].append('DUP_LOF') - - new_svs = bool(annotations.pop(NEW_SV_FIELD, False)) - splice_ai = annotations.pop(SPLICE_AI_FIELD, None) - screen = annotations.pop(SCREEN_KEY, None) - self._allowed_consequences = sorted({ann for anns in annotations.values() for ann in anns}) - if clinvar_terms: - self._consequence_overrides[CLINVAR_KEY] = clinvar_terms - if hgmd_classes: - self._consequence_overrides[HGMD_KEY] = hgmd_classes - if splice_ai: - self._consequence_overrides[SPLICE_AI_FIELD] = float(splice_ai) - if screen: - self._consequence_overrides[SCREEN_KEY] = screen - if new_svs: - self._consequence_overrides[NEW_SV_FIELD] = new_svs - - def _filter_by_location(self, genes, intervals, variant_ids, rs_ids, exclude_locations): - if genes or intervals: - self._filter(_location_filter(genes, intervals, exclude_locations)) - if genes and not exclude_locations: - self._filtered_gene_ids = set(genes.keys()) - elif variant_ids: - self.filter_by_variant_ids(variant_ids) - elif rs_ids: - self._filter(Q('terms', rsid=rs_ids)) - - def _filter_custom(self, custom_query): - if custom_query: - if not isinstance(custom_query, list): - custom_query = [custom_query] - for q_dict in custom_query: - self._filter(Q(q_dict)) - - def _filter_by_in_silico(self, in_silico_filters): - in_silico_filters = deepcopy(in_silico_filters or {}) - require_score = in_silico_filters.pop('requireScore', False) - in_silico_filters = {k: v for k, v in in_silico_filters.items() if v is not None and len(v) != 0} - if in_silico_filters: - self._filter(self._in_silico_filter(in_silico_filters, require_score=require_score)) - - def _in_silico_filter(self, in_silico_filters, require_score): - prediction_key_lookup = { - in_silico_filter: PREDICTION_FIELD_LOOKUP.get(in_silico_filter.lower(), in_silico_filter) - for in_silico_filter in in_silico_filters.keys() - } - - multi_predictor_fields = { - in_silico_filter: MULTI_FIELD_PREDICTORS[in_silico_filter.lower()] - for in_silico_filter in in_silico_filters.keys() if in_silico_filter.lower() in MULTI_FIELD_PREDICTORS - } - if multi_predictor_fields: - all_fields = set() - for metadata in self.index_metadata.values(): - all_fields.update(metadata['fields']) - index_fields = { - k: next(field for field in fields if field in all_fields) - for k, fields in multi_predictor_fields.items() - } - prediction_key_lookup.update(index_fields) - - return _in_silico_filter({prediction_key_lookup[k]: v for k, v in in_silico_filters.items()}, require_score) - - def _filter_by_frequency(self, frequencies): - frequencies = {pop: v for pop, v in (frequencies or {}).items() if pop in POPULATIONS} - if not frequencies: - return - - path_filter = self._get_clinvar_pathogenic_override_filter() - path_override = path_filter is not None and any( - freqs.get('af') or 1 < PATH_FREQ_OVERRIDE_CUTOFF for freqs in frequencies.values()) - - q = Q() - path_q = Q() - for pop, freqs in sorted(frequencies.items()): - if freqs.get('af') is not None: - filter_field = next( - (field_key for field_key in POPULATIONS[pop]['filter_AF'] - if any(field_key in index_metadata['fields'] for index_metadata in self.index_metadata.values())), - POPULATIONS[pop]['AF']) - q &= _pop_freq_filter(filter_field, freqs['af']) - if path_override: - path_q &= _pop_freq_filter(filter_field, max(freqs['af'], PATH_FREQ_OVERRIDE_CUTOFF)) - elif freqs.get('ac') is not None: - q &= _pop_freq_filter(POPULATIONS[pop]['AC'], freqs['ac']) - - if freqs.get('hh') is not None: - q &= _pop_freq_filter(POPULATIONS[pop]['Hom'], freqs['hh']) - q &= _pop_freq_filter(POPULATIONS[pop]['Hemi'], freqs['hh']) - - if path_override: - q |= (path_filter & path_q) - - self._filter(q) - - def _get_clinvar_pathogenic_override_filter(self): - clinvar_path_terms = [ - f for f in self._consequence_overrides.get(CLINVAR_KEY, []) if f in CLINVAR_PATH_SIGNIFICANCES - ] - if clinvar_path_terms: - return _pathogenicity_filter(clinvar_path_terms) - return None - - def _get_annotation_override_filter(self): - filters = [] - pathogenicity_filter = _pathogenicity_filter( - self._consequence_overrides.get(CLINVAR_KEY), self._consequence_overrides.get(HGMD_KEY), - ) - if pathogenicity_filter: - filters.append(pathogenicity_filter) - splice_ai = self._consequence_overrides.get(SPLICE_AI_FIELD) - if splice_ai: - filters.append(self._in_silico_filter({SPLICE_AI_FIELD: splice_ai}, require_score=True)) - screen = self._consequence_overrides.get(SCREEN_KEY) - if screen: - filters.append(Q('terms', screen_region_type=screen)) - - if not filters: - return None - return _or_filters(filters) - - def _filter_by_annotations(self): - annotation_override_filter = self._get_annotation_override_filter() - new_svs = self._consequence_overrides.get(NEW_SV_FIELD) - - if self._allowed_consequences: - consequences_filter = _annotations_filter(self._allowed_consequences) - - if annotation_override_filter: - # Pathogencicity and transcript consequences act as "OR" filters instead of the usual "AND" - consequences_filter |= annotation_override_filter - self._filter(consequences_filter) - elif annotation_override_filter and not new_svs: - self._filter(annotation_override_filter) - def filter_by_variant_ids(self, variant_ids): self._filter(Q('terms', variantId=variant_ids)) return self - def _filter_by_genotype(self, inheritance_mode, inheritance_filter, quality_filters_by_family, dataset_type): - has_inheritance_filter = inheritance_filter or inheritance_mode - all_sample_search = (not quality_filters_by_family) and (inheritance_mode == ANY_AFFECTED or not has_inheritance_filter) - no_filter_indices = set() - - indices = self._dataset_type_indices(dataset_type) if dataset_type else self._indices - for index in indices: - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - - if all_sample_search: - index_skipped_sample_count = sum([s.elasticsearch_index == index for s in self._skipped_samples or []]) - search_sample_count = sum(len(samples) for samples in family_samples_by_id.values()) + index_skipped_sample_count - index_sample_count = Sample.objects.filter(elasticsearch_index=index, is_active=True).count() - if search_sample_count == index_sample_count: - if inheritance_mode == ANY_AFFECTED: - sample_ids = [] - for family_guid, samples_by_id in family_samples_by_id.items(): - sample_ids += [ - sample_id for sample_id, sample in samples_by_id.items() - if self._family_individual_affected_status[family_guid][sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED] - self._index_searches[index].append(self._search.filter(_any_affected_sample_filter(sample_ids))) - self._any_affected_sample_filters = True - else: - # If searching across all families in an index with no inheritance mode we do not need to explicitly - # filter on inheritance, as all variants have some inheritance for at least one family - self._no_sample_filters = True - no_filter_indices.add(index) - continue - - genotypes_qs = [ - self._get_family_sample_query( - family_guid, family_samples_by_id, quality_filters_by_family, - index_fields, inheritance_mode, inheritance_filter - ) for family_guid in sorted(family_samples_by_id.keys()) - ] - if len(genotypes_qs) > MAX_SEARCH_CLAUSES: - self._index_searches[index].append(self._search.filter(_or_filters(genotypes_qs[:MAX_SEARCH_CLAUSES]))) - genotypes_qs = genotypes_qs[MAX_SEARCH_CLAUSES:] - - self._index_searches[index].append(self._search.filter(_or_filters(genotypes_qs))) - - if no_filter_indices and self._index_searches: - for index in no_filter_indices: - self._index_searches[index].append(self._search) - - def _get_family_sample_query(self, family_guid, family_samples_by_id, quality_filters_by_family, index_fields, inheritance_mode, inheritance_filter): - samples_by_id = family_samples_by_id[family_guid] - affected_status = self._family_individual_affected_status.get(family_guid) - - # Filter samples by inheritance - if inheritance_mode == ANY_AFFECTED: - # Only return variants where at least one of the affected samples has an alt allele - sample_ids = [sample_id for sample_id, sample in samples_by_id.items() - if affected_status[sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED] - family_samples_q = _any_affected_sample_filter(sample_ids) - elif inheritance_filter or inheritance_mode: - if inheritance_mode: - inheritance_filter.update(INHERITANCE_FILTERS[inheritance_mode]) - - family_samples_q = _family_genotype_inheritance_filter( - inheritance_mode, inheritance_filter, samples_by_id, affected_status, index_fields, - ) - else: - # If no inheritance specified only return variants where at least one of the requested samples has an alt allele - family_samples_q = _any_affected_sample_filter(list(samples_by_id.keys())) - - return _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - - def _filter_compound_hets(self, quality_filters_by_family, annotations_secondary, dataset_type): - comp_het_consequences = set(self._allowed_consequences) - annotation_override_filter = self._get_annotation_override_filter() - if annotations_secondary: - self._allowed_consequences_secondary = sorted({ann for anns in annotations_secondary.values() for ann in anns}) - comp_het_consequences.update(self._allowed_consequences_secondary) - - annotation_filter = _annotations_filter(comp_het_consequences) - if annotation_override_filter: - annotation_filter |= annotation_override_filter - comp_het_search = self._search.filter(annotation_filter) - - comp_het_qs_by_index = defaultdict(list) - if dataset_type or len(self._indices) <= 1: - indices = self.indices_by_dataset_type[dataset_type] if dataset_type else self._indices - for index in sorted(indices, reverse=True): - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - - for family_guid, samples_by_id in sorted(family_samples_by_id.items()): - affected_status = self._family_individual_affected_status[family_guid] - family_samples_q = _family_genotype_inheritance_filter( - COMPOUND_HET, INHERITANCE_FILTERS[COMPOUND_HET], samples_by_id, affected_status, index_fields, - ) - samples_q = _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - comp_het_qs_by_index[index].append(samples_q) - else: - self._get_paired_indices_comp_het_queries(comp_het_qs_by_index, quality_filters_by_family) - - for index, compound_het_qs in comp_het_qs_by_index.items(): - comp_het_qs_list = [ - compound_het_qs[:MAX_SEARCH_CLAUSES], compound_het_qs[MAX_SEARCH_CLAUSES:] - ] if len(compound_het_qs) > MAX_SEARCH_CLAUSES else [compound_het_qs] - for compound_het_q in comp_het_qs_list: - compound_het_search = comp_het_search.filter(_or_filters(compound_het_q)) - compound_het_search.aggs.bucket( - 'genes', 'terms', field='geneIds', min_doc_count=2, size=MAX_COMPOUND_HET_GENES + 1 - ).metric( - 'vars_by_gene', 'top_hits', size=100, sort=self._sort, _source=QUERY_FIELD_NAMES - ) - self._index_searches[index].append(compound_het_search) - - def _get_paired_indices_comp_het_queries(self, comp_het_qs_by_index, quality_filters_by_family): - paired_index_families = defaultdict(dict) - sv_indices = self.indices_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS] - variant_indices = self.indices_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] - if sv_indices and variant_indices: - for sv_index in sv_indices: - sv_families = set(self.samples_by_family_index[sv_index].keys()) - for var_index in variant_indices: - overlapping_families = sv_families & set(self.samples_by_family_index[var_index].keys()) - if overlapping_families: - paired_index_families[sv_index].update({var_index: overlapping_families}) - paired_index_families[var_index].update({sv_index: overlapping_families}) - - seen_paired_indices = set() - for index in sorted(self._indices, reverse=True): - family_samples_by_id = self.samples_by_family_index[index] - index_fields = self.index_metadata[index]['fields'] - seen_paired_indices.add(index) - - paired_families = {} - for pair_index, families in paired_index_families[index].items(): - paired_families.update({family: pair_index for family in families}) - - for family_guid in sorted(family_samples_by_id.keys()): - paired_index = paired_families.get(family_guid) - if paired_index and paired_index in seen_paired_indices: - continue - - samples_by_id = family_samples_by_id[family_guid] - - affected_status = self._family_individual_affected_status[family_guid] - inheritance_filters = self._comp_het_inheritance_filter(index, paired_index) - family_samples_q = _family_genotype_inheritance_filter( - COMPOUND_HET, inheritance_filters, samples_by_id, affected_status, index_fields, - ) - - family_index = index - if paired_index: - self._paired_index_comp_het = True - pair_index_fields = self.index_metadata[paired_index]['fields'] - pair_samples_by_id = self.samples_by_family_index[paired_index][family_guid] - pair_inheritance_filters = self._comp_het_inheritance_filter(paired_index, True) - family_samples_q |= _family_genotype_inheritance_filter( - COMPOUND_HET, pair_inheritance_filters, pair_samples_by_id, affected_status, pair_index_fields, - ) - family_index = ','.join(sorted([index, paired_index])) - - samples_q = _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family) - comp_het_qs_by_index[family_index].append(samples_q) - - def _comp_het_inheritance_filter(self, index, has_paired_index): - if has_paired_index and self._get_index_dataset_type(index) == Sample.DATASET_TYPE_VARIANT_CALLS: - # SNPs in trans with deletions may be called as hom alt instead of ref alt - return { - AFFECTED: HAS_ALT, - UNAFFECTED: INHERITANCE_FILTERS[COMPOUND_HET][UNAFFECTED], - } - return INHERITANCE_FILTERS[COMPOUND_HET] - def search(self, page=1, num_results=100): indices = self._indices @@ -639,118 +248,6 @@ def _process_single_search_response(self, parsed_response, page=1, num_results=1 return variant_results[:num_results] - def _execute_multi_search(self, **kwargs): - indices = sorted(self._index_searches.keys(), reverse = True) or self._indices - - if self.CACHED_COUNTS_KEY and not self.previous_search_results.get(self.CACHED_COUNTS_KEY): - self.previous_search_results[self.CACHED_COUNTS_KEY] = {} - - paginated_index_searches = {} - index_logs = {} - for index_name in indices: - start_index = 0 - if self.CACHED_COUNTS_KEY: - if self.previous_search_results[self.CACHED_COUNTS_KEY].get(index_name): - index_total = self.previous_search_results[self.CACHED_COUNTS_KEY][index_name]['total'] - start_index = self.previous_search_results[self.CACHED_COUNTS_KEY][index_name]['loaded'] - if start_index >= index_total: - continue - else: - self.previous_search_results[self.CACHED_COUNTS_KEY][index_name] = {'loaded': 0, 'total': 0} - - searches, log_messages = self._get_paginated_searches(index_name, start_index=start_index, **kwargs) - if searches: - paginated_index_searches[index_name] = searches - index_logs[index_name] = log_messages - - if len(paginated_index_searches) > MAX_INDEX_SEARCHES: - index_possible_variants, all_inheritance_response = self._get_possible_hit_indices() - if index_possible_variants: - paginated_index_searches = { - index_name: [ - search.query('ids', values=possible_ids) for search in paginated_index_searches[index_name] - ] for index_name, possible_ids in index_possible_variants.items() - if paginated_index_searches.get(index_name) - } - elif all_inheritance_response is not None: - # all inheritance search succeeded but has no results, return an empty response - return self._process_single_search_response(self._parse_response(all_inheritance_response), **kwargs) - - ms = MultiSearch() - for index_name, searches in paginated_index_searches.items(): - for search in searches: - ms = ms.add(search) - for message in index_logs[index_name]: - logger.info(message, self._user) - - responses = self._execute_search(ms) if ms._searches else [] - parsed_responses = [self._parse_response(response) for response in responses] - return self._process_multi_search_responses(parsed_responses, **kwargs) - - def _get_possible_hit_indices(self): - no_inheritance_search = self._search.index(self.index_name).source('')[:PREFILTER_SEARCH_SIZE] - response = no_inheritance_search.using(self._client).execute() - if response.hits.total['value'] > len(response.hits): - return None, None - - index_possible_variants = defaultdict(list) - for hit in response.hits: - index_possible_variants[hit.meta.index].append(hit.meta.id) - - logger.info(f'Filtering search to {len(index_possible_variants)} indices with possible hits', self._user) - return index_possible_variants, response - - def _process_multi_search_responses(self, parsed_responses, page=1, num_results=100): - new_results = [] - compound_het_results = self.previous_search_results.get('compound_het_results', []) - loaded_counts = defaultdict(lambda: defaultdict(int)) - for response_hits, response_total, is_compound_het, index_name in parsed_responses: - if not response_hits: - continue - - if is_compound_het: - compound_het_results += response_hits - loaded_count = response_total - index_name = '{}_compound_het'.format(index_name) - if not self.previous_search_results['loaded_variant_counts'].get(index_name): - self.previous_search_results['loaded_variant_counts'][index_name] = {'total': 0, 'loaded': 0} - else: - new_results += response_hits - loaded_count = len(response_hits) - - loaded_counts[index_name]['total'] += response_total - loaded_counts[index_name]['loaded'] += loaded_count - - for index_name, counts in loaded_counts.items(): - self.previous_search_results['loaded_variant_counts'][index_name]['total'] = counts['total'] - self.previous_search_results['loaded_variant_counts'][index_name]['loaded'] += counts['loaded'] - - total_results = sum( - counts['total'] for counts in self.previous_search_results['loaded_variant_counts'].values()) - self.previous_search_results['total_results'] = total_results - - # combine new results with unsorted previously loaded results to correctly sort/paginate - all_loaded_results = self.previous_search_results.get('all_results', []) - new_results += self.previous_search_results.get('variant_results', []) - - new_results = sorted(new_results, key=lambda variant: variant['_sort']) - variant_results = self._deduplicate_results(new_results) - - if compound_het_results or self.previous_search_results.get('grouped_results'): - if compound_het_results: - compound_het_results = self._deduplicate_compound_het_results(compound_het_results) - compound_het_results = _sort_compound_hets(compound_het_results) - loaded_results = sum( - counts['loaded'] for counts in self.previous_search_results['loaded_variant_counts'].values()) - return self._process_compound_hets( - compound_het_results, variant_results, num_results, all_loaded=loaded_results == total_results) - else: - end_index = num_results * page - num_loaded = num_results * page - len(all_loaded_results) - self.previous_search_results['all_results'] = all_loaded_results + variant_results[:num_loaded] - self.previous_search_results['variant_results'] = variant_results[num_loaded:] - return self.previous_search_results['all_results'][end_index-num_results:end_index] - def _parse_response(self, response): index_name = response.hits[0].meta.index if response.hits else None if hasattr(response.aggregations, 'genes') and response.hits: @@ -761,7 +258,6 @@ def _parse_response(self, response): logger.info('Total hits: {} ({} seconds)'.format(response_total, response.took / 1000.0), self._user) return [self._parse_hit(hit) for hit in response], response_total, False, index_name - def _parse_hit(self, raw_hit): hit = {k: raw_hit[k] for k in QUERY_FIELD_NAMES if k in raw_hit} index_name = raw_hit.meta.index @@ -881,28 +377,6 @@ def _is_matched_sample(family_guid, sample): return family_guids, genotypes - @classmethod - def _set_sv_genotype_coords(cls, genotypes, result): - # If an SV has genotype-specific coordinates that differ from the main coordinates, use those - if any(not gen.get('isRef') for gen in genotypes.values()) and all( - (gen.get('isRef') or gen.get('start') or gen.get('end')) for gen in genotypes.values()): - for field, conf in SV_SAMPLE_OVERRIDE_FIELD_CONFIGS.items(): - gen_field = conf.get('genotype_field', field) - val = conf['select_val']([ - gen.get(gen_field) or result.get(field) for gen in genotypes.values() if not gen.get('isRef') - ]) - if val != result.get(field): - result[field] = val - if field == 'pos': - result['xpos'] = get_xpos(result['chrom'], val) - - for gen in genotypes.values(): - for field, conf in SV_SAMPLE_OVERRIDE_FIELD_CONFIGS.items(): - gen_field = conf.get('genotype_field', field) - compare_func = conf.get('equal') or (lambda a, b: a == b) - if compare_func(gen.get(gen_field), result.get(field)): - gen[gen_field] = None - def _get_main_transcript(self, sorted_transcripts): main_transcript_id = sorted_transcripts[0]['transcriptId'] \ if len(sorted_transcripts) and 'transcriptRank' in sorted_transcripts[0] else None @@ -928,219 +402,6 @@ def _get_main_transcript(self, sorted_transcripts): return main_transcript_id, selected_main_transcript_id - @staticmethod - def _is_matched_transcript(t, allowed_consequences): - is_match = t.get('majorConsequence') in allowed_consequences - if CANONICAL_TRANSCRIPT_FILTER in allowed_consequences and t.get('canonical') and \ - t.get('majorConsequence') == 'non_coding_transcript_exon_variant': - is_match = True - return is_match - - @staticmethod - def _add_liftover(result, hit): - lifted_over_genome_version = None - lifted_over_chrom = None - lifted_over_pos = None - grch37_locus = hit.get(GRCH38_LOCUS_FIELD, None) - if grch37_locus: - lifted_over_genome_version = GENOME_VERSION_GRCh37 - lifted_over_chrom = grch37_locus['contig'] - lifted_over_pos = grch37_locus['position'] - else: - from seqr.utils.search.utils import run_liftover - grch37_coords = run_liftover(GENOME_VERSION_GRCh37, hit['contig'], hit['start']) - if grch37_coords: - lifted_over_genome_version = GENOME_VERSION_GRCh37 - lifted_over_chrom, lifted_over_pos = grch37_coords - result.update({ - 'liftedOverGenomeVersion': lifted_over_genome_version, - 'liftedOverChrom': lifted_over_chrom, - 'liftedOverPos': lifted_over_pos, - }) - - def _parse_compound_het_response(self, response): - if len(response.aggregations.genes.buckets) > MAX_COMPOUND_HET_GENES: - from seqr.utils.search.utils import InvalidSearchException - raise InvalidSearchException('This search returned too many compound heterozygous variants. Please add stricter filters') - - family_unaffected_individual_guids = { - family_guid: {individual_guid for individual_guid, affected_status in individual_affected_status.items() if - affected_status == Individual.AFFECTED_STATUS_UNAFFECTED} - for family_guid, individual_affected_status in self._family_individual_affected_status.items() - } - - self._allowed_consequences += self._consequence_overrides.keys() - if self._allowed_consequences_secondary: - self._allowed_consequences_secondary += self._consequence_overrides.keys() - - compound_het_pairs_by_gene = {} - for gene_agg in response.aggregations.genes.buckets: - self._parse_compound_het_gene(gene_agg, compound_het_pairs_by_gene, family_unaffected_individual_guids) - - total_compound_het_results = sum(len(compound_het_pairs) for compound_het_pairs in compound_het_pairs_by_gene.values()) - logger.info('Total compound het hits: {}'.format(total_compound_het_results), self._user) - - compound_het_results = [] - for k, compound_het_pairs in compound_het_pairs_by_gene.items(): - compound_het_results.extend([{k: compound_het_pair} for compound_het_pair in compound_het_pairs]) - return compound_het_results, total_compound_het_results - - def _parse_compound_het_gene(self, gene_agg, compound_het_pairs_by_gene, family_unaffected_individual_guids): - gene_variants = [self._parse_hit(hit) for hit in gene_agg['vars_by_gene']] - gene_id = gene_agg['key'] - - if gene_id in compound_het_pairs_by_gene: - return - - # Variants are returned if any transcripts have the filtered consequence, but to be compound het - # the filtered consequence needs to be present in at least one transcript in the gene of interest - if self._allowed_consequences: - gene_variants = self._filter_invalid_annotation_compound_hets(gene_id, gene_variants) - - if len(gene_variants) < 2: - return - - # Do not include groups multiple times if identical variants are in the same multiple genes - if any((not variant['mainTranscriptId']) or all(t['transcriptId'] != variant['mainTranscriptId'] - for t in variant['transcripts'][gene_id]) for variant in - gene_variants): - if not self._is_primary_compound_het_gene(gene_id, gene_variants, compound_het_pairs_by_gene): - return - - family_compound_het_pairs = defaultdict(list) - for variant in gene_variants: - for family_guid in variant['familyGuids']: - family_compound_het_pairs[family_guid].append(variant) - - self._filter_invalid_family_compound_hets(gene_id, family_compound_het_pairs, - family_unaffected_individual_guids) - - gene_compound_het_pairs = [ch_pair for ch_pairs in family_compound_het_pairs.values() for ch_pair in ch_pairs] - for compound_het_pair in gene_compound_het_pairs: - for variant in compound_het_pair: - variant['familyGuids'] = [family_guid for family_guid in variant['familyGuids'] - if len(family_compound_het_pairs[family_guid]) > 0] - variant.pop('gene_consequences', None) - gene_compound_het_pairs = [compound_het_pair for compound_het_pair in gene_compound_het_pairs - if compound_het_pair[0]['familyGuids'] and compound_het_pair[1]['familyGuids']] - if gene_compound_het_pairs: - compound_het_pairs_by_gene[gene_id] = gene_compound_het_pairs - - def _is_primary_compound_het_gene(self, gene_id, gene_variants, compound_het_pairs_by_gene): - primary_genes = set() - for variant in gene_variants: - if variant['mainTranscriptId']: - for gene, transcripts in variant['transcripts'].items(): - if any(t['transcriptId'] == variant['mainTranscriptId'] for t in transcripts): - primary_genes.add(gene) - break - if len(primary_genes) == 1: - is_valid_gene = True - primary_gene = primary_genes.pop() - if self._allowed_consequences: - is_valid_gene = all(any( - consequence in self._allowed_consequences for consequence in - variant['gene_consequences'].get(primary_gene, []) - ) for variant in gene_variants) - if is_valid_gene: - if primary_gene != gene_id: - return False - - else: - variant_ids = [variant['variantId'] for variant in gene_variants] - for gene in primary_genes: - if variant_ids == [compound_het_pair[0]['variantId'] for compound_het_pair in - compound_het_pairs_by_gene.get(gene, [])] and \ - variant_ids == [compound_het_pair[1]['variantId'] for compound_het_pair in - compound_het_pairs_by_gene.get(gene, [])]: - return False - return True - - def _filter_invalid_annotation_compound_hets(self, gene_id, gene_variants): - allowed_consequences = self._allowed_consequences + (self._allowed_consequences_secondary or []) - has_canonical_transcript_filter = CANONICAL_TRANSCRIPT_FILTER in allowed_consequences - for variant in gene_variants: - all_gene_consequences = [] - if variant.get('svType'): - all_gene_consequences.append(variant['svType']) - if _is_matched_clinvar_significance(variant.get(CLINVAR_KEY, {}).get('clinicalSignificance'), self._consequence_overrides.get(CLINVAR_KEY)): - all_gene_consequences.append(CLINVAR_KEY) - if variant.get(HGMD_KEY, {}).get('class') in self._consequence_overrides.get(HGMD_KEY, []): - all_gene_consequences.append(HGMD_KEY) - splice_ai = variant.get('predictions', {}).get(SPLICE_AI_FIELD) - if splice_ai and splice_ai >= self._consequence_overrides.get(SPLICE_AI_FIELD, 100): - all_gene_consequences.append(SPLICE_AI_FIELD) - - variant['gene_consequences'] = {} - for k, transcripts in variant['transcripts'].items(): - transcript_consequences = [ - (transcript['majorConsequence'], transcript.get('canonical')) - for transcript in transcripts if transcript.get('majorConsequence') - ] - variant['gene_consequences'][k] = all_gene_consequences + [ - consequence for consequence, _ in transcript_consequences - ] - if has_canonical_transcript_filter and any( - canonical for consequence, canonical in transcript_consequences - if consequence == 'non_coding_transcript_exon_variant'): - variant['gene_consequences'][k].append(CANONICAL_TRANSCRIPT_FILTER) - - return [variant for variant in gene_variants if any( - consequence in allowed_consequences - for consequence in variant['gene_consequences'].get(gene_id, []) - )] - - def _filter_invalid_family_compound_hets(self, gene_id, family_compound_het_pairs, family_unaffected_individual_guids): - for family_guid, variants in family_compound_het_pairs.items(): - unaffected_individuals = family_unaffected_individual_guids.get(family_guid, []) - - hom_alt_variant_ids = set() - if self._paired_index_comp_het: - hom_alt_variant_ids = { - var['variantId'] for var in variants if any(gen.get('numAlt') == 2 for gen in var['genotypes'].values()) - } - - valid_combinations = [] - for ch_1_index, ch_2_index in combinations(range(len(variants)), 2): - variant_1 = variants[ch_1_index] - variant_2 = variants[ch_2_index] - - if hom_alt_variant_ids: - # SNPs overlapped by trans deletions may be incorrectly called as hom alt, and should be - # considered comp hets with said deletions. Any other hom alt variants are not valid comp hets - hom_alt_var = next( - (var for var in [variant_1, variant_2] if var['variantId'] in hom_alt_variant_ids), None) - if hom_alt_var: - pair_var = variant_1 if hom_alt_var == variant_2 else variant_2 - is_valid = pair_var.get('svType') == 'DEL' and pair_var['pos'] <= hom_alt_var['pos'] <= pair_var['end'] - if not is_valid: - continue - - is_valid_for_individual = True - for individual_guid in unaffected_individuals: - genotype_1 = variant_1['genotypes'].get(individual_guid) - genotype_2 = variant_2['genotypes'].get(individual_guid) - if genotype_1 and genotype_2 and genotype_1.get('numAlt') != 0 and not genotype_1.get('isRef') and \ - genotype_2.get('numAlt') != 0 and not genotype_2.get('isRef'): - is_valid_for_individual = False - break - if not is_valid_for_individual: - continue - - if self._allowed_consequences and self._allowed_consequences_secondary: - # Make a copy of lists to prevent blowing up memory usage - consequences = [] + variant_1['gene_consequences'].get(gene_id, []) - consequences += variant_2['gene_consequences'].get(gene_id, []) - if all(consequence not in self._allowed_consequences for consequence in consequences) or all( - consequence not in self._allowed_consequences_secondary for consequence in consequences): - continue - - valid_combinations.append([ch_1_index, ch_2_index]) - - family_compound_het_pairs[family_guid] = [ - [variants[valid_ch_1_index], variants[valid_ch_2_index]] for - valid_ch_1_index, valid_ch_2_index in valid_combinations] - def _deduplicate_results(self, sorted_new_results): original_result_count = len(sorted_new_results) variant_results = [] @@ -1158,82 +419,6 @@ def _deduplicate_results(self, sorted_new_results): return variant_results - @classmethod - def _merge_duplicate_variants(cls, variant, duplicate_variant): - for guid, genotype in duplicate_variant['genotypes'].items(): - if guid in variant['genotypes']: - variant['genotypes'][guid]['otherSample'] = {k: v for k, v in genotype.items() if k != 'otherSample'} - else: - variant['genotypes'][guid] = genotype - variant['familyGuids'] = sorted(set(variant['familyGuids'] + duplicate_variant['familyGuids'])) - - # Always show the most up-to-date clinvar - clinvar_version = variant[CLINVAR_KEY]['version'] or '1900-01-01' - dup_clinvar_version = duplicate_variant[CLINVAR_KEY]['version'] or '1900-01-01' - if dup_clinvar_version > clinvar_version: - variant[CLINVAR_KEY] = duplicate_variant[CLINVAR_KEY] - - def _deduplicate_compound_het_results(self, compound_het_results): - duplicates = 0 - results = {} - for gene_compound_het_pair in compound_het_results: - gene = next(iter(gene_compound_het_pair)) - compound_het_pair = gene_compound_het_pair[gene] - if gene in results: - variant_ids = {variant['variantId'] for variant in compound_het_pair} - existing_index = next( - (i for i, existing in enumerate(results[gene]) if - {variant['variantId'] for variant in existing} == variant_ids), None, - ) - if existing_index is not None: - existing_compound_het_pair = results[gene][existing_index] - self._merge_duplicate_variants(existing_compound_het_pair[0], compound_het_pair[0]) - self._merge_duplicate_variants(existing_compound_het_pair[1], compound_het_pair[1]) - duplicates += 1 - else: - results[gene].append(compound_het_pair) - else: - results[gene] = [compound_het_pair] - - deduplicated_results = [] - for gene, compound_het_pairs in results.items(): - deduplicated_results += [{gene: ch_pair} for ch_pair in compound_het_pairs] - - self.previous_search_results['duplicate_doc_count'] = duplicates + self.previous_search_results.get('duplicate_doc_count', 0) - self.previous_search_results['total_results'] -= duplicates - - return deduplicated_results - - def _process_compound_hets(self, compound_het_results, variant_results, num_results, all_loaded=False): - if not self.previous_search_results.get('grouped_results'): - self.previous_search_results['grouped_results'] = [] - - # Sort merged result sets - grouped_variants = [{None: [var]} for var in variant_results] - grouped_variants = compound_het_results + grouped_variants - grouped_variants = _sort_compound_hets(grouped_variants) - - # Get requested page of variants - merged_variant_results = [] - num_compound_hets = 0 - num_single_variants = 0 - for variants_group in grouped_variants: - variants = next(iter(variants_group.values())) - - self.previous_search_results['grouped_results'].append(variants_group) - if len(variants) > 1: - merged_variant_results.append(variants) - num_compound_hets += 1 - else: - merged_variant_results += variants - num_single_variants += 1 - if not all_loaded and len(merged_variant_results) >= num_results: - break - - self.previous_search_results['compound_het_results'] = compound_het_results[num_compound_hets:] - self.previous_search_results['variant_results'] = variant_results[num_single_variants:] - return merged_variant_results - def _get_paginated_searches(self, index_name, page=1, num_results=100, start_index=None): searches = [] log_messages = [] @@ -1268,60 +453,6 @@ def _execute_search(self, search): logger.warning('ES Query Timeout. No long running searches found', self._user) raise e - def _get_long_running_tasks(self): - search_tasks = self._client.tasks.list(actions='*search', group_by='parents') - long_running = [] - for parent_id, task in search_tasks['tasks'].items(): - if task['running_time_in_nanos'] > 10 ** 11: - long_running.append({'task': task, 'parent_task_id': parent_id}) - return long_running - - def _get_quality_filters_by_family(self, quality_filter, affected_only): - quality_field_configs = { - 'min_{}'.format(field): {'field': field, 'step': step} for field, step in QUALITY_QUERY_FIELDS.items() - } - quality_filter = dict({field: 0 for field in quality_field_configs.keys()}, **(quality_filter or {})) - for field, config in quality_field_configs.items(): - if quality_filter[field] % config['step'] != 0: - raise Exception('Invalid {} filter {}'.format(config['field'], quality_filter[field])) - - quality_filters_by_family = {} - new_svs = self._consequence_overrides.get(NEW_SV_FIELD) - if new_svs or any(quality_filter[field] for field in quality_field_configs.keys()): - family_sample_ids = defaultdict(set) - for index in self._indices: - family_samples_by_id = self.samples_by_family_index[index] - for family_guid, samples_by_id in family_samples_by_id.items(): - if affected_only: - family_affected_status = self._family_individual_affected_status[family_guid] - sample_ids = { - sample_id for sample_id, sample in samples_by_id.items() - if family_affected_status[sample.individual.guid] == Individual.AFFECTED_STATUS_AFFECTED - } - else: - sample_ids = samples_by_id.keys() - family_sample_ids[family_guid].update(sample_ids) - - path_filter = self._get_clinvar_pathogenic_override_filter() - for family_guid, sample_ids in sorted(family_sample_ids.items()): - quality_q = Q('terms', samples_new_call=sorted(sample_ids)) if new_svs else Q() - for sample_id in sorted(sample_ids): - for field, config in sorted(quality_field_configs.items()): - if quality_filter[field]: - q = _build_or_filter('term', [ - {'samples_{}_{}_to_{}'.format(config['field'], i, i + config['step']): sample_id} - for i in range(0, quality_filter[field], config['step']) - ]) - if field == 'min_ab': - # AB only relevant for hets - quality_q &= ~Q(q) | ~Q('term', samples_num_alt_1=sample_id) - else: - quality_q &= ~Q(q) - if path_filter: - quality_q |= path_filter - quality_filters_by_family[family_guid] = quality_q - return quality_filters_by_family - def _get_family_affected_status(samples_by_id, inheritance_filter): individual_affected_status = inheritance_filter.get('affected') or {} @@ -1333,224 +464,10 @@ def _get_family_affected_status(samples_by_id, inheritance_filter): return affected_status -def _any_affected_sample_filter(sample_ids): - sample_ids = sorted(sample_ids) - return Q('terms', samples_num_alt_1=sample_ids) | Q('terms', samples_num_alt_2=sample_ids) | Q('terms', samples=sample_ids) - - -def _family_genotype_inheritance_filter(inheritance_mode, inheritance_filter, samples_by_id, individual_affected_status, index_fields): - samples_q = None - - individuals = [sample.individual for sample in samples_by_id.values()] - - individual_genotype_filter = inheritance_filter.get('genotype') or {} - - if inheritance_mode == X_LINKED_RECESSIVE: - samples_q = Q('range', xpos={'gte': get_xpos('X', 1), 'lte': get_xpos('Y', 1)}) - for individual in individuals: - if individual_affected_status[individual.guid] == Individual.AFFECTED_STATUS_UNAFFECTED \ - and individual.sex == Individual.SEX_MALE: - individual_genotype_filter[individual.guid] = REF_REF - - is_sv_comp_het = inheritance_mode == COMPOUND_HET and 'samples' in index_fields - for sample_id, sample in sorted(samples_by_id.items()): - - individual_guid = sample.individual.guid - affected = individual_affected_status[individual_guid] - - genotype = individual_genotype_filter.get(individual_guid) or inheritance_filter.get(affected) - if genotype: - if is_sv_comp_het and affected == Individual.AFFECTED_STATUS_UNAFFECTED: - # Unaffected individuals for SV compound het search can have any genotype so are not included - continue - - not_allowed_num_alt = [ - num_alt for num_alt in GENOTYPE_QUERY_MAP[genotype].get('not_allowed_num_alt', []) - if num_alt in index_fields - ] - allowed_num_alt = [ - num_alt for num_alt in GENOTYPE_QUERY_MAP[genotype].get('allowed_num_alt', []) - if num_alt in index_fields - ] - num_alt_to_filter = not_allowed_num_alt or allowed_num_alt - sample_filters = [{num_alt_key: sample_id} for num_alt_key in num_alt_to_filter] - - sample_q = _build_or_filter('term', sample_filters) - if not_allowed_num_alt: - sample_q = ~Q(sample_q) - - if not samples_q: - samples_q = sample_q - else: - samples_q &= sample_q - - return samples_q - - -def _named_family_sample_q(family_samples_q, family_guid, quality_filters_by_family): - sample_queries = [family_samples_q] - quality_q = quality_filters_by_family.get(family_guid) - if quality_q: - sample_queries.append(quality_q) - - return Q('bool', must=sample_queries, _name=family_guid) - - -def _location_filter(genes, intervals, exclude_locations): - q = None - - if genes: - q = Q('terms', geneIds=list(genes.keys())) - - if intervals: - for interval in intervals: - if interval.get('offset'): - offset_pos = int((interval['end'] - interval['start']) * interval['offset']) - interval_q = Q( - 'range', xpos=_pos_offset_range_filter(interval['chrom'], interval['start'], offset_pos)) & Q( - 'range', xstop=_pos_offset_range_filter(interval['chrom'], interval['end'], offset_pos)) - else: - xstart = get_xpos(interval['chrom'], interval['start']) - xstop = get_xpos(interval['chrom'], interval['end']) - range_filters = [{ - key: { - 'gte': xstart, - 'lte': xstop, - } - } for key in ['xpos', 'xstop']] - interval_q = _build_or_filter('range', range_filters) - interval_q |= Q('range', xpos={'lte': xstart}) & Q('range', xstop={'gte': xstop}) & \ - Q('range', xpos={'gte': get_xpos(interval['chrom'], MIN_POS)}) & \ - Q('range', xstop={'lte': get_xpos(interval['chrom'], MAX_POS)}) - - if q: - q |= interval_q - else: - q = interval_q - - if exclude_locations: - return ~q - else: - return q - - -def _pos_offset_range_filter(chrom, pos, offset): - return { - 'lte': get_xpos(chrom, min(pos + offset, MAX_POS)), - 'gte': get_xpos(chrom, max(pos - offset, MIN_POS)), - } - -def _parse_pathogenicity_filter(pathogenicity): - clinvar_filters = pathogenicity.get(CLINVAR_KEY, []) - hgmd_filters = pathogenicity.get(HGMD_KEY, []) - - hgmd_class = set() - if hgmd_filters: - for hgmd_filter in hgmd_filters: - hgmd_class.update(HGMD_CLASS_MAP.get(hgmd_filter, [])) - - return sorted(clinvar_filters), sorted(hgmd_class) - - VUS_FILTER = 'vus_or_conflicting' VUS_REGEX = 'Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))' -def _pathogenicity_filter(clinvar_filters, hgmd_classes=None): - pathogenicity_filter = None - - path_regex = '|'.join([ - VUS_REGEX if clinvar_filter == VUS_FILTER else f'.*{clinvar_filter.capitalize()}.*' - for clinvar_filter in clinvar_filters or [] - ]) - if path_regex: - pathogenicity_filter = Q('regexp', clinvar_clinical_significance=path_regex) - - if hgmd_classes: - hgmd_q = Q('terms', hgmd_class=hgmd_classes) - pathogenicity_filter = pathogenicity_filter | hgmd_q if pathogenicity_filter else hgmd_q - - return pathogenicity_filter - - -def _is_matched_clinvar_significance(clinical_significance, clinvar_filters): - if not (clinical_significance and clinvar_filters): - return False - - if VUS_FILTER in clinvar_filters: - exclude = [ - path.capitalize() for path in ['pathogenic', 'likely_pathogenic', 'likely_benign', 'benign'] - if path not in clinvar_filters - ] - return all(substring not in clinical_significance for substring in exclude) - - return any(clinvar_filter.capitalize() in clinical_significance for clinvar_filter in clinvar_filters) - - -def _annotations_filter(vep_consequences): - consequences_filter = Q('terms', transcriptConsequenceTerms=sorted(vep_consequences)) - - if 'intergenic_variant' in vep_consequences: - # VEP doesn't add annotations for many intergenic variants so match variants where no transcriptConsequenceTerms - consequences_filter |= ~Q('exists', field='transcriptConsequenceTerms') - - return consequences_filter - - -def _in_silico_filter(in_silico_filters, require_score): - in_silico_qs = [] - for prediction_key, value in in_silico_filters.items(): - try: - score_q = Q('range', **{prediction_key: {'gte': float(value)}}) - except ValueError: - score_q = Q('prefix', **{prediction_key: value}) - - if not require_score: - score_q |= ~Q('exists', field=prediction_key) - - in_silico_qs.append(score_q) - - return _or_filters(in_silico_qs) - - -def _pop_freq_filter(filter_key, value): - return Q('range', **{filter_key: {'lte': value}}) | ~Q('exists', field=filter_key) - - -def _build_or_filter(op, filters): - return _or_filters([Q(op, **filter_kwargs) for filter_kwargs in filters]) - - -def _or_filters(filter_qs): - q = filter_qs[0] - for filter_q in filter_qs[1:]: - q |= filter_q - return q - - -def _sort_compound_hets(grouped_variants): - return sorted(grouped_variants, key=lambda variants: next(iter(variants.values()))[0]['_sort']) - - -def get_compound_het_page(grouped_variants, start_index, end_index): - skipped = 0 - variant_results = [] - variant_count = 0 - for variants in grouped_variants: - curr_variant = next(iter(variants.values())) - if skipped < start_index: - skipped += 1 - else: - if len(curr_variant) == 1: - variant_results += curr_variant - else: - variant_results.append(curr_variant) - variant_count += 1 - if variant_count + skipped >= end_index: - return variant_results - return None - - def _parse_es_sort(sort, sort_config): if sort in {'Infinity', '-Infinity', None}: # ES returns these values for sort when a sort field is missing, using the correct value for the given direction diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 5fe4a68106..aa7fda492a 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -7,8 +7,7 @@ from seqr.models import Sample, SavedVariant from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY -from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch -from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page +from seqr.utils.search.elasticsearch.es_search import EsSearch from seqr.views.utils.json_utils import _to_camel_case from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER @@ -32,21 +31,6 @@ class InvalidIndexException(Exception): MAX_VARIANTS_FETCH = 1000 -def _get_transport_error_type(error): - error_type = 'no detail' - if isinstance(error, dict): - root_cause = error.get('root_cause') - error_info = error.get('error') - if (not root_cause) and isinstance(error_info, dict): - root_cause = error_info.get('root_cause') - - if root_cause: - error_type = root_cause[0].get('type') or root_cause[0].get('reason') - elif error_info and not isinstance(error_info, dict): - error_type = repr(error_info) - return error_type - - def es_backend_enabled(): return bool(ELASTICSEARCH_SERVICE_HOSTNAME) @@ -273,68 +257,6 @@ def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): return variants.search(num_results=len(variant_ids)) -def get_es_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=None, num_results=None, - gene_agg=False, skip_genotype_filter=False): - es_search_cls = EsGeneAggSearch if gene_agg else EsSearch - - es_search = es_search_cls( - samples, - genome_version, - previous_search_results=previous_search_results, - user=user, - sort=sort, - skipped_samples=search.get('skipped_samples'), - ) - - es_search.filter_variants( - inheritance_mode=search.get('inheritance_mode'), inheritance_filter=search.get('inheritance_filter'), - frequencies=search.get('freqs'), pathogenicity=search.get('pathogenicity'), - annotations=search.get('annotations'), annotations_secondary=search.get('annotations_secondary'), - in_silico=search.get('in_silico'), quality_filter=search.get('qualityFilter'), - custom_query=search.get('customQuery'), locus=search.get('locus'), skip_genotype_filter=skip_genotype_filter, - dataset_type=search.get('dataset_type'), secondary_dataset_type=search.get('secondary_dataset_type'), - genes=search.get('genes'), intervals=search.get('intervals'), rs_ids=search.get('rs_ids'), - variant_ids=search.get('variant_ids'), exclude_locations=search.get('exclude_locations'), - ) - - return es_search.search(page=page, num_results=num_results) - - -def process_es_previously_loaded_results(previous_search_results, start_index, end_index): - grouped_results = previous_search_results.get('grouped_results') - results = None - if grouped_results: - results = get_compound_het_page(grouped_results, start_index, end_index) - - return results - - -def process_es_previously_loaded_gene_aggs(previous_search_results): - total_results = previous_search_results.get('total_results') - if total_results is None or 'all_results' in previous_search_results or 'grouped_results' not in previous_search_results: - return None - - loaded = sum(counts.get('loaded', 0) for counts in previous_search_results.get('loaded_variant_counts', {}).values()) - if loaded != total_results: - return None - - gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - - for group in previous_search_results['grouped_results']: - variants = next(iter(group.values())) - gene_id = next(iter(group)) - if not gene_id or gene_id == 'null': - gene_id = next(( - gene_id for gene_id, transcripts in variants[0]['transcripts'].items() - if any(t['transcriptId'] == variants[0]['mainTranscriptId'] for t in transcripts) - ), None) if variants[0]['mainTranscriptId'] else None - if gene_id: - gene_aggs[gene_id]['total'] += len(variants) - for family_guid in variants[0]['familyGuids']: - gene_aggs[gene_id]['families'][family_guid] += len(variants) - return gene_aggs - - def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): from seqr.views.utils.variant_utils import get_saved_variants saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 0a3a348d27..e4b5431e87 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -1288,9 +1288,6 @@ def get_search_callback(request): response = create_mock_response(body, get_indices_from_url(request.url)) return 200, {}, json.dumps(response) -def parse_msearch_body(body): - return [json.loads(row) for row in body.decode().split('\n') if row] - def get_msearch_callback(request): body = parse_msearch_body(request.body) response = { @@ -1336,13 +1333,6 @@ def assertExecutedSearch(self, filters=None, start_index=0, size=2, index=INDEX_ expected_source_fields=expected_source_fields, ) - def assertExecutedSearches(self, searches): - executed_search = parse_msearch_body(urllib3_responses.calls[-1].request.body) - self.assertEqual(len(executed_search), len(searches) * 2) - for i, expected_search in enumerate(searches): - self.assertDictEqual(executed_search[i * 2], {'index': expected_search.get('index', INDEX_NAME).split(',')}) - self.assertSameSearch(executed_search[(i * 2) + 1], expected_search) - def assertSameSearch(self, executed_search, expected_search_params, expected_source_fields=SOURCE_FIELDS): expected_search = { 'from': expected_search_params['start_index'], @@ -1384,12 +1374,6 @@ def assertSameSearch(self, executed_search, expected_search_params, expected_sou if expected_search_params.get('gene_aggs') else executed_search['_source'] self.assertSetEqual(expected_source_fields, set(source)) - def assertCachedResults(self, results_model, expected_results, sort='xpos'): - cache_key = 'search_results__{}__{}'.format(results_model.guid, sort) - self.assertIn(cache_key, REDIS_CACHE.keys()) - self.assertDictEqual(json.loads(REDIS_CACHE[cache_key]), expected_results) - MOCK_REDIS.expire.assert_called_with(cache_key, timedelta(weeks=2)) - @urllib3_responses.activate def test_get_es_variants_for_variant_ids(self): setup_responses() From 43e6d0bf2e39d841aea53dc88047ee3feeae4f00 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Feb 2026 12:12:46 -0500 Subject: [PATCH 025/252] clean up --- seqr/utils/search/elasticsearch/es_search.py | 81 ------------------- .../search/elasticsearch/es_utils_tests.py | 21 ----- 2 files changed, 102 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index 3f3d637bfd..09c0e872d1 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -89,16 +89,6 @@ def __init__(self, samples, genome_version, previous_search_results=None, return @staticmethod def _parse_xstop(result): xstop = result.pop(XSTOP_FIELD, None) - if xstop: - end_chrom, end = get_chrom_pos(xstop) - if end_chrom != result['chrom'] or end != result['end']: - if result['svType'] == 'INS': - result['svSourceDetail'] = {'chrom': end_chrom} - else: - result.update({ - 'endChrom': end_chrom, - 'end': end, - }) def _get_index_dataset_type(self, index): return self.get_index_metadata_dataset_type(self.index_metadata[index]) @@ -116,15 +106,6 @@ def _set_indices(self, indices): def _set_index_name(self): self.index_name = ','.join(sorted(self._indices)) - if len(self.index_name) > MAX_INDEX_NAME_LENGTH: - alias = hashlib.md5(self.index_name.encode('utf-8')).hexdigest() # nosec - cache_key = 'index_alias__{}'.format(alias) - if safe_redis_get_json(cache_key) != self.index_name: - self._client.indices.update_aliases(body={'actions': [ - {'add': {'indices': self._indices, 'alias': alias}} - ]}) - safe_redis_set_json(cache_key, self.index_name) - self.index_name = alias def _set_index_metadata(self): from seqr.utils.search.elasticsearch.es_utils import get_index_metadata @@ -133,29 +114,6 @@ def _set_index_metadata(self): def _sort_variants(self, sample_data): main_sort_dict = self._sort[0] if len(self._sort) and isinstance(self._sort[0], dict) else None - # Add parameters to scripts - if main_sort_dict and main_sort_dict.get('_script', {}).get('script', {}).get('params'): - called_params = None - for key, val_func in self._sort[0]['_script']['script']['params'].items(): - if callable(val_func): - self._sort[0]['_script']['script']['params'][key] = val_func(sample_data) - called_params = self._sort[0]['_script']['script']['params'] - if called_params: - for sort_dict in self._sort[1:]: - sort_dict['_script']['script']['params'] = called_params - - - # Add unmapped_type - if main_sort_dict and 'unmapped_type' in list(main_sort_dict.values())[0]: - sort_field = list(main_sort_dict.keys())[0] - field_type = next(( - metadata['fields'][sort_field] for metadata in self.index_metadata.values() - if metadata['fields'].get(sort_field) - ), 'double') - if field_type == 'keyword': - self._sort[0][sort_field]['unmapped_type'] = field_type - self._sort[0][sort_field].pop('numeric_type') - if XPOS_SORT_KEY not in self._sort: self._sort.append(XPOS_SORT_KEY) @@ -182,10 +140,6 @@ def search(self, page=1, num_results=100): if is_single_search: return self._execute_single_search(**search_kwargs) - elif not self._index_searches: - return self._execute_single_search(**search_kwargs) - else: - return self._execute_multi_search(**search_kwargs) def _is_single_search(self): return len(self._indices) == 1 and len(self._index_searches) < 2 and \ @@ -231,11 +185,6 @@ def _process_single_search_response(self, parsed_response, page=1, num_results=1 self.previous_search_results['total_results'] = total_results results_start_index = (page - 1) * num_results - if is_compound_het: - variant_results = _sort_compound_hets(variant_results) - self.previous_search_results['grouped_results'] = variant_results - end_index = min(results_start_index + num_results, total_results) - return get_compound_het_page(variant_results, results_start_index, end_index) if deduplicate: variant_results = self._deduplicate_results(variant_results) @@ -250,9 +199,6 @@ def _process_single_search_response(self, parsed_response, page=1, num_results=1 def _parse_response(self, response): index_name = response.hits[0].meta.index if response.hits else None - if hasattr(response.aggregations, 'genes') and response.hits: - response_hits, response_total = self._parse_compound_het_response(response) - return response_hits, response_total, True, index_name response_total = response.hits.total['value'] logger.info('Total hits: {} ({} seconds)'.format(response_total, response.took / 1000.0), self._user) @@ -365,16 +311,6 @@ def _is_matched_sample(family_guid, sample): genotype_hit['sample_type'] = sample.sample_type genotypes[sample.individual.guid] = _get_field_values(genotype_hit, genotype_fields_config) - if len(samples_by_id) != len(genotypes) and data_type == Sample.DATASET_TYPE_SV_CALLS: - # Family members with no variants are not included in the SV index - for sample_id, sample in samples_by_id.items(): - if sample.individual.guid not in genotypes: - genotypes[sample.individual.guid] = _get_field_values( - {'sample_id': sample_id}, genotype_fields_config) - genotypes[sample.individual.guid]['isRef'] = True - genotypes[sample.individual.guid]['cn'] = \ - 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 - return family_guids, genotypes def _get_main_transcript(self, sorted_transcripts): @@ -382,23 +318,6 @@ def _get_main_transcript(self, sorted_transcripts): if len(sorted_transcripts) and 'transcriptRank' in sorted_transcripts[0] else None selected_main_transcript_id = None - if main_transcript_id and (self._filtered_gene_ids or self._allowed_consequences): - gene_transcripts = [ - t for t in sorted_transcripts if t.get('geneId') in self._filtered_gene_ids - ] if self._filtered_gene_ids else sorted_transcripts - - selected_main_transcript_id = gene_transcripts[0].get('transcriptId') - if self._allowed_consequences: - consequence_transcript_id = next(( - t.get('transcriptId') for t in gene_transcripts if - self._is_matched_transcript(t, self._allowed_consequences)), None) - if not consequence_transcript_id and self._allowed_consequences_secondary: - consequence_transcript_id = next(( - t.get('transcriptId') for t in gene_transcripts if self._is_matched_transcript(t, self._allowed_consequences_secondary) - ), None) - selected_main_transcript_id = consequence_transcript_id or selected_main_transcript_id - if selected_main_transcript_id == main_transcript_id: - selected_main_transcript_id = None return main_transcript_id, selected_main_transcript_id diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index e4b5431e87..52c092f158 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -1185,23 +1185,10 @@ def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=Non matched_queries = hit.pop('matched_queries') if include_matched_queries: hit['matched_queries'] = [] - for subindex in index.split(','): - if subindex in matched_queries: - hit['_index'] = subindex - hit['matched_queries'] += matched_queries[subindex] if sort or increment_sort: sort_key = sort[0] if sort else 'xpos' - if isinstance(sort_key, dict): - if '_script' in sort_key: - sort_key = sort_key['_script']['script']['params'].get('field', 'xpos') # pylint: disable=invalid-sequence-index - else: - sort_key = next(iter(sort_key.keys())) sort_value = jmespath.search(sort_key, hit['_source']) - if sort_value is None: - sort_value = 'Infinity' - if increment_sort: - sort_value += 100 hit['_sort'] = [sort_value] return parsed_hits @@ -1218,10 +1205,6 @@ def create_mock_response(search, index=INDEX_NAME): variant_id_filters = search_filter.get('terms', {}).get('variantId') if not gene_ids_filters and not search.get('aggs'): gene_ids_filters.update(search_filter.get('terms', {}).get('geneIds') or []) - possible_inheritance_filters = search_filter.get('bool', {}).get('should', []) + [search_filter] - if any('_name' in possible_filter.get('bool', {}) for possible_filter in possible_inheritance_filters): - include_matched_queries = True - break response_dict = { 'took': 1, @@ -1233,10 +1216,6 @@ def create_mock_response(search, index=INDEX_NAME): index=index_name) if variant_id_filters: index_hits = [hit for hit in index_hits if hit['_id'] in variant_id_filters] - elif gene_ids_filters: - index_hits = [hit for hit in index_hits if any( - gene_ids_filters.intersection({t['gene_id'] for t in hit['_source']['sortedTranscriptConsequences']}) - )] response_dict['hits']['hits'] += index_hits try: From 0ba4feec4a840cc672c0a941998b6588549780eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Feb 2026 15:08:19 -0500 Subject: [PATCH 026/252] clean up --- seqr/utils/search/elasticsearch/es_search.py | 53 +------------------- 1 file changed, 1 insertion(+), 52 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index 09c0e872d1..c32a3086cb 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -44,21 +44,10 @@ def __init__(self, samples, genome_version, previous_search_results=None, return self._set_indices(sorted(list(self.samples_by_family_index.keys()))) self._set_index_metadata() - if len(self.samples_by_family_index) > len(self.index_metadata): - raise InvalidIndexException('Could not find expected indices: {}'.format( - ', '.join(sorted(set(self._indices) - set(self.index_metadata.keys()), reverse = True)) - )) - elif len(self.index_metadata) > len(self.samples_by_family_index): - # Some of the indices are an alias - self._update_alias_metadata() - invalid_genome_indices = [ f"{index} ({meta['genomeVersion']})" for index, meta in self.index_metadata.items() if meta['genomeVersion'] != genome_version ] - if invalid_genome_indices: - raise InvalidSearchException( - f'The following indices do not have the expected genome version {genome_version}: {", ".join(invalid_genome_indices)}') self._genome_version = genome_version self._skipped_samples = skipped_samples @@ -96,8 +85,6 @@ def _get_index_dataset_type(self, index): @staticmethod def get_index_metadata_dataset_type(index_metadata): data_type = index_metadata.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) - if data_type == 'VARIANTS': - data_type = Sample.DATASET_TYPE_VARIANT_CALLS return data_type def _set_indices(self, indices): @@ -114,9 +101,6 @@ def _set_index_metadata(self): def _sort_variants(self, sample_data): main_sort_dict = self._sort[0] if len(self._sort) and isinstance(self._sort[0], dict) else None - if XPOS_SORT_KEY not in self._sort: - self._sort.append(XPOS_SORT_KEY) - # always final sort on variant ID to keep different variants at the same position grouped properly if 'variantId' not in self._sort: self._sort.append('variantId') @@ -151,9 +135,6 @@ def _should_execute_single_search(self, page=1, num_results=100): if is_single_search and not self.previous_search_results.get('grouped_results'): start_index = None - if (page - 1) * num_results < num_loaded: - start_index = num_loaded - return True, {'page': page, 'num_results': num_results, 'start_index': start_index} elif not self._index_searches: # If doing all project-families all inheritance search, do it as a single query # Load all variants, do not skip pages @@ -163,13 +144,9 @@ def _should_execute_single_search(self, page=1, num_results=100): else: start_index = 0 return True, {'page': page, 'num_results': num_results, 'start_index': start_index, 'deduplicate': True} - else: - return False, {'page': page, 'num_results': num_results} def _execute_single_search(self, page=1, num_results=100, start_index=None, deduplicate=False, **kwargs): num_results_for_search = num_results * len(self._indices) if deduplicate else num_results - if num_results_for_search > MAX_VARIANTS and deduplicate: - num_results_for_search = MAX_VARIANTS searches, log_messages = self._get_paginated_searches( self.index_name, page=page, num_results=num_results_for_search, start_index=start_index ) @@ -221,15 +198,9 @@ def _parse_hit(self, raw_hit): result['_sort'] = [_parse_es_sort(sort, self._sort[i]) for i, sort in enumerate(raw_hit.meta.sort)] result['genomeVersion'] = self._genome_version - if self._genome_version == GENOME_VERSION_GRCh38: - self._add_liftover(result, hit) self._parse_xstop(result) result[CLINVAR_KEY]['version'] = self.index_metadata[index_name].get('clinvar_version') - # If an SV has genotype-specific coordinates that differ from the main coordinates, use those - if data_type == Sample.DATASET_TYPE_SV_CALLS and genotypes: - self._set_sv_genotype_coords(genotypes, result) - populations = { population: _get_field_values( hit, POPULATION_RESPONSE_FIELD_CONFIGS, format_response_key=lambda key: key.lower(), @@ -246,21 +217,11 @@ def _parse_hit(self, raw_hit): for transcript in hit[SORTED_TRANSCRIPTS_FIELD_KEY] or [] ] - # Temporary code, remove it after all the SV indices are reloaded - for trans in sorted_transcripts: - if trans.get('majorConsequence'): - if trans['majorConsequence'] == 'MSV_EXON_OVR': - trans['majorConsequence'] = 'MSV_EXON_OVERLAP' - elif trans['majorConsequence'] == 'DUP_LOF': - trans['majorConsequence'] = 'INTRAGENIC_EXON_DUP' - transcripts = defaultdict(list) for transcript in sorted_transcripts: if transcript['geneId']: transcripts[transcript['geneId']].append(transcript) gene_ids = result.pop('geneIds', None) - if gene_ids: - transcripts = {gene_id: ts for gene_id, ts in transcripts.items() if gene_id in gene_ids} main_transcript_id, selected_main_transcript_id = self._get_main_transcript(sorted_transcripts) result.update({ @@ -280,8 +241,6 @@ def _parse_hit(self, raw_hit): def _parse_genotypes(self, raw_hit, hit, index_family_samples, data_type): if hasattr(raw_hit.meta, 'matched_queries'): family_guids = list(raw_hit.meta.matched_queries) - elif self._return_all_queried_families: - family_guids = list(index_family_samples.keys()) else: # Searches for all inheritance and all families do not filter on inheritance so there are no matched_queries alt_allele_samples = set() @@ -290,10 +249,7 @@ def _parse_genotypes(self, raw_hit, hit, index_family_samples, data_type): alt_allele_samples.update(hit[alt_samples_field]) if self._any_affected_sample_filters: - # If using the any inheritance filter only include matched families - def _is_matched_sample(family_guid, sample): - return self._family_individual_affected_status[family_guid][sample.individual.guid] == \ - Individual.AFFECTED_STATUS_AFFECTED + pass else: _is_matched_sample = lambda *args: True @@ -350,8 +306,6 @@ def _get_paginated_searches(self, index_name, page=1, num_results=100, start_ind log_messages.append('Loading {}s for {}'.format(self.AGGREGATION_NAME, index_name)) else: end_index = page * num_results - if start_index is None: - start_index = end_index - num_results search = search[start_index:end_index] search = search.source(QUERY_FIELD_NAMES) @@ -365,11 +319,6 @@ def _execute_search(self, search): try: return search.using(self._client).execute() except elasticsearch.exceptions.ConnectionTimeout as e: - tasks = self._get_long_running_tasks() - if tasks: - logger.error('ES Query Timeout: Found {} long running searches'.format(len(tasks)), self._user, detail=tasks) - else: - logger.warning('ES Query Timeout. No long running searches found', self._user) raise e From 7203f13e8555560c736b59ff70b7801b27da2729 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Feb 2026 15:35:05 -0500 Subject: [PATCH 027/252] clean up --- seqr/utils/search/elasticsearch/es_search.py | 10 +---- .../search/elasticsearch/es_utils_tests.py | 42 +------------------ 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index c32a3086cb..59503053f1 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -133,16 +133,12 @@ def _should_execute_single_search(self, page=1, num_results=100): is_single_search = self._is_single_search() num_loaded = len(self.previous_search_results.get('all_results', [])) - if is_single_search and not self.previous_search_results.get('grouped_results'): - start_index = None - elif not self._index_searches: + if not self._index_searches: # If doing all project-families all inheritance search, do it as a single query # Load all variants, do not skip pages num_loaded += self.previous_search_results.get('duplicate_doc_count', 0) if num_loaded >= (page - 1) * num_results: start_index = num_loaded - else: - start_index = 0 return True, {'page': page, 'num_results': num_results, 'start_index': start_index, 'deduplicate': True} def _execute_single_search(self, page=1, num_results=100, start_index=None, deduplicate=False, **kwargs): @@ -301,8 +297,6 @@ def _get_paginated_searches(self, index_name, page=1, num_results=100, start_ind search = search.index(index_name.split(',')) if search.aggs.to_dict(): - # For compound het search get results from aggregation instead of top level hits - search = search[:1] log_messages.append('Loading {}s for {}'.format(self.AGGREGATION_NAME, index_name)) else: end_index = page * num_results @@ -340,8 +334,6 @@ def _parse_es_sort(sort, sort_config): if sort in {'Infinity', '-Infinity', None}: # ES returns these values for sort when a sort field is missing, using the correct value for the given direction sort = maxsize - elif hasattr(sort_config, 'values') and any(cfg.get('order') == 'desc' for cfg in sort_config.values()): - sort = float(sort) * -1 return sort diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 52c092f158..503136fa2a 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -1183,8 +1183,6 @@ def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=Non '_type': 'structural_variant' if SV_INDEX_NAME in index or SV_WGS_INDEX_NAME in index else 'variant', }) matched_queries = hit.pop('matched_queries') - if include_matched_queries: - hit['matched_queries'] = [] if sort or increment_sort: sort_key = sort[0] if sort else 'xpos' @@ -1223,35 +1221,6 @@ def create_mock_response(search, index=INDEX_NAME): except (KeyError, TypeError): pass - if search.get('aggs'): - index_vars = COMPOUND_HET_INDEX_VARIANTS.get(index, {}) - buckets = [{'key': gene_id, 'doc_count': 3} for gene_id in ['ENSG00000135953', 'ENSG00000228198']] - if search['aggs']['genes']['aggs'].get('vars_by_gene'): - for bucket in buckets: - bucket['vars_by_gene'] = { - 'hits': { - 'hits': mock_hits(index_vars.get(bucket['key'], ES_VARIANTS), increment_sort=True, index=index) - }} - else: - for bucket in buckets: - doc_count = 0 - for sample_field in ['samples', 'samples_num_alt_1', 'samples_num_alt_2']: - gene_samples = defaultdict(int) - for var in index_vars.get(bucket['key'], ES_VARIANTS): - for sample in var['_source'].get(sample_field, []): - gene_samples[sample] += 1 - bucket[sample_field] = {'buckets': [{'key': k, 'doc_count': v} for k, v in gene_samples.items()]} - doc_count += sum(gene_samples.values()) - bucket['doc_count'] = doc_count - - response_dict['aggregations'] = {'genes': {'buckets': buckets}} - - if len(response_dict['hits']['hits']) == 0: - response_dict['hits']['total']['value'] = 0 - elif gene_ids_filters == {'ENSG00000186092'}: - response_dict['hits']['total']['value'] = 1 - - return response_dict def get_indices_from_url(url): @@ -1271,7 +1240,7 @@ def get_msearch_callback(request): body = parse_msearch_body(request.body) response = { 'responses': [ - create_mock_response(exec_search, index=','.join(body[i-1]['index'])) + create_mock_response(exec_search, index=','.join(body[i - 1]['index'])) for i, exec_search in enumerate(body) if not exec_search.get('index')] } return 200, {}, json.dumps(response) @@ -1325,9 +1294,6 @@ def assertSameSearch(self, executed_search, expected_search_params, expected_sou } } - if expected_search_params.get('query'): - expected_search['query']['bool']['must'] = expected_search_params['query'] - if not expected_search_params.get('unsorted'): expected_search['sort'] = expected_search_params.get('sort') or ['xpos', 'variantId'] @@ -1338,12 +1304,6 @@ def assertSameSearch(self, executed_search, expected_search_params, expected_sou 'top_hits': {'sort': expected_search['sort'], '_source': mock.ANY, 'size': 100} } }}} - elif expected_search_params.get('gene_count_aggs'): - expected_search['aggs'] = {'genes': { - 'terms': {'field': 'mainTranscript_gene_id', 'size': 1001}, - 'aggs': expected_search_params['gene_count_aggs'] - }} - del expected_search['sort'] else: expected_search['_source'] = mock.ANY self.assertDictEqual(executed_search, expected_search) From edeea2ebd7e772c86017c296b97316f2ac3c2e1d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Feb 2026 15:52:39 -0500 Subject: [PATCH 028/252] fix test --- seqr/views/apis/summary_data_api_tests.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 97d5506b68..32bc54acc2 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -30,7 +30,7 @@ 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', } SAVED_VARIANT_RESPONSE_KEYS = { - *VARIANT_TAG_RESPONSE_KEYS, 'projectsByGuid', 'locusListsByGuid', 'genesById', + *VARIANT_TAG_RESPONSE_KEYS, 'projectsByGuid', 'locusListsByGuid', 'genesById', 'totalSampleCounts', 'individualsByGuid', 'familiesByGuid', 'familyNotesByGuid', 'mmeSubmissionsByGuid', 'transcriptsById', } @@ -394,7 +394,7 @@ def test_saved_variants_page(self): response = self.client.get(gene_url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) expected_variant_guids = { 'SV0000001_2103343353_r0390_100', 'SV0000007_prefix_19107_DEL_r00', 'SV0000006_1248367227_r0003_tes', } @@ -419,7 +419,7 @@ def test_saved_variants_page(self): response = self.client.get(gene_url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json['savedVariantsByGuid'].keys()), expected_variant_guids) multi_tag_url = reverse(saved_variants_page, args=['Review;Tier 1 - Novel gene and phenotype']) @@ -848,7 +848,6 @@ class LocalSummaryDataAPITest(AuthenticationTestCase, SummaryDataAPITest): NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = ['NA21234', 'NA21987', 'NA21654'] HAS_AIRTABLE = False - SAVED_VARIANT_RESPONSE_KEYS = SAVED_VARIANT_RESPONSE_KEYS def _test_metadata_airtable_responses(self, include_airtable_url, expected_individuals): # Returns successfully without airtable data when disabled @@ -872,7 +871,6 @@ class AnvilSummaryDataAPITest(AnvilAuthenticationTestCase, SummaryDataAPITest): NUM_MANAGER_SUBMISSIONS = 4 ADDITIONAL_SAMPLES = [] HAS_AIRTABLE = True - SAVED_VARIANT_RESPONSE_KEYS = {*SAVED_VARIANT_RESPONSE_KEYS, 'totalSampleCounts'} def test_mme_details(self, *args): super(AnvilSummaryDataAPITest, self).test_mme_details(*args) From ac7faa22de33ded343d5fd3f54c1c6daae875111 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 10:46:08 -0500 Subject: [PATCH 029/252] fix test --- seqr/views/apis/report_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index a5efdf6ed6..e8a4751eca 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1506,7 +1506,7 @@ def test_variant_metadata(self): class LocalReportAPITest(AuthenticationTestCase, ReportAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'report_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'report_variants', 'clickhouse_saved_variants'] ADDITIONAL_FAMILIES = ['F000014_14'] ADDITIONAL_FINDINGS = ['NA21234_1_248367227', 'NA21234_MT_14783', 'NA21234_1_249045487_DEL'] HAS_PM_OVERRIDE = True From 23e41420cd9a26ef17cd62e5852fad4a37547624 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 10:48:13 -0500 Subject: [PATCH 030/252] fix test --- seqr/views/apis/project_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 824632c7a1..951c2a33fa 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -1055,7 +1055,7 @@ def _get_expected_read_file_subprocess_calls(mock_subprocess, file_name, additio # Tests for AnVIL access disabled class LocalProjectAPITest(AuthenticationTestCase, ProjectAPITest): - fixtures = ['users', '1kg_project', 'reference_data'] + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] PROJECT_COLLABORATORS = BASE_COLLABORATORS CREATE_PROJECT_JSON = BASE_CREATE_PROJECT_JSON PROJECT_COLLABORATOR_GROUPS = [{'name': 'analysts', 'hasViewPermissions': True, 'hasEditPermissions': True}] From 00919e1ebe795fe10aecdd19f6de37b6bb7a50d8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 10:54:49 -0500 Subject: [PATCH 031/252] fix test --- seqr/views/apis/individual_api_tests.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 843f80b255..015b3fd2a2 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -364,13 +364,18 @@ def test_delete_individuals(self, mock_pm_group): })) self.assertEqual(response.status_code, 400) + response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ + 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] + })) + self.assertEqual(response.status_code, 400) + self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) + Sample.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000173_na21987']).update(is_active=False) + # send valid requests response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] })) - self._assert_expected_delete_individuals(response, mock_pm_group) - def _assert_expected_delete_individuals(self, response, mock_pm_group): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), {'individualsByGuid', 'familiesByGuid'}) @@ -613,7 +618,8 @@ def test_individuals_table_handler(self): self.assertEqual(response.status_code, 200) def _assert_expected_reload_calls(self, project_guid): - self.assertEqual(len(responses.calls), 0) + self.assertEqual(len(responses.calls), 1) + self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @mock.patch('seqr.views.utils.pedigree_info_utils.NO_VALIDATE_MANIFEST_PROJECT_CATEGORIES') @@ -1454,11 +1460,3 @@ def _mock_subprocess(self, command, **kwargs): else: self.mock_subprocess.stdout.__iter__.return_value = self.gs_files[file_name] return self.mock_subprocess - - def _assert_expected_delete_individuals(self, response, mock_pm_group): - self.assertEqual(response.status_code, 400) - self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) - - def _assert_expected_reload_calls(self, project_guid): - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) From 3ae6bb165d6d9e2574dfac2b697749837208a01c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:00:16 -0500 Subject: [PATCH 032/252] fix tests --- .../tests/check_for_new_samples_from_pipeline_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index bb856d7dd3..3898a7638c 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -722,7 +722,7 @@ def test_command(self): }) -class LocalCheckNewSamplesTest(DifferentDbTransactionSupportMixin, AuthenticationTestCase, CheckNewSamplesTest): +class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] databases = '__all__' From d92b7290b765aaaf16a086908c4de1cdb2783a81 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:12:25 -0500 Subject: [PATCH 033/252] remove clickhouse only decorator --- .../commands/tag_seqr_prioritized_variants.py | 3 +- seqr/utils/search/search_utils_tests.py | 10 +---- seqr/utils/search/utils.py | 9 ----- seqr/views/apis/data_manager_api.py | 4 +- seqr/views/apis/data_manager_api_tests.py | 37 ++++++++----------- 5 files changed, 18 insertions(+), 45 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 2201ff19d7..7a9ea32d7f 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -17,7 +17,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.gene_utils import get_genes from seqr.utils.search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO -from seqr.utils.search.utils import clickhouse_only, get_search_samples, COMPOUND_HET +from seqr.utils.search.utils import get_search_samples, COMPOUND_HET from seqr.views.utils.orm_to_json_utils import SEQR_TAG_TYPE from seqr.views.utils.variant_utils import bulk_create_tagged_variants, gene_ids_annotated_queryset from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL @@ -410,7 +410,6 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('project') - @clickhouse_only def handle(self, *args, **options): family_guid_map = {} family_name_map = {} diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 42703de4ee..363d52d353 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -65,6 +65,7 @@ def set_up(self): ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) + @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') def test_variant_lookup(self, mock_variant_lookup): mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] variants = variant_lookup(self.user, '1-10439-AC-A', '38', affected_only=True) @@ -565,11 +566,6 @@ def setUp(self): def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) - def test_variant_lookup(self, *args, **kwargs): - with self.assertRaises(ValueError) as cm: - super().test_variant_lookup(mock.MagicMock()) - self.assertEqual(str(cm.exception), 'variant_lookup is disabled without the clickhouse backend') - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') def test_get_single_variant(self, mock_get_variants_for_ids): mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] @@ -742,7 +738,3 @@ def test_cached_get_variant_query_gene_counts(self): 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, }) - - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_variant_lookup(self, mock_call): - super().test_variant_lookup(mock_call) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f10eb0ec09..9f5de481a3 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -60,14 +60,6 @@ def _wrapped(*args, **kwargs): return _wrapped -def clickhouse_only(func): - def _wrapped(*args, **kwargs): - if es_backend_enabled(): - raise ValueError(f'{func.__name__} is disabled without the clickhouse backend') - return func(*args, **kwargs) - return _wrapped - - def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func @@ -153,7 +145,6 @@ def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome ) -@clickhouse_only def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): cache_fields = ['variant_lookup_results', variant_id, genome_version] if affected_only: diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 188c40d622..345d0bbf7d 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -15,7 +15,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index -from seqr.utils.search.utils import clickhouse_only, es_only, InvalidSearchException +from seqr.utils.search.utils import es_only, InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list @@ -404,7 +404,6 @@ def _get_valid_search_individuals(project, airtable_samples, vcf_samples, datase @data_manager_required -@clickhouse_only def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') @@ -422,7 +421,6 @@ def trigger_delete_project(request): @data_manager_required -@clickhouse_only def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 0e6adab699..7530e20c2b 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1513,11 +1513,22 @@ def test_trigger_delete_family(self): Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') response = self.client.post(url, content_type='application/json', data=json.dumps({'family': 'F000002_2'})) - self._assert_expected_delete_family(response) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': [ + 'Disabled search for 7 samples in the following 1 families: 2', + 'Triggered delete family data', + ], + }) - def _assert_expected_delete_family(self, response): - self.assertEqual(response.status_code, 500) - self.assertDictEqual(response.json(), {'error': 'trigger_delete_family is disabled without the clickhouse backend'}) + family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) + self.assertEqual(family_samples.count(), 0) + + self.assertEqual(len(responses.calls), 1) + self.assertDictEqual(json.loads(responses.calls[-1].request.body), { + 'project_guid': 'R0001_1kg', + 'family_guids': ['F000002_2'], + }) class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): @@ -1969,24 +1980,6 @@ def _test_validate_dataset_type(self, url): self._set_file_not_found() - def _assert_expected_delete_family(self, response): - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'info': [ - 'Disabled search for 7 samples in the following 1 families: 2', - 'Triggered delete family data', - ], - }) - - family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) - self.assertEqual(family_samples.count(),0) - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[-1].request.body), { - 'project_guid': 'R0001_1kg', - 'family_guids': ['F000002_2'], - }) - def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', From 3ea285c8021a4c7f7aa62bd9f8f7885510b5bb76 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:17:33 -0500 Subject: [PATCH 034/252] remove es only manage commands --- .../commands/deactivate_project_search.py | 23 ---- .../commands/reload_saved_variant_json.py | 34 ------ .../tests/deactivate_project_search_tests.py | 64 ----------- .../tests/reload_saved_variant_json_tests.py | 100 ------------------ seqr/views/utils/variant_utils.py | 1 + 5 files changed, 1 insertion(+), 221 deletions(-) delete mode 100644 seqr/management/commands/deactivate_project_search.py delete mode 100644 seqr/management/commands/reload_saved_variant_json.py delete mode 100644 seqr/management/tests/deactivate_project_search_tests.py delete mode 100644 seqr/management/tests/reload_saved_variant_json_tests.py diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py deleted file mode 100644 index df48683bea..0000000000 --- a/seqr/management/commands/deactivate_project_search.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - -from seqr.models import Project, Sample -from seqr.utils.search.utils import es_only - -import logging -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument('project') - - @es_only - def handle(self, *args, **options): - project = Project.objects.get(guid=options['project']) - - if input(f'Are you sure you want to deactivate search for {project.name} (y/n)? ') != 'y': - raise CommandError('Error: user did not confirm') - - updated = Sample.bulk_update(user=None, update_json={'is_active': False}, individual__family__project=project) - - logger.info(f'Deactivated {len(updated)} samples') diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py deleted file mode 100644 index 7c9f14efc7..0000000000 --- a/seqr/management/commands/reload_saved_variant_json.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging -from django.core.management.base import BaseCommand -from django.db.models.query_utils import Q -from seqr.models import Project -from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import es_only -from seqr.views.utils.variant_utils import update_projects_saved_variant_json - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = 'Transfer projects to the new seqr schema' - - def add_arguments(self, parser): - parser.add_argument('projects', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.') - parser.add_argument('--family-guid', help='optional family to reload variants for') - - @es_only - def handle(self, *args, **options): - projects_to_process = options['projects'] - family_guid = options['family_guid'] - - if projects_to_process: - projects = Project.objects.filter(Q(name__in=projects_to_process) | Q(guid__in=projects_to_process)) - logging.info("Processing %s projects" % len(projects)) - else: - projects = Project.objects.all() - logging.info("Processing all %s projects" % len(projects)) - - family_ids = [family_guid] if family_guid else None - project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, update_function=update_project_saved_variant_json) - logger.info("Done") diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py deleted file mode 100644 index 0f83df98d6..0000000000 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -import mock -import responses - -from django.core.exceptions import ObjectDoesNotExist -from django.core.management import call_command -from django.core.management.base import CommandError - -from seqr.models import Sample, Project -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase - -PROJECT_GUID = 'R0001_1kg' -VARIANT_ID = '21-3343353-GAGA-G' - - -class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - @responses.activate - @mock.patch('seqr.management.commands.deactivate_project_search.input') - def test_command(self, mock_input): - mock_input.return_value = 'n' - - # Test invalid project - with self.assertRaises(ObjectDoesNotExist): - call_command('deactivate_project_search', 'foo') - - # Test user did not confirm. - with self.assertRaises(CommandError) as e: - call_command('deactivate_project_search', PROJECT_GUID) - self.assertEqual(str(e.exception), 'Error: user did not confirm') - - # Test success - Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') - mock_input.return_value = 'y' - self.reset_logs() - self.maxDiff = None - call_command('deactivate_project_search', PROJECT_GUID) - self.assert_json_logs(user=None, expected=[ - ('update 11 Samples', {'dbUpdate': { - 'dbEntity': 'Sample', - 'entityIds': mock.ANY, - 'updateFields': ['is_active'], - 'updateType': 'bulk_update'}, - }), - ('Deactivated 11 samples', None), - ]) - - active_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True) - self.assertEqual(active_samples.count(), 0) - - # Re-running has no effect - self.reset_logs() - call_command('deactivate_project_search', PROJECT_GUID) - self.assert_json_logs(user=None, expected=[('Deactivated 0 samples', None)]) - - -class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] - - def test_command(self): - with self.assertRaises(ValueError) as e: - call_command('deactivate_project_search', 'foo') - self.assertEqual(str(e.exception), 'handle is disabled without the elasticsearch backend') diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py deleted file mode 100644 index 51b9f3e5c5..0000000000 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ /dev/null @@ -1,100 +0,0 @@ -#-*- coding: utf-8 -*- -import mock - -from django.core.management import call_command -from django.test import TestCase - -PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' -PROJECT_GUID = 'R0001_1kg' -FAMILY_GUID = 'F000001_1' - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -class ReloadSavedVariantJsonTest(TestCase): - fixtures = ['users', '1kg_project'] - - @mock.patch('seqr.views.utils.variant_utils.logger') - @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') - def test_with_param_command(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ - [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples}) if samples else ['F000012_12']} - for variant_id in variant_ids] - - # Test with a specific project and a family id. - call_command('reload_saved_variant_json', - PROJECT_NAME, - '--family-guid={}'.format(FAMILY_GUID)) - - mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A','21-3343353-GAGA-G'], None) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) - - logger_info_calls = [ - mock.call('Updated 2 variants in 1 families for project 1kg project n\xe5me with uni\xe7\xf8de'), - mock.call('Reload Summary: '), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Updated 2 variants') - ] - mock_logger.info.assert_has_calls(logger_info_calls) - mock_get_variants.reset_mock() - mock_logger.reset_mock() - - # Test for all projects and no specific family ids - call_command('reload_saved_variant_json') - - self.assertEqual(mock_get_variants.call_count, 3) - mock_get_variants.assert_has_calls([ - mock.call( - mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], None, - ), - mock.call(mock.ANY, '37', ['1-248367227-TC-T', 'prefix_19107_DEL'], None), - mock.call(mock.ANY, '38', ['1-248367227-TC-T'], None) - ]) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), { - 129, 130, 132, 133, 134, 145, 146, 148, 149, - }) - self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), set()) - self.assertSetEqual(set(mock_get_variants.call_args_list[2].args[0].values_list('id', flat=True)), {147, 173, 174}) - - logger_info_calls = [ - mock.call('Reloading saved variants in 4 projects'), - mock.call('Updated 3 variants for project 1kg project n\xe5me with uni\xe7\xf8de'), - mock.call('Updated 2 variants for project Test Reprocessed Project'), - mock.call('Updated 1 variants for project Non-Analyst Project'), - mock.call('Reload Summary: '), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Updated 3 variants'), - mock.call(' Test Reprocessed Project: Updated 2 variants'), - mock.call(' Non-Analyst Project: Updated 1 variants'), - mock.call('Skipped the following 1 project with no saved variants: Empty Project'), - ] - mock_logger.info.assert_has_calls(logger_info_calls) - mock_get_variants.reset_mock() - mock_logger.reset_mock() - - # Test with an exception. - mock_get_variants.side_effect = Exception("Database error.") - call_command('reload_saved_variant_json', - PROJECT_GUID, - '--family-guid={}'.format(FAMILY_GUID)) - - mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A', '21-3343353-GAGA-G'], None) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) - - logger_info_calls = [ - mock.call('Reload Summary: '), - mock.call('1 failed projects'), - mock.call(' 1kg project n\xe5me with uni\xe7\xf8de: Database error.') - ] - mock_logger.info.assert_has_calls(logger_info_calls) - - mock_logger.error.assert_called_with('Error reloading variants in 1kg project n\xe5me with uni\xe7\xf8de: Database error.') - - -class ClickhouseReloadSavedVariantJsonTest(TestCase): - - fixtures = ['users', '1kg_project'] - - def test_command(self): - with self.assertRaises(ValueError) as ce: - call_command('reload_saved_variant_json', PROJECT_NAME) - self.assertEqual(str(ce.exception), 'handle is disabled without the elasticsearch backend') - - diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index f1be32a486..11161f2a58 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -35,6 +35,7 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 +# TODO def update_projects_saved_variant_json(projects, update_function, **kwargs): success = {} skipped = {} From 6bea97860b17f35316781e0da16516caeb364957 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:30:16 -0500 Subject: [PATCH 035/252] clean up shared utility function --- .../check_for_new_samples_from_pipeline.py | 32 ++--------------- seqr/views/utils/variant_utils.py | 34 +++++++++++++++---- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index d9ebd0bde3..92575fee99 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -7,9 +7,8 @@ import logging import re -from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, SavedVariant, Project, Individual +from seqr.models import Family, Sample, Project, Individual from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status @@ -17,8 +16,7 @@ from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil -from seqr.views.utils.variant_utils import reset_cached_search_results, update_projects_saved_variant_json, \ - get_saved_variants +from seqr.views.utils.variant_utils import reset_cached_search_results, update_projects_saved_variant_json from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \ SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL @@ -258,9 +256,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - update_projects_saved_variant_json([ - (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) + update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type, updated_samples) @classmethod def _is_internal_project(cls, project): @@ -394,28 +390,6 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) fields=['filter_flags', 'pop_platform_filters', 'population'], ) - @classmethod - def _update_project_saved_variant_genotypes(cls, project_id, genome_version, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): - updates = {} - for family_guid in family_guids: - variant_models_by_key = { - v.key: v for v in get_saved_variants(genome_version, project_id, [family_guid], clickhouse_dataset_type=clickhouse_dataset_type) - } - if not variant_models_by_key: - continue - variants = [] - genotypes_by_key = get_clickhouse_genotypes( - project_guid, [family_guid], genome_version, clickhouse_dataset_type, variant_models_by_key.keys(), samples, - ) - for key, genotypes in genotypes_by_key.items(): - variant = variant_models_by_key[key] - variant.genotypes = genotypes - variants.append(variant) - logger.info(f'Reloading genotypes for {len(variants)} {clickhouse_dataset_type} variants in family {family_guid}') - SavedVariant.bulk_update_models(None, variants, ['genotypes']) - updates.update({v.id: v for v in variants}) - return updates - update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 11161f2a58..9e0e5fe294 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -35,16 +35,15 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -# TODO -def update_projects_saved_variant_json(projects, update_function, **kwargs): +def update_projects_saved_variant_json(families_by_project, dataset_type, samples): success = {} skipped = {} error = {} - logger.info(f'Reloading saved variants in {len(projects)} projects') - for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): + logger.info(f'Reloading saved variants in {len(families_by_project)} projects') + for project, family_guids in tqdm(families_by_project.items(), unit=' project'): + project_name = project.name try: - updated_saved_variants = update_function( - project_id, genome_version, family_guids=family_guids, project_guid=project_guid, **kwargs) + updated_saved_variants = _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) if updated_saved_variants is None: skipped[project_name] = True else: @@ -69,6 +68,29 @@ def update_projects_saved_variant_json(projects, update_function, **kwargs): logger.info(f' {k}: {v}') +def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples): + updates = {} + for family_guid in family_guids: + variant_models_by_key = { + # TODO + v.key: v for v in get_saved_variants(project.genome_version, project.id, [family_guid], clickhouse_dataset_type=dataset_type) + } + if not variant_models_by_key: + continue + variants = [] + genotypes_by_key = get_clickhouse_genotypes( + project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), samples, + ) + for key, genotypes in genotypes_by_key.items(): + variant = variant_models_by_key[key] + variant.genotypes = genotypes + variants.append(variant) + logger.info(f'Reloading genotypes for {len(variants)} {dataset_type} variants in family {family_guid}') + SavedVariant.bulk_update_models(None, variants, ['genotypes']) + updates.update({v.id: v for v in variants}) + return updates + + def get_saved_variants(genome_version, project_id=None, family_guids=None, clickhouse_dataset_type=None): saved_variants = SavedVariant.objects.filter( Q(saved_variant_json__genomeVersion__isnull=True) | From b5ca1993d11bb930d83e598f1c398011dd3d2a25 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:39:59 -0500 Subject: [PATCH 036/252] clean up reload endpoint --- seqr/urls.py | 4 +- seqr/utils/search/elasticsearch/es_utils.py | 34 +--------------- seqr/views/apis/saved_variant_api.py | 19 +-------- seqr/views/apis/saved_variant_api_tests.py | 45 +-------------------- seqr/views/utils/variant_utils.py | 20 ++------- 5 files changed, 9 insertions(+), 113 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index 8b21f25637..54488c7043 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -59,8 +59,7 @@ create_variant_note_handler, \ update_variant_note_handler, \ delete_variant_note_handler, \ - update_variant_main_transcript, \ - update_saved_variant_json + update_variant_main_transcript from seqr.views.apis.dashboard_api import dashboard_page_data @@ -254,7 +253,6 @@ 'project/(?P[^/]+)/dynamic_analysis_groups/create': update_dynamic_analysis_group_handler, 'project/(?P[^/]+)/dynamic_analysis_groups/(?P[^/]+)/update': update_dynamic_analysis_group_handler, 'project/(?P[^/]+)/dynamic_analysis_groups/(?P[^/]+)/delete': delete_dynamic_analysis_group_handler, - 'project/(?P[^/]+)/update_saved_variant_json': update_saved_variant_json, 'project/(?P[^/]+)/add_workspace_data': add_workspace_data, 'search/variant/(?P[^/]+)': query_single_variant_handler, diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 5fe4a68106..c98c56d264 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -4,7 +4,7 @@ import elasticsearch_dsl from urllib3.connectionpool import connection_from_url -from seqr.models import Sample, SavedVariant +from seqr.models import Sample from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch @@ -334,35 +334,3 @@ def process_es_previously_loaded_gene_aggs(previous_search_results): gene_aggs[gene_id]['families'][family_guid] += len(variants) return gene_aggs - -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): - from seqr.views.utils.variant_utils import get_saved_variants - saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') - - if not saved_variants: - return None - - family_ids = set() - variant_ids = set() - saved_variants_map = {} - for v in saved_variants: - family_ids.add(v.family_id) - variant_ids.add(v.variant_id) - saved_variants_map[(v.variant_id, v.family.guid)] = v - - variant_ids = sorted(variant_ids) - samples = Sample.objects.filter(individual__family_id__in=family_ids, is_active=True) - variants_json = [] - for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: - variants_json += get_es_variants_for_variant_ids(samples, genome_version, sub_var_ids, user) - - updated_saved_variants = {} - for var in variants_json: - for family_guid in var['familyGuids']: - saved_variant = saved_variants_map.get((var['variantId'], family_guid)) - if saved_variant: - saved_variant.saved_variant_json = var - updated_saved_variants[saved_variant.guid] = saved_variant - SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) - - return updated_saved_variants diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 6fa1fd5b29..eb4a797b51 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -8,8 +8,7 @@ from clickhouse_search.search import get_transcripts_queryset from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project -from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import backend_specific_call, es_only +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -17,7 +16,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import reset_cached_search_results, get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json logger = logging.getLogger(__name__) @@ -312,20 +311,6 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari model.saved_variants.set(saved_variants) -@login_and_policies_required -@es_only -def update_saved_variant_json(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) - reset_cached_search_results(project) - try: - updated_saved_variant_guids = update_project_saved_variant_json(project.id, project.genome_version, user=request.user) - except Exception as e: - logger.error('Unable to reset saved variant json for {}: {}'.format(project_guid, e)) - updated_saved_variant_guids = [] - - return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids or []}) - - @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 5a51ff12f5..76b480a1f8 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -7,7 +7,7 @@ from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Sample, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ - update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, update_saved_variant_json, \ + update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ @@ -975,42 +975,6 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) - @mock.patch('seqr.utils.search.elasticsearch.es_utils.MAX_VARIANTS_FETCH', 2) - @mock.patch('seqr.views.apis.saved_variant_api.logger') - @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') - def test_update_saved_variant_json(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ - [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples})} - for variant_id in variant_ids] - - url = reverse(update_saved_variant_json, args=['R0001_1kg']) - self.check_manager_login(url) - - response = self.client.post(url) - - self.assertEqual(response.status_code, 200) - - self.assertDictEqual( - response.json(), - {'SV0000002_1248367227_r0390_100': None, 'SV0000001_2103343353_r0390_100': None, - 'SV0059956_11560662_f019313_1': None} - ) - - mock_get_variants.assert_has_calls([ - mock.call(mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A'], self.manager_user), - mock.call(mock.ANY, '37', ['21-3343353-GAGA-G'], self.manager_user), - ]) - samples = set(Sample.objects.filter(individual__family_id__in=[1, 2], is_active=True).values_list('id', flat=True)) - self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), samples) - self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), samples) - mock_logger.error.assert_not_called() - - # Test handles update error - mock_get_variants.side_effect = Exception('Unable to fetch variants') - response = self.client.post(url) - self.assertEqual(response.status_code, 200) - mock_logger.error.assert_called_with('Unable to reset saved variant json for R0001_1kg: Unable to fetch variants') - def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) @@ -1130,13 +1094,6 @@ def test_update_compound_hets_variant_functional_data(self): super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_functional_data() assert_no_list_ws_has_al(self, 3) - def test_update_saved_variant_json(self, *args): - url = reverse(update_saved_variant_json, args=['R0001_1kg']) - self.check_require_login(url) - response = self.client.post(url) - self.assertEqual(response.status_code, 500) - self.assertDictEqual(response.json(), {'error': 'update_saved_variant_json is disabled without the elasticsearch backend'}) - def test_update_variant_main_transcript(self): super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() assert_no_list_ws_has_al(self, 2) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 9e0e5fe294..496f77d381 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -72,8 +72,10 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, updates = {} for family_guid in family_guids: variant_models_by_key = { - # TODO - v.key: v for v in get_saved_variants(project.genome_version, project.id, [family_guid], clickhouse_dataset_type=dataset_type) + v.key: v for v in SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid).filter( + Q(saved_variant_json__genomeVersion__isnull=True) | + Q(saved_variant_json__genomeVersion=project.genome_version.replace('GRCh', '')) + ) } if not variant_models_by_key: continue @@ -91,20 +93,6 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, return updates -def get_saved_variants(genome_version, project_id=None, family_guids=None, clickhouse_dataset_type=None): - saved_variants = SavedVariant.objects.filter( - Q(saved_variant_json__genomeVersion__isnull=True) | - Q(saved_variant_json__genomeVersion=genome_version.replace('GRCh', '')) - ) - if project_id: - saved_variants = saved_variants.filter(family__project_id=project_id) - if family_guids: - saved_variants = saved_variants.filter(family__guid__in=family_guids) - if clickhouse_dataset_type: - saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) - return saved_variants - - def parse_saved_variant_json(variant_json, family_id, variant_id=None,): if 'xpos' not in variant_json: variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos']) From 01fabcee467b98cdc54bda5cb073a209b1c2fbc0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:48:33 -0500 Subject: [PATCH 037/252] remove deprecated datasets api --- seqr/urls.py | 2 - seqr/utils/search/elasticsearch/es_utils.py | 37 -- seqr/views/apis/dataset_api.py | 94 ---- seqr/views/apis/dataset_api_tests.py | 467 -------------------- seqr/views/utils/dataset_utils.py | 15 +- 5 files changed, 1 insertion(+), 614 deletions(-) delete mode 100644 seqr/views/apis/dataset_api.py delete mode 100644 seqr/views/apis/dataset_api_tests.py diff --git a/seqr/urls.py b/seqr/urls.py index 54488c7043..a6196d2259 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -5,7 +5,6 @@ """ from seqr.views.react_app import main_app, no_login_main_app from seqr.views.status import status_view -from seqr.views.apis.dataset_api import add_variants_dataset_handler from settings import ENABLE_DJANGO_DEBUG_TOOLBAR, MEDIA_ROOT, API_LOGIN_REQUIRED_URL, LOGIN_URL, DEBUG, \ API_POLICY_REQUIRED_URL from django.conf.urls import include @@ -240,7 +239,6 @@ 'project/(?P[^/]+)/upload_individuals_table': receive_individuals_table_handler, 'project/(?P[^/]+)/save_individuals_table/(?P[^/]+)': save_individuals_table_handler, 'project/(?P[^/]+)/upload_igv_dataset': receive_igv_table_handler, - 'project/(?P[^/]+)/add_dataset/variants': add_variants_dataset_handler, 'project/(?P[^/]+)/update_rna_seq': update_project_rna_seq, 'project/(?P[^/]+)/igv_track/(?P.+)': fetch_igv_track, diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index c98c56d264..a9b9bff984 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -107,43 +107,6 @@ def get_index_metadata(index_name, client, include_fields=False, use_cache=True) return index_metadata -def validate_es_index_metadata_and_get_samples(request_json, project): - if 'elasticsearchIndex' not in request_json: - raise ValueError('request must contain field: "elasticsearchIndex"') - - elasticsearch_index = request_json['elasticsearchIndex'].strip() - kwargs = { - 'project': project, - 'dataset_type': request_json['datasetType'], - 'genome_version': request_json.get('genomeVersion'), - } - - es_client = get_es_client() - - all_index_metadata = get_index_metadata(elasticsearch_index, es_client, include_fields=True) - if elasticsearch_index in all_index_metadata: - index_metadata = all_index_metadata.get(elasticsearch_index) - _validate_index_metadata(index_metadata, elasticsearch_index, **kwargs) - sample_field = _get_samples_field(index_metadata) - sample_type = index_metadata['sampleType'] - else: - # Aliases return the mapping for all indices in the alias - metadatas = list(all_index_metadata.values()) - sample_field = _get_samples_field(metadatas[0]) - sample_type = metadatas[0]['sampleType'] - for metadata in metadatas[1:]: - _validate_index_metadata(metadata, elasticsearch_index, **kwargs) - if sample_field != _get_samples_field(metadata): - raise ValueError('Found mismatched sample fields for indices in alias') - if sample_type != metadata['sampleType']: - raise ValueError('Found mismatched sample types for indices in alias') - - sample_ids = _get_es_sample_ids(elasticsearch_index, sample_field, es_client) - sample_data = {'elasticsearch_index': elasticsearch_index} - - return sample_ids, sample_type, sample_data - - def _get_es_sample_ids(elasticsearch_index, sample_field, es_client): s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index) s = s.params(size=0) diff --git a/seqr/views/apis/dataset_api.py b/seqr/views/apis/dataset_api.py deleted file mode 100644 index 360d861736..0000000000 --- a/seqr/views/apis/dataset_api.py +++ /dev/null @@ -1,94 +0,0 @@ -import json - -from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded -from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples -from seqr.utils.search.utils import es_only -from seqr.models import Individual, Family, Sample -from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file -from seqr.views.utils.json_utils import create_json_response -from seqr.views.utils.orm_to_json_utils import get_json_for_samples -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, data_manager_required - - -@data_manager_required -@es_only -def add_variants_dataset_handler(request, project_guid): - """Create or update samples for the given variant dataset - - Args: - request: Django request object - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'elasticsearchIndex': (required) - 'ignoreExtraSamplesInCallset': - 'mappingFilePath': - } - - Response body - will contain the following structure: - - """ - - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) - request_json = json.loads(request.body) - - try: - inactivated_sample_guids, updated_family_guids, updated_samples = _add_new_es_search_samples( - request_json, project, request.user, notify=True) - except ValueError as e: - return create_json_response({'errors': [str(e)]}, status=400) - - response_json = _get_samples_json(updated_samples, inactivated_sample_guids, project_guid) - response_json['familiesByGuid'] = {family_guid: {'analysisStatus': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS} - for family_guid in updated_family_guids} - - return create_json_response(response_json) - - -def _add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): - dataset_type = request_json.get('datasetType') - if dataset_type not in Sample.DATASET_TYPE_LOOKUP: - raise ValueError(f'Invalid dataset type "{dataset_type}"') - - sample_ids, sample_type, sample_data = validate_es_index_metadata_and_get_samples(request_json, project) - if not sample_ids: - raise ValueError('No samples found. Make sure the specified caller type is correct') - - sample_id_to_individual_id_mapping = load_mapping_file( - request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} - ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') - sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] - new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( - projects=[project], - user=user, - sample_project_tuples=sample_project_tuples, - sample_data=sample_data, - sample_type=sample_type, - dataset_type=dataset_type, - expected_families=expected_families, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' - ) - - if notify: - basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id', flat=True)) - - return inactivated_sample_guids, updated_family_guids, updated_samples - - -def _get_samples_json(updated_samples, inactivated_sample_guids, project_guid): - updated_sample_json = get_json_for_samples(updated_samples, project_guid=project_guid) - sample_response = {sample_guid: {'isActive': False} for sample_guid in inactivated_sample_guids} - sample_response.update({s['sampleGuid']: s for s in updated_sample_json}) - response = { - 'samplesByGuid': sample_response - } - updated_individuals = {s['individualGuid'] for s in updated_sample_json} - if updated_individuals: - individuals = Individual.objects.filter(guid__in=updated_individuals).prefetch_related('sample_set') - response['individualsByGuid'] = { - ind.guid: {'sampleGuids': [s.guid for s in ind.sample_set.all()]} for ind in individuals - } - return response diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py deleted file mode 100644 index 02dccf0297..0000000000 --- a/seqr/views/apis/dataset_api_tests.py +++ /dev/null @@ -1,467 +0,0 @@ -import json -import mock -from copy import deepcopy -from datetime import datetime -from django.urls.base import reverse -from io import StringIO - -from seqr.models import Sample, Family -from seqr.views.apis.dataset_api import add_variants_dataset_handler -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses - -SEQR_URL = 'https://seqr.broadinstitute.org' -PROJECT_GUID = 'R0001_1kg' -NON_ANALYST_PROJECT_GUID = 'R0004_non_analyst_project' -INDEX_NAME = 'test_index' -SV_INDEX_NAME = 'test_new_sv_index' -NEW_SAMPLE_TYPE_INDEX_NAME = 'test_new_index' -ADD_DATASET_PAYLOAD = json.dumps({'elasticsearchIndex': INDEX_NAME, 'datasetType': 'SNV_INDEL'}) -MAPPING_PROPS_SAMPLES_NUM_ALT_1 = { - "samples_num_alt_1": {"type": "keyword"}, -} -MAPPING_PROPS_WITH_SAMPLES = { - "samples": {"type": "keyword"}, -} -MAPPING_JSON = { - INDEX_NAME: { - 'mappings': { - '_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, - "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1 - }}} -MAPPING_JSON_38 = deepcopy(MAPPING_JSON) -MAPPING_JSON_38[INDEX_NAME]['mappings']['_meta']['genomeVersion'] = '38' - -MOCK_REDIS = mock.MagicMock() -MOCK_OPEN = mock.MagicMock() -MOCK_FILE_ITER = MOCK_OPEN.return_value.__enter__.return_value.__iter__ - - -@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) -@mock.patch('seqr.utils.file_utils.open', MOCK_OPEN) -class DatasetAPITest(object): - - @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda path: True) - @mock.patch('seqr.models.random.randint') - @mock.patch('seqr.utils.communication_utils.send_html_email') - @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://seqr.broadinstitute.org/') - @urllib3_responses.activate - def test_add_variants_dataset(self, mock_send_email, mock_random): - url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) - self.check_data_manager_login(url) - - # Confirm test DB is as expected - existing_index_sample = Sample.objects.get(sample_id='NA19675') - self.assertEqual(existing_index_sample.elasticsearch_index, INDEX_NAME) - self.assertTrue(existing_index_sample.is_active) - existing_index_sample_guid = existing_index_sample.guid - existing_old_index_sample = Sample.objects.get(guid='S000130_na19678') - self.assertNotEqual(existing_old_index_sample.elasticsearch_index, INDEX_NAME) - self.assertTrue(existing_old_index_sample.is_active) - existing_old_index_sample_guid = existing_old_index_sample.guid - existing_sample = Sample.objects.get(sample_id='NA19679') - self.assertEqual(existing_sample.elasticsearch_index, INDEX_NAME) - self.assertFalse(existing_sample.is_active) - existing_sample_guid = existing_sample.guid - self.assertEqual(Sample.objects.filter(sample_id='NA19678_1').count(), 0) - self.assertEqual(Sample.objects.filter(sample_id='NA20878').count(), 0) - - mock_random.return_value = 98765432101234567890 - - urllib3_responses.add_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON) - urllib3_responses.add_json('/{}/_search?size=0'.format(INDEX_NAME), {'aggregations': { - 'sample_ids': {'buckets': [{'key': 'NA19675'}, {'key': 'NA19679'}, {'key': 'NA19678_1'}, {'key': 'NA20878'}]} - }}, method=urllib3_responses.POST) - MOCK_FILE_ITER.return_value = StringIO('NA19678_1,NA19678\n') - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - MOCK_OPEN.assert_called_with('mapping.csv', 'r') - MOCK_REDIS.get.assert_called_with('index_metadata__test_index') - MOCK_REDIS.set.assert_called_with('index_metadata__test_index', '{"test_index": {"sampleType": "WES", "genomeVersion": "37", "sourceFilePath": "test_data.vcf", "fields": {"samples_num_alt_1": "keyword"}}}') - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - - new_sample_guid = 'S98765432101234567890_na20878' - replaced_sample_guid = 'S98765432101234567890_na19678_' - self.assertSetEqual( - set(response_json['samplesByGuid'].keys()), - {existing_sample_guid, existing_old_index_sample_guid, replaced_sample_guid, new_sample_guid} - ) - self.assertDictEqual(response_json['individualsByGuid'], { - 'I000002_na19678': {'sampleGuids': mock.ANY}, - 'I000003_na19679': {'sampleGuids': mock.ANY}, - 'I000013_na20878': {'sampleGuids': [new_sample_guid]}, - }) - self.assertSetEqual( - set(response_json['individualsByGuid']['I000002_na19678']['sampleGuids']), - {replaced_sample_guid, existing_old_index_sample_guid} - ) - self.assertSetEqual( - set(response_json['individualsByGuid']['I000003_na19679']['sampleGuids']), - {existing_sample_guid} - ) - - self.assertDictEqual(response_json['familiesByGuid'], { - 'F000001_1': {'analysisStatus': 'I'}, - 'F000009_9': {'analysisStatus': 'I'}, - }) - updated_family = Family.objects.get(guid='F000001_1') - self.assertEqual(updated_family.analysis_status, 'I') - self.assertIsNone(updated_family.analysis_status_last_modified_date) - self.assertIsNone(updated_family.analysis_status_last_modified_by) - - updated_samples = [sample for sample_guid, sample in response_json['samplesByGuid'].items() if sample_guid != existing_old_index_sample_guid] - self.assertSetEqual( - {'WES'}, - {sample['sampleType'] for sample in updated_samples} - ) - self.assertSetEqual( - {True}, - {sample['isActive'] for sample in updated_samples} - ) - self.assertSetEqual( - {datetime.now().strftime('%Y-%m-%d')}, - {sample['loadedDate'][:10] for sample in updated_samples} - ) - self.assertDictEqual(response_json['samplesByGuid'][existing_old_index_sample_guid], {'isActive': False}) - - updated_sample_models = Sample.objects.filter(guid__in=[sample['sampleGuid'] for sample in updated_samples]) - self.assertEqual(len(updated_sample_models), 3) - self.assertSetEqual({INDEX_NAME}, {sample.elasticsearch_index for sample in updated_sample_models}) - - existing_index_sample_model = Sample.objects.get(guid=existing_index_sample_guid) - self.assertEqual(existing_index_sample_model.sample_type, 'WES') - self.assertTrue(existing_index_sample_model.is_active) - self.assertTrue(str(existing_index_sample_model.loaded_date).startswith('2017-02-05')) - - self._assert_expected_notification(mock_send_email, sample_type='WES', count=2) - - # Adding an SV index works additively with the regular variants index - mock_random.return_value = 1234567 - mock_send_email.reset_mock() - urllib3_responses.add_json('/{}/_mapping'.format(SV_INDEX_NAME), { - SV_INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.bed', - 'datasetType': 'SV', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}}) - urllib3_responses.add_json('/{}/_search?size=0'.format(SV_INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19675_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': SV_INDEX_NAME, - 'datasetType': 'SV', - })) - self.assertEqual(response.status_code, 200) - - self.assertDictEqual( - urllib3_responses.call_request_json(), - {'aggs': {'sample_ids': {'terms': {'field': 'samples', 'size': 10000}}}} - ) - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - sv_sample_guid = 'S0001234567_na19675_1' - self.assertDictEqual(response_json['familiesByGuid'], {}) - self.assertListEqual(list(response_json['samplesByGuid'].keys()), [sv_sample_guid]) - self.assertEqual(response_json['samplesByGuid'][sv_sample_guid]['datasetType'], 'SV') - self.assertEqual(response_json['samplesByGuid'][sv_sample_guid]['sampleType'], 'WES') - self.assertTrue(response_json['samplesByGuid'][sv_sample_guid]['isActive']) - self.assertListEqual(list(response_json['individualsByGuid'].keys()), ['I000001_na19675']) - self.assertListEqual(list(response_json['individualsByGuid']['I000001_na19675'].keys()), ['sampleGuids']) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675']['sampleGuids']), - {sv_sample_guid, existing_index_sample_guid}) - - # Regular variant sample should still be active - sample_models = Sample.objects.filter(individual__guid='I000001_na19675') - self.assertEqual(len(sample_models), 2) - self.assertSetEqual({sv_sample_guid, existing_index_sample_guid}, - {sample.guid for sample in sample_models}) - self.assertSetEqual({True}, {sample.is_active for sample in sample_models}) - - self._assert_expected_notification(mock_send_email, sample_type='WES SV', count=1) - - # Adding an index for a different sample type works additively - mock_random.return_value = 987654 - mock_send_email.reset_mock() - urllib3_responses.add_json('/{}/_mapping'.format(NEW_SAMPLE_TYPE_INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data_1.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data_2.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - }) - urllib3_responses.add_json('/{}/_search?size=0'.format(NEW_SAMPLE_TYPE_INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19675_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': NEW_SAMPLE_TYPE_INDEX_NAME, - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'samplesByGuid', 'individualsByGuid', 'familiesByGuid'}) - new_sample_type_sample_guid = 'S0000987654_na19675_1' - self.assertDictEqual(response_json['familiesByGuid'], {}) - self.assertListEqual(list(response_json['samplesByGuid'].keys()), [new_sample_type_sample_guid]) - self.assertEqual(response_json['samplesByGuid'][new_sample_type_sample_guid]['datasetType'], 'SNV_INDEL') - self.assertEqual(response_json['samplesByGuid'][new_sample_type_sample_guid]['sampleType'], 'WGS') - self.assertTrue(response_json['samplesByGuid'][new_sample_type_sample_guid]['isActive']) - self.assertListEqual(list(response_json['individualsByGuid'].keys()), ['I000001_na19675']) - self.assertListEqual(list(response_json['individualsByGuid']['I000001_na19675'].keys()), ['sampleGuids']) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675']['sampleGuids']), - {sv_sample_guid, existing_index_sample_guid, new_sample_type_sample_guid}) - - self._assert_expected_notification(mock_send_email, sample_type='WGS', count=1) - - # Previous variant samples should still be active - sample_models = Sample.objects.filter(individual__guid='I000001_na19675') - self.assertEqual(len(sample_models), 3) - self.assertSetEqual( - {sv_sample_guid, existing_index_sample_guid, new_sample_type_sample_guid}, - {sample.guid for sample in sample_models}) - self.assertSetEqual({True}, {sample.is_active for sample in sample_models}) - - # Test sending email for adding dataset to a non-analyst project - url = reverse(add_variants_dataset_handler, args=[NON_ANALYST_PROJECT_GUID]) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON_38) - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), {'aggregations': { - 'sample_ids': {'buckets': [{'key': 'NA21234'}]} - }}, method=urllib3_responses.POST) - - self.reset_logs() - mock_send_email.reset_mock() - mock_send_email.side_effect = Exception('Email server is not configured') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 200) - - self._assert_expected_notification( - mock_send_email, sample_type='WES', count=1, project_guid=NON_ANALYST_PROJECT_GUID, - project_name='Non-Analyst Project', recipient='test_user_collaborator@test.com', - ) - self.assert_json_logs(user=None, offset=6, expected=[( - 'Error sending project email for R0004_non_analyst_project: Email server is not configured', {'detail': { - 'email_body': mock.ANY, - 'subject': 'New WES data available in seqr', 'to': ['test_user_collaborator@test.com'], - }, - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - 'severity': 'ERROR', - }) - ]) - - def _assert_expected_notification(self, mock_send_email, sample_type, count, email_content=None, - project_guid=PROJECT_GUID, project_name='1kg project nåme with uniçøde', - recipient='test_user_manager@test.com'): - if not email_content: - email_content = f'This is to notify you that data for {count} new {sample_type} samples has been loaded in seqr project {project_name}' - mock_send_email.assert_called_once_with( - email_body=f'Dear seqr user,\n\n{email_content}\n\nAll the best,\nThe seqr team', - subject=f'New {sample_type} data available in seqr', to=[recipient], process_message=mock.ANY, - ) - - @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda path: True) - @urllib3_responses.activate - def test_add_variants_dataset_errors(self): - url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) - self.check_data_manager_login(url) - - self._assert_expected_add_dataset_errors(url) - - def _assert_expected_add_dataset_errors(self, url): - response = self.client.post(url, content_type='application/json', data=json.dumps({})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "None"']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, 'datasetType': 'NOT_A_TYPE'})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "NOT_A_TYPE"']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({'datasetType': 'SV'})) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['request must contain field: "elasticsearchIndex"']}) - - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], 'test_index - Error accessing index: Connection refused: GET /test_index/_mapping') - - urllib3_responses.add_json('/{}/_mapping'.format(INDEX_NAME), - {INDEX_NAME: {'mappings': {"properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'errors': ['Index metadata must contain fields: genomeVersion, sampleType, sourceFilePath']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'NOT_A_TYPE', - 'genomeVersion': '37', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Sample type not supported: NOT_A_TYPE']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '38', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Index "test_index" has genome version 38 but this project uses version 37']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'invalidpath.txt', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Variant call dataset path must end with .vcf or .vcf.gz or .bgz or .bed or .mt']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - INDEX_NAME: {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - 'datasetType': 'SV', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}}) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Index "test_index" has dataset type SV but expects SNV_INDEL']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_SAMPLES_NUM_ALT_1}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - }) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Found mismatched sample fields for indices in alias']}) - - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), { - 'sub_index_1': {'mappings': {'_meta': { - 'sampleType': 'WES', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - 'sub_index_2': {'mappings': {'_meta': { - 'sampleType': 'WGS', - 'genomeVersion': '37', - 'sourceFilePath': 'test_data.vcf', - }, "properties": MAPPING_PROPS_WITH_SAMPLES}}, - }) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Found mismatched sample types for indices in alias']}) - - urllib3_responses.add_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': []}} - }, method=urllib3_responses.POST) - urllib3_responses.replace_json('/{}/_mapping'.format(INDEX_NAME), MAPPING_JSON) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'errors': ['No samples found. Make sure the specified caller type is correct']}) - - self.assertDictEqual( - urllib3_responses.call_request_json(), - {'aggs': {'sample_ids': {'terms': {'field': 'samples_num_alt_1', 'size': 10000}}}} - ) - - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19679'}, {'key': 'NA19678_1'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Matches not found for sample ids: NA19678_1. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.']}) - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - 'ignoreExtraSamplesInCallset': True, - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['The following families are included in the callset but are missing some family members: 1 (NA19675_1, NA19678).']}) - - urllib3_responses.replace_json('/{}/_search?size=0'.format(INDEX_NAME), { - 'aggregations': {'sample_ids': {'buckets': [{'key': 'NA19673'}]}} - }, method=urllib3_responses.POST) - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'datasetType': 'SNV_INDEL', - 'ignoreExtraSamplesInCallset': True, - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': [ - 'None of the individuals or samples in the project matched the 1 expected sample id(s)']}) - - MOCK_FILE_ITER.return_value = StringIO('NA19678_1,NA19678,metadata\n') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['Must contain 2 columns: NA19678_1, NA19678, metadata']}) - - MOCK_FILE_ITER.side_effect = Exception('Unhandled base exception') - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'elasticsearchIndex': INDEX_NAME, - 'mappingFilePath': 'mapping.csv', - 'datasetType': 'SNV_INDEL', - })) - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'Unhandled base exception') - MOCK_FILE_ITER.side_effect = None - - -# Tests for AnVIL access disabled -class LocalDatasetAPITest(AuthenticationTestCase, DatasetAPITest): - fixtures = ['users', '1kg_project'] - - -# Test for permissions from AnVIL only -class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): - fixtures = ['users', 'social_auth', '1kg_project'] - - def _assert_expected_add_dataset_errors(self, url): - response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'add_variants_dataset_handler is disabled without the elasticsearch backend') - - def test_add_variants_dataset(self, *args): - # Adding dataset is always disabled when ES is disabled, which is tested in test_add_variants_dataset_errors - pass diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index b2e2c90c7a..26528e5fc6 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -22,20 +22,6 @@ logger = SeqrLogger(__name__) -def load_mapping_file(mapping_file_path, user): - file_content = parse_file(mapping_file_path, file_iter(mapping_file_path, user=user)) - return load_mapping_file_content(file_content) - - -def load_mapping_file_content(file_content): - id_mapping = {} - for line in file_content: - if len(line) != 2: - raise ValueError("Must contain 2 columns: " + ', '.join(line)) - id_mapping[line[0]] = line[1] - return id_mapping - - def _find_or_create_samples( sample_project_tuples, projects, @@ -199,6 +185,7 @@ def _update_variant_samples(samples_guids, individual_ids, user, dataset_type, s return activated_sample_guids, inactivate_sample_guids +# TODO def match_and_update_search_samples( projects, sample_project_tuples, sample_type, dataset_type, sample_data, user, expected_families=None, sample_id_to_individual_id_mapping=None, raise_unmatched_error_template='Matches not found for sample ids: {sample_ids}', From e1fde5b80c34153875cdca6ab477bc7bd18b3ae6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 11:56:41 -0500 Subject: [PATCH 038/252] move single use helper functions --- .../check_for_new_samples_from_pipeline.py | 66 ++++++++++++++++++- seqr/views/utils/variant_utils.py | 60 ----------------- 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 92575fee99..7a42b76499 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -3,12 +3,14 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand, CommandError +from django.db.models import Q import json import logging import re +from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, Project, Individual +from seqr.models import Family, Sample, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status @@ -16,7 +18,7 @@ from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil -from seqr.views.utils.variant_utils import reset_cached_search_results, update_projects_saved_variant_json +from seqr.views.utils.variant_utils import reset_cached_search_results from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \ SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL @@ -256,7 +258,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type, updated_samples) + cls._update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type, updated_samples) @classmethod def _is_internal_project(cls, project): @@ -390,6 +392,64 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) fields=['filter_flags', 'pop_platform_filters', 'population'], ) + @staticmethod + def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples): + updates = {} + for family_guid in family_guids: + variant_models_by_key = { + v.key: v for v in + SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid).filter( + Q(saved_variant_json__genomeVersion__isnull=True) | + Q(saved_variant_json__genomeVersion=project.genome_version.replace('GRCh', '')) + ) + } + if not variant_models_by_key: + continue + variants = [] + genotypes_by_key = get_clickhouse_genotypes( + project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), + samples, + ) + for key, genotypes in genotypes_by_key.items(): + variant = variant_models_by_key[key] + variant.genotypes = genotypes + variants.append(variant) + logger.info(f'Reloading genotypes for {len(variants)} {dataset_type} variants in family {family_guid}') + SavedVariant.bulk_update_models(None, variants, ['genotypes']) + updates.update({v.id: v for v in variants}) + return updates + + @classmethod + def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, samples): + success = {} + skipped = {} + error = {} + logger.info(f'Reloading saved variants in {len(families_by_project)} projects') + for project, family_guids in families_by_project.items(): + project_name = project.name + try: + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) + if updated_saved_variants is None: + skipped[project_name] = True + else: + success[project_name] = len(updated_saved_variants) + family_summary = f' in {len(family_guids)} families' if family_guids else '' + logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') + except Exception as e: + logger.error(f'Error reloading variants in {project_name}: {e}') + error[project_name] = e + + logger.info('Reload Summary: ') + for k, v in success.items(): + if v > 0: + logger.info(f' {k}: Updated {v} variants') + if skipped: + logger.info(f'Skipped the following {len(skipped)} project with no saved variants: {", ".join(skipped)}') + if len(error): + logger.info(f'{len(error)} failed projects') + for k, v in error.items(): + logger.info(f' {k}: {v}') + update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 496f77d381..6013fec6eb 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -6,8 +6,6 @@ import json import logging import redis -from tqdm import tqdm -import traceback from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap from clickhouse_search.search import get_clickhouse_key_lookup, get_annotations_queryset, get_clickhouse_genotypes @@ -35,64 +33,6 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(families_by_project, dataset_type, samples): - success = {} - skipped = {} - error = {} - logger.info(f'Reloading saved variants in {len(families_by_project)} projects') - for project, family_guids in tqdm(families_by_project.items(), unit=' project'): - project_name = project.name - try: - updated_saved_variants = _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) - if updated_saved_variants is None: - skipped[project_name] = True - else: - success[project_name] = len(updated_saved_variants) - family_summary = f' in {len(family_guids)} families' if family_guids else '' - logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - except Exception as e: - traceback_message = traceback.format_exc() - logger.error(traceback_message) - logger.error(f'Error reloading variants in {project_name}: {e}') - error[project_name] = e - - logger.info('Reload Summary: ') - for k, v in success.items(): - if v > 0: - logger.info(f' {k}: Updated {v} variants') - if skipped: - logger.info(f'Skipped the following {len(skipped)} project with no saved variants: {", ".join(skipped)}') - if len(error): - logger.info(f'{len(error)} failed projects') - for k, v in error.items(): - logger.info(f' {k}: {v}') - - -def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples): - updates = {} - for family_guid in family_guids: - variant_models_by_key = { - v.key: v for v in SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid).filter( - Q(saved_variant_json__genomeVersion__isnull=True) | - Q(saved_variant_json__genomeVersion=project.genome_version.replace('GRCh', '')) - ) - } - if not variant_models_by_key: - continue - variants = [] - genotypes_by_key = get_clickhouse_genotypes( - project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), samples, - ) - for key, genotypes in genotypes_by_key.items(): - variant = variant_models_by_key[key] - variant.genotypes = genotypes - variants.append(variant) - logger.info(f'Reloading genotypes for {len(variants)} {dataset_type} variants in family {family_guid}') - SavedVariant.bulk_update_models(None, variants, ['genotypes']) - updates.update({v.id: v for v in variants}) - return updates - - def parse_saved_variant_json(variant_json, family_id, variant_id=None,): if 'xpos' not in variant_json: variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos']) From 2d140e00469fe147327cf3970ea2378c19122d2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 13:28:30 -0500 Subject: [PATCH 039/252] move sample matching to check samples job --- .../check_for_new_samples_from_pipeline.py | 119 +++++++++++- seqr/views/utils/dataset_utils.py | 183 +----------------- 2 files changed, 116 insertions(+), 186 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 7a42b76499..4068d63722 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -4,6 +4,7 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand, CommandError from django.db.models import Q +from django.utils import timezone import json import logging import re @@ -15,7 +16,6 @@ from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS -from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil from seqr.views.utils.variant_utils import reset_cached_search_results @@ -228,13 +228,14 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples, updated_samples, *args = match_and_update_search_samples( + # TODO clean up args + new_samples, updated_samples = cls._match_and_update_search_samples( projects=samples_by_project.keys(), sample_project_tuples=sample_project_tuples, - sample_data={'data_source': run_version, 'elasticsearch_index': ';'.join(metadata['callsets'])}, sample_type=sample_type, dataset_type=dataset_type, - user=None, + data_source=run_version, + elasticsearch_index=';'.join(metadata['callsets']), ) new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( @@ -450,6 +451,116 @@ def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, for k, v in error.items(): logger.info(f' {k}: {v}') + @classmethod + def _match_and_update_search_samples(cls, projects, sample_project_tuples, sample_type, dataset_type, **sample_data): + loaded_date = timezone.now() + samples_guids, individual_ids = cls._find_or_create_samples( + sample_project_tuples, projects, loaded_date, sample_type=sample_type, dataset_type=dataset_type, **sample_data, + ) + + included_families = dict(Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status')) + matched_samples = Sample.objects.filter( + individual__family_id__in=included_families, + is_active=True, + dataset_type=dataset_type, + sample_type=sample_type, + ).exclude(guid__in=samples_guids) + + missing_individuals = matched_samples.exclude(individual_id__in=individual_ids).values( + 'individual__family__family_id', + ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) + if missing_individuals: + # TODO test + missing_summary = ', '.join(sorted([ + f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])}" for agg in missing_individuals + ])) + raise ValueError( + f'The following families are included in the callset but are missing some family members: {missing_summary}.' + ) + + activated_sample_guids = Sample.bulk_update(user=None, update_json={ + 'is_active': True, + 'loaded_date': loaded_date, + **sample_data, + }, guid__in=samples_guids, is_active=False) + updated_samples = Sample.objects.filter(guid__in=activated_sample_guids) + + inactivated_sample_guids = Sample.bulk_update( + user=None, update_json={'is_active': False}, queryset=matched_samples.filter(individual_id__in=individual_ids), + ) + + family_ids_to_update = [ + family_id for family_id, analysis_status in included_families.items() + if analysis_status in {Family.ANALYSIS_STATUS_WAITING_FOR_DATA, Family.ANALYSIS_STATUS_LOADING_FAILED} + ] + Family.bulk_update( + user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update, + ) + + previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) + new_samples = updated_samples.exclude(individual_id__in=previous_loaded_individuals) + + return new_samples, updated_samples + + @staticmethod + def _find_or_create_samples(sample_project_tuples, projects, loaded_date, **sample_params): + samples_by_key = { + (s.pop('sample_id'), s.pop('individual__family__project__name')): s + for s in Sample.objects.filter( + individual__family__project__in=projects, + sample_id__in={sample_id for sample_id, _ in sample_project_tuples}, + **sample_params + ).values('guid', 'individual_id', 'sample_id', 'individual__family__project__name') + } + existing_samples = { + key: s for key, s in samples_by_key.items() if key in sample_project_tuples + } + remaining_sample_keys = set(sample_project_tuples) - set(existing_samples.keys()) + + matched_individual_ids = {sample['individual_id'] for sample in existing_samples.values()} + samples_guids = [sample['guid'] for sample in existing_samples.values()] + individual_ids = {sample['individual_id'] for sample in existing_samples.values()} + if len(remaining_sample_keys) > 0: + individuals = Individual.objects.filter(family__project__in=projects) + if matched_individual_ids: + individuals = individuals.exclude(id__in=matched_individual_ids) + remaining_individuals_dict = { + (i['individual_id'], i.pop('family__project__name')): i + for i in individuals.values('id', 'individual_id', 'family__project__name') + } + + # find Individual records with exactly-matching individual_ids + sample_id_to_individual_record = {} + for sample_key in remaining_sample_keys: + if sample_key not in remaining_individuals_dict: + continue + sample_id_to_individual_record[sample_key] = remaining_individuals_dict[sample_key] + del remaining_individuals_dict[sample_key] + + remaining_sample_keys -= set(sample_id_to_individual_record.keys()) + if remaining_sample_keys: + sample_ids = ', '.join(sorted([sample_id for sample_id, _ in remaining_sample_keys])) + raise ValueError(f'Matches not found for sample ids: {sample_ids}') + + # create new Sample records for Individual records that matches + new_sample_args = { + sample_key: { + 'individual_id': individual['id'], + 'sample_id': sample_key[0], + } for sample_key, individual in sample_id_to_individual_record.items() + } + individual_ids.update({sample['individual_id'] for sample in new_sample_args.values()}) + new_samples = [ + Sample( + loaded_date=loaded_date, + **created_sample_data, + **sample_params, + ) for created_sample_data in new_sample_args.values()] + new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) + samples_guids += [s.guid for s in new_sample_models] + + return samples_guids, individual_ids + update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 26528e5fc6..ecd418b22f 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -2,13 +2,12 @@ from datetime import datetime from django.contrib.postgres.aggregates import ArrayAgg from django.db.models import F, Q -from django.utils import timezone import gzip import json import os from tqdm import tqdm -from seqr.models import Sample, Individual, Family, Project, RnaSample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier +from seqr.models import Individual, Project, RnaSample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier from seqr.utils.file_utils import file_iter from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -22,88 +21,6 @@ logger = SeqrLogger(__name__) -def _find_or_create_samples( - sample_project_tuples, - projects, - user, - sample_type, - dataset_type, - sample_id_to_individual_id_mapping, - raise_no_match_error=False, - raise_unmatched_error_template=None, - sample_data=None, -): - sample_params = {'sample_type': sample_type, 'dataset_type': dataset_type} - sample_params.update(sample_data or {}) - - samples_by_key = { - (s.pop('sample_id'), s.pop('individual__family__project__name')): s - for s in Sample.objects.filter( - individual__family__project__in=projects, - sample_id__in={sample_id for sample_id, _ in sample_project_tuples}, - **sample_params - ).values('guid', 'individual_id', 'sample_id', 'individual__family__project__name') - } - existing_samples = { - key: s for key, s in samples_by_key.items() if key in sample_project_tuples - } - remaining_sample_keys = set(sample_project_tuples) - set(existing_samples.keys()) - - matched_individual_ids = {sample['individual_id'] for sample in existing_samples.values()} - loaded_date = timezone.now() - samples_guids = [sample['guid'] for sample in existing_samples.values()] - individual_ids = {sample['individual_id'] for sample in existing_samples.values()} - if len(remaining_sample_keys) > 0: - remaining_individuals_dict = _get_individuals_by_key(projects, matched_individual_ids) - - # find Individual records with exactly-matching individual_ids - sample_id_to_individual_record = {} - for sample_key in remaining_sample_keys: - individual_key = _get_individual_key(sample_key, sample_id_to_individual_id_mapping) - if individual_key not in remaining_individuals_dict: - continue - sample_id_to_individual_record[sample_key] = remaining_individuals_dict[individual_key] - del remaining_individuals_dict[individual_key] - - logger.debug(str(len(sample_id_to_individual_record)) + " matched individual ids", user) - - remaining_sample_keys -= set(sample_id_to_individual_record.keys()) - if raise_no_match_error and len(remaining_sample_keys) == len(sample_project_tuples): - raise ValueError( - 'None of the individuals or samples in the project matched the {} expected sample id(s)'.format(len(sample_project_tuples))) - if raise_unmatched_error_template and remaining_sample_keys: - raise ValueError(raise_unmatched_error_template.format( - sample_ids=(', '.join(sorted([sample_id for sample_id, _ in remaining_sample_keys]))))) - - # create new Sample records for Individual records that matches - new_sample_args = { - sample_key: { - 'individual_id': individual['id'], - 'sample_id': sample_key[0], - } for sample_key, individual in sample_id_to_individual_record.items() - } - individual_ids.update({sample['individual_id'] for sample in new_sample_args.values()}) - new_sample_models = _create_samples( - new_sample_args.values(), - user, - loaded_date=loaded_date, - **sample_params, - ) - samples_guids += [s.guid for s in new_sample_models] - - return samples_guids, individual_ids, loaded_date - - -def _create_samples(sample_data, user, loaded_date=timezone.now(), **kwargs): - new_samples = [ - Sample( - loaded_date=loaded_date, - **created_sample_data, - **kwargs, - ) for created_sample_data in sample_data] - return Sample.bulk_create(user, new_samples) - - def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): new_samples = [ RnaSample(individual_id=individual_id, tissue_type=tissue_type, **kwargs) @@ -124,104 +41,6 @@ def _get_rna_sample_data_by_key(values=None, **kwargs): } -def _get_individuals_by_key(projects, matched_individual_ids=None): - individuals = Individual.objects.filter(family__project__in=projects) - if matched_individual_ids: - individuals = individuals.exclude(id__in=matched_individual_ids) - return { - (i['individual_id'], i.pop('family__project__name')): i - for i in individuals.values('id', 'individual_id', 'family__project__name') - } - - -def _get_individual_key(sample_key, sample_id_to_individual_id_mapping): - return ((sample_id_to_individual_id_mapping or {}).get(sample_key[0], sample_key[0]), sample_key[1]) - - -def _validate_samples_families(samples_guids, included_family_guids, sample_type, dataset_type, expected_families=None): - missing_individuals = Individual.objects.filter( - family__guid__in=included_family_guids, - sample__is_active=True, - sample__dataset_type=dataset_type, - sample__sample_type=sample_type, - ).exclude(sample__guid__in=samples_guids).select_related('family') - missing_family_individuals = defaultdict(list) - for individual in missing_individuals: - missing_family_individuals[individual.family].append(individual) - - if missing_family_individuals: - raise ValueError( - 'The following families are included in the callset but are missing some family members: {}.'.format( - ', '.join(sorted( - ['{} ({})'.format(family.family_id, ', '.join(sorted([i.individual_id for i in missing_indivs]))) - for family, missing_indivs in missing_family_individuals.items()] - )))) - - if expected_families: - missing_families = [f.family_id for f in expected_families if f.guid not in included_family_guids] - if missing_families: - raise ValueError( - 'The following families have saved variants but are missing from the callset: {}.'.format( - ', '.join(missing_families) - )) - - -def _update_variant_samples(samples_guids, individual_ids, user, dataset_type, sample_type, sample_data): - - activated_sample_guids = Sample.bulk_update(user, { - 'is_active': True, - **sample_data, - }, guid__in=samples_guids, is_active=False) - - inactivate_samples = Sample.objects.filter( - individual_id__in=individual_ids, - is_active=True, - dataset_type=dataset_type, - sample_type=sample_type, - ).exclude(guid__in=samples_guids) - - inactivate_sample_guids = Sample.bulk_update(user, {'is_active': False}, queryset=inactivate_samples) - - return activated_sample_guids, inactivate_sample_guids - - -# TODO -def match_and_update_search_samples( - projects, sample_project_tuples, sample_type, dataset_type, sample_data, user, expected_families=None, - sample_id_to_individual_id_mapping=None, raise_unmatched_error_template='Matches not found for sample ids: {sample_ids}', -): - samples_guids, individual_ids, loaded_date = _find_or_create_samples( - sample_project_tuples=sample_project_tuples, - projects=projects, - user=user, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_no_match_error=not raise_unmatched_error_template, - raise_unmatched_error_template=raise_unmatched_error_template, - sample_type=sample_type, - dataset_type=dataset_type, - sample_data=sample_data, - ) - - included_families = dict(Family.objects.filter(individual__id__in=individual_ids).values_list('guid', 'analysis_status')) - _validate_samples_families(samples_guids, included_families.keys(), sample_type, dataset_type, expected_families=expected_families) - - activated_sample_guids, inactivated_sample_guids = _update_variant_samples( - samples_guids, individual_ids, user, dataset_type, sample_type, sample_data={'loaded_date': loaded_date, **sample_data}) - updated_samples = Sample.objects.filter(guid__in=activated_sample_guids) - - family_guids_to_update = [ - family_guid for family_guid, analysis_status in included_families.items() - if analysis_status in {Family.ANALYSIS_STATUS_WAITING_FOR_DATA, Family.ANALYSIS_STATUS_LOADING_FAILED} - ] - Family.bulk_update( - user, {'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, guid__in=family_guids_to_update) - - previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - new_samples = updated_samples.exclude(individual_id__in=previous_loaded_individuals) - - return new_samples, updated_samples, inactivated_sample_guids, family_guids_to_update - - def _parse_tsv_row(row): return [s.strip().strip('"') for s in row.rstrip('\n').split('\t')] From 73bf4ad47cfc270797fb96ac6d571461df1a21d9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 13:49:47 -0500 Subject: [PATCH 040/252] clean up --- .../check_for_new_samples_from_pipeline.py | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 4068d63722..6542460984 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -215,7 +215,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_project_tuples = [] invalid_genome_version_projects = [] for project, sample_ids in samples_by_project.items(): - sample_project_tuples += [(sample_id, project.name) for sample_id in sample_ids] + sample_project_tuples += [(sample_id, project.id) for sample_id in sample_ids] project_genome_version = GENOME_VERSION_LOOKUP.get(project.genome_version, project.genome_version) if project_genome_version != genome_version: invalid_genome_version_projects.append((project.guid, project_genome_version)) @@ -228,15 +228,10 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - # TODO clean up args new_samples, updated_samples = cls._match_and_update_search_samples( - projects=samples_by_project.keys(), - sample_project_tuples=sample_project_tuples, - sample_type=sample_type, - dataset_type=dataset_type, - data_source=run_version, - elasticsearch_index=';'.join(metadata['callsets']), + sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), ) + # TIODo clea nup return val usage new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( samples=ArrayAgg('sample_id', distinct=True), @@ -452,10 +447,10 @@ def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, logger.info(f' {k}: {v}') @classmethod - def _match_and_update_search_samples(cls, projects, sample_project_tuples, sample_type, dataset_type, **sample_data): + def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): loaded_date = timezone.now() samples_guids, individual_ids = cls._find_or_create_samples( - sample_project_tuples, projects, loaded_date, sample_type=sample_type, dataset_type=dataset_type, **sample_data, + sample_project_tuples, loaded_date, sample_type=sample_type, dataset_type=dataset_type, **sample_data, ) included_families = dict(Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status')) @@ -503,30 +498,30 @@ def _match_and_update_search_samples(cls, projects, sample_project_tuples, sampl return new_samples, updated_samples @staticmethod - def _find_or_create_samples(sample_project_tuples, projects, loaded_date, **sample_params): + def _find_or_create_samples(sample_project_tuples, loaded_date, **sample_params): samples_by_key = { - (s.pop('sample_id'), s.pop('individual__family__project__name')): s + (s.pop('sample_id'), s.pop('individual__family__project_id')): s for s in Sample.objects.filter( - individual__family__project__in=projects, + individual__family__project_id__in={project_id for _, project_id in sample_project_tuples}, sample_id__in={sample_id for sample_id, _ in sample_project_tuples}, **sample_params - ).values('guid', 'individual_id', 'sample_id', 'individual__family__project__name') + ).values('guid', 'sample_id', 'individual__family__project_id') } existing_samples = { - key: s for key, s in samples_by_key.items() if key in sample_project_tuples + key: s['guid'] for key, s in samples_by_key.items() if key in sample_project_tuples } remaining_sample_keys = set(sample_project_tuples) - set(existing_samples.keys()) - matched_individual_ids = {sample['individual_id'] for sample in existing_samples.values()} - samples_guids = [sample['guid'] for sample in existing_samples.values()] - individual_ids = {sample['individual_id'] for sample in existing_samples.values()} + samples_guids = list(existing_samples.values()) + individual_ids = {sample_id for sample_id, _ in existing_samples.keys()} if len(remaining_sample_keys) > 0: - individuals = Individual.objects.filter(family__project__in=projects) - if matched_individual_ids: - individuals = individuals.exclude(id__in=matched_individual_ids) + remaining_projects = {project_id for _, project_id in remaining_sample_keys} + individuals = Individual.objects.filter(family__project_id__in=remaining_projects) + if individual_ids: + individuals = individuals.exclude(id__in=individual_ids) remaining_individuals_dict = { - (i['individual_id'], i.pop('family__project__name')): i - for i in individuals.values('id', 'individual_id', 'family__project__name') + (i['individual_id'], i.pop('family__project_id')): i + for i in individuals.values('id', 'individual_id', 'family__project_id') } # find Individual records with exactly-matching individual_ids From f763cdb642d3524f4d87674d9ecdbb818965775a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 15:22:12 -0500 Subject: [PATCH 041/252] clean up match samples --- .../check_for_new_samples_from_pipeline.py | 167 ++++++++---------- ...eck_for_new_samples_from_pipeline_tests.py | 20 ++- 2 files changed, 86 insertions(+), 101 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 6542460984..313b37c0fc 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -228,14 +228,9 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples, updated_samples = cls._match_and_update_search_samples( + new_samples_by_project = cls._match_and_update_search_samples( sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), ) - # TIODo clea nup return val usage - - new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( - samples=ArrayAgg('sample_id', distinct=True), - ).values_list('individual__family__project', 'samples')) split_project_pdos = cls._report_loading_success( dataset_type, sample_type, run_version, samples_by_project, new_samples_by_project, @@ -254,7 +249,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - cls._update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type, updated_samples) + cls._update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -416,7 +411,7 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, return updates @classmethod - def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, samples): + def _update_projects_saved_variant_json(cls, families_by_project, dataset_type): success = {} skipped = {} error = {} @@ -424,7 +419,7 @@ def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, for project, family_guids in families_by_project.items(): project_name = project.name try: - updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, Sample.objects.filter(is_active=True)) if updated_saved_variants is None: skipped[project_name] = True else: @@ -448,20 +443,72 @@ def _update_projects_saved_variant_json(cls, families_by_project, dataset_type, @classmethod def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): + individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples) + individual_ids = individual_ids_by_keys.values() + matched_samples = Sample.objects.filter(is_active=True, dataset_type=dataset_type, sample_type=sample_type) + cls._update_matched_families(individual_ids, matched_samples) + + sample_guids_by_individual = dict(Sample.objects.filter( + individual_id__in=individual_ids, sample_type=sample_type, dataset_type=dataset_type, **sample_data, + ).values_list('individual_id', 'guid')) + remaining_key_individuals = { + key: individual_id for key, individual_id in individual_ids_by_keys.items() + if individual_id not in sample_guids_by_individual + } loaded_date = timezone.now() - samples_guids, individual_ids = cls._find_or_create_samples( - sample_project_tuples, loaded_date, sample_type=sample_type, dataset_type=dataset_type, **sample_data, + created_sample_guids = cls._create_samples( + remaining_key_individuals, loaded_date=loaded_date, sample_type=sample_type, dataset_type=dataset_type, + **sample_data, + ) if remaining_key_individuals else [] + + sample_guids = list(sample_guids_by_individual.values()) + created_sample_guids + + activated_sample_guids = Sample.bulk_update(user=None, update_json={ + 'is_active': True, + 'loaded_date': loaded_date, + **sample_data, + }, guid__in=sample_guids, is_active=False) + + inactivate_samples = matched_samples.filter(individual_id__in=individual_ids).exclude(guid__in=sample_guids) + inactivated_sample_guids = Sample.bulk_update( + user=None, update_json={'is_active': False}, queryset=inactivate_samples, ) - included_families = dict(Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status')) - matched_samples = Sample.objects.filter( - individual__family_id__in=included_families, - is_active=True, - dataset_type=dataset_type, - sample_type=sample_type, - ).exclude(guid__in=samples_guids) + previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) + return dict(Sample.objects.filter( + guid__in=activated_sample_guids + ).exclude(individual_id__in=previous_loaded_individuals).values( + 'individual__family__project' + ).annotate(samples=ArrayAgg('sample_id', distinct=True)).values_list( + 'individual__family__project', 'samples', + )) - missing_individuals = matched_samples.exclude(individual_id__in=individual_ids).values( + @staticmethod + def _get_matched_individuals(sample_project_tuples): + individual_ids_by_keys = { + (individual_id, project_id): individual_db_id + for individual_db_id, individual_id, project_id in Individual.objects.filter( + family__project_id__in={project_id for _, project_id in sample_project_tuples}, + individual_id__in={sample_id for sample_id, _ in sample_project_tuples}, + ).values_list('id', 'individual_id', 'family__project_id') + if (individual_id, project_id) in sample_project_tuples + } + + missing_keys = set(sample_project_tuples) - set(individual_ids_by_keys.keys()) + if missing_keys: + sample_ids = ', '.join(sorted([sample_id for sample_id, _ in missing_keys])) + raise ValueError(f'Matches not found for sample ids: {sample_ids}') + + return individual_ids_by_keys + + @staticmethod + def _update_matched_families(individual_ids, matched_samples): + included_families = dict( + Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status') + ) + missing_individuals = matched_samples.filter( + individual__family_id__in=included_families, + ).exclude(individual_id__in=individual_ids).values( 'individual__family__family_id', ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) if missing_individuals: @@ -473,17 +520,6 @@ def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, da f'The following families are included in the callset but are missing some family members: {missing_summary}.' ) - activated_sample_guids = Sample.bulk_update(user=None, update_json={ - 'is_active': True, - 'loaded_date': loaded_date, - **sample_data, - }, guid__in=samples_guids, is_active=False) - updated_samples = Sample.objects.filter(guid__in=activated_sample_guids) - - inactivated_sample_guids = Sample.bulk_update( - user=None, update_json={'is_active': False}, queryset=matched_samples.filter(individual_id__in=individual_ids), - ) - family_ids_to_update = [ family_id for family_id, analysis_status in included_families.items() if analysis_status in {Family.ANALYSIS_STATUS_WAITING_FOR_DATA, Family.ANALYSIS_STATUS_LOADING_FAILED} @@ -492,69 +528,16 @@ def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, da user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update, ) - previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - new_samples = updated_samples.exclude(individual_id__in=previous_loaded_individuals) - - return new_samples, updated_samples - @staticmethod - def _find_or_create_samples(sample_project_tuples, loaded_date, **sample_params): - samples_by_key = { - (s.pop('sample_id'), s.pop('individual__family__project_id')): s - for s in Sample.objects.filter( - individual__family__project_id__in={project_id for _, project_id in sample_project_tuples}, - sample_id__in={sample_id for sample_id, _ in sample_project_tuples}, - **sample_params - ).values('guid', 'sample_id', 'individual__family__project_id') - } - existing_samples = { - key: s['guid'] for key, s in samples_by_key.items() if key in sample_project_tuples - } - remaining_sample_keys = set(sample_project_tuples) - set(existing_samples.keys()) - - samples_guids = list(existing_samples.values()) - individual_ids = {sample_id for sample_id, _ in existing_samples.keys()} - if len(remaining_sample_keys) > 0: - remaining_projects = {project_id for _, project_id in remaining_sample_keys} - individuals = Individual.objects.filter(family__project_id__in=remaining_projects) - if individual_ids: - individuals = individuals.exclude(id__in=individual_ids) - remaining_individuals_dict = { - (i['individual_id'], i.pop('family__project_id')): i - for i in individuals.values('id', 'individual_id', 'family__project_id') - } - - # find Individual records with exactly-matching individual_ids - sample_id_to_individual_record = {} - for sample_key in remaining_sample_keys: - if sample_key not in remaining_individuals_dict: - continue - sample_id_to_individual_record[sample_key] = remaining_individuals_dict[sample_key] - del remaining_individuals_dict[sample_key] - - remaining_sample_keys -= set(sample_id_to_individual_record.keys()) - if remaining_sample_keys: - sample_ids = ', '.join(sorted([sample_id for sample_id, _ in remaining_sample_keys])) - raise ValueError(f'Matches not found for sample ids: {sample_ids}') - - # create new Sample records for Individual records that matches - new_sample_args = { - sample_key: { - 'individual_id': individual['id'], - 'sample_id': sample_key[0], - } for sample_key, individual in sample_id_to_individual_record.items() - } - individual_ids.update({sample['individual_id'] for sample in new_sample_args.values()}) - new_samples = [ - Sample( - loaded_date=loaded_date, - **created_sample_data, - **sample_params, - ) for created_sample_data in new_sample_args.values()] - new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) - samples_guids += [s.guid for s in new_sample_models] - - return samples_guids, individual_ids + def _create_samples(remaining_key_individuals, **sample_params): + new_samples = [ + Sample( + individual_id=individual_id, + sample_id=sample_key[0], + **sample_params, + ) for sample_key, individual_id in remaining_key_individuals.items()] + new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) + return [s.guid for s in new_sample_models] update_individuals_sample_qc = Command._update_individuals_sample_qc diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index bb856d7dd3..8ba8fbc1be 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -349,11 +349,6 @@ def mock_opened_file(index): @mock.patch('seqr.utils.file_utils.os.path.isfile', lambda *args: True) class CheckNewSamplesTest(object): - CREATE_SNV_INDEL_SAMPLES_LOGS = [ - ('Loading 4 WES SNV_INDEL samples in 2 projects', None), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ] UPDATE_SAMPLE_LOGS = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], @@ -466,9 +461,12 @@ def _test_success_call(self, anvil_email_calls): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) self._test_call(run_loading_logs={ - 'GRCh38/SNV_INDEL': self.CREATE_SNV_INDEL_SAMPLES_LOGS + [ - ('update 1 Samples', {'dbUpdate': mock.ANY}), + 'GRCh38/SNV_INDEL': [ + ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), + ('create 4 Samples', {'dbUpdate': mock.ANY}), + ('update 4 Samples', {'dbUpdate': mock.ANY}), + ('update 1 Samples', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, @@ -477,10 +475,10 @@ def _test_success_call(self, anvil_email_calls): ], 'GRCh38/SV': [ ('Loading 4 WES SV samples in 2 projects', None), + ('update 1 Familys', {'dbUpdate': mock.ANY}), ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ('update 3 Samples', {'dbUpdate': mock.ANY}), - ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), @@ -718,7 +716,11 @@ def test_command(self): if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) self._test_call(num_runs=2, run_loading_logs={ - 'GRCh38/SNV_INDEL': self.CREATE_SNV_INDEL_SAMPLES_LOGS + airtable_logs + self.UPDATE_SAMPLE_LOGS, + 'GRCh38/SNV_INDEL': [ + ('Loading 4 WES SNV_INDEL samples in 2 projects', None), + ('create 4 Samples', {'dbUpdate': mock.ANY}), + ('update 4 Samples', {'dbUpdate': mock.ANY}), + ] + airtable_logs + self.UPDATE_SAMPLE_LOGS, }) From dd929b0fa468fe3e019a2804d817ff65bd141cb7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 15:52:29 -0500 Subject: [PATCH 042/252] test partial family loaded --- .../check_for_new_samples_from_pipeline.py | 5 ++-- ...eck_for_new_samples_from_pipeline_tests.py | 26 ++++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 313b37c0fc..81cd420947 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -512,12 +512,11 @@ def _update_matched_families(individual_ids, matched_samples): 'individual__family__family_id', ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) if missing_individuals: - # TODO test missing_summary = ', '.join(sorted([ - f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])}" for agg in missing_individuals + f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals ])) raise ValueError( - f'The following families are included in the callset but are missing some family members: {missing_summary}.' + f'The following families are included in the callset but are missing some family members: {missing_summary}' ) family_ids_to_update = [ diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 8ba8fbc1be..d142664297 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -165,6 +165,7 @@ '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2025-12-02/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', @@ -184,6 +185,9 @@ b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/_SUCCESS', + b'gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/_CLICKHOUSE_LOAD_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', b'gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', @@ -325,8 +329,12 @@ 'family_samples': {'F0000123_ABC': ['NA22882', 'NA20885']}, }, { 'callsets': ['invalid_sample.vcf'], - 'sample_type': 'WGS', + 'sample_type': 'WES', 'family_samples': {'F000003_3': ['NA22882', 'NA20885']}, +}, { + 'callsets': ['invalid_sample.vcf'], + 'sample_type': 'WES', + 'family_samples': {'F000002_2': ['HG00732', 'HG00733']}, }, { 'callsets': ['gcnv.bed.gz'], 'sample_type': 'WES', @@ -417,7 +425,7 @@ def set_up(self): Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): + def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): self._set_loading_files() self.reset_logs() @@ -429,7 +437,7 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): logs = self.LIST_FILE_LOGS[:1] + [(f'Loading new samples from {num_runs} run(s)', None)] runs = [ ('GRCh38/SNV_INDEL', 'auto__2023-08-09'), ('GRCh37/SNV_INDEL', 'manual__2023-11-02'), - ('GRCh38/MITO', 'auto__2024-08-12'), ('GRCh38/SV', 'auto__2024-09-14'), + ('GRCh38/MITO', 'auto__2024-08-12'), ('GRCh38/MITO', 'auto__2025-12-02'), ('GRCh38/SV', 'auto__2024-09-14'), ('GRCh38/SNV_INDEL', 'hail_search_to_clickhouse_migration_WGS_R0877_neptune'), ] if single_call: @@ -471,7 +479,7 @@ def _test_success_call(self, anvil_email_calls): ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, 'GRCh38/MITO': [ - ('Loading 2 WGS MITO samples in 1 projects', None) + ('Loading 2 WES MITO samples in 1 projects', None) ], 'GRCh38/SV': [ ('Loading 4 WES SV samples in 2 projects', None), @@ -487,6 +495,7 @@ def _test_success_call(self, anvil_email_calls): }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', + 'auto__2025-12-02': 'The following families are included in the callset but are missing some family members: 2 (HG00731)', }) # Test notifications @@ -586,6 +595,7 @@ def test_command(self): 'auto__2023-08-09': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0003_test (GRCh37)', 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37)', + 'auto__2025-12-02': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37)', 'auto__2024-09-14': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37), R0003_test (GRCh37)', } self._test_call(error_logs=error_logs) @@ -771,7 +781,7 @@ def _assert_has_expected_empty_list_file_calls(self): self.mock_glob.assert_called_with('/seqr/seqr-hail-search-data/GRCh37/MITO/runs/*/*', recursive=False) def _set_reloading_loading_files(self): - self.mock_glob.return_value = [LOCAL_RUN_PATHS[3], LOCAL_RUN_PATHS[6]] + self.mock_glob.return_value = [LOCAL_RUN_PATHS[3], LOCAL_RUN_PATHS[7]] def _set_loading_files(self): if not self.mock_glob.return_value: @@ -790,7 +800,7 @@ def _assert_expected_loading_file_calls(self, single_call): self.assertEqual(self.mock_mkdir.call_count, 0 if single_call else 2) self.assertEqual(list(self.mock_written_files.keys()), [ file.replace('validation_errors.json', '_ERRORS_REPORTED') - for file in [LOCAL_RUN_PATHS[2], LOCAL_RUN_PATHS[7]] + for file in [LOCAL_RUN_PATHS[2], LOCAL_RUN_PATHS[8]] ]) def _assert_expected_airtable_calls(self, *args, **kwargs): @@ -918,7 +928,7 @@ def _assert_has_expected_empty_list_file_calls(self): ) def _set_reloading_loading_files(self): - self.mock_ls_process.communicate.return_value = b'\n'.join(RUN_PATHS[6:8] + RUN_PATHS[15:17]), b'' + self.mock_ls_process.communicate.return_value = b'\n'.join(RUN_PATHS[6:8] + RUN_PATHS[18:20]), b'' self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): @@ -942,6 +952,8 @@ def _assert_expected_loading_file_calls(self, single_call): ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh37/SNV_INDEL/runs/manual__2023-11-02/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/metadata.json', -2), ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2024-08-12/metadata.json', -2), + ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/metadata.json', -2), + ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/MITO/runs/auto__2025-12-02/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/metadata.json', -2), ('gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/GCNV/runs/auto__2024-09-14/metadata.json', -2), ('gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', -2), From 190a6708d805652f9f392bc644af6a2eaee13904 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 15:57:49 -0500 Subject: [PATCH 043/252] clean up --- .../check_for_new_samples_from_pipeline.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 81cd420947..2a413b2729 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -228,10 +228,14 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers sample_type = metadata['sample_type'] logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples_by_project = cls._match_and_update_search_samples( + new_samples = cls._match_and_update_search_samples( sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), ) + new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( + samples=ArrayAgg('sample_id', distinct=True), + ).values_list('individual__family__project', 'samples')) + split_project_pdos = cls._report_loading_success( dataset_type, sample_type, run_version, samples_by_project, new_samples_by_project, ) @@ -475,13 +479,7 @@ def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, da ) previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - return dict(Sample.objects.filter( - guid__in=activated_sample_guids - ).exclude(individual_id__in=previous_loaded_individuals).values( - 'individual__family__project' - ).annotate(samples=ArrayAgg('sample_id', distinct=True)).values_list( - 'individual__family__project', 'samples', - )) + return Sample.objects.filter(guid__in=activated_sample_guids).exclude(individual_id__in=previous_loaded_individuals) @staticmethod def _get_matched_individuals(sample_project_tuples): From fe98fa1bf4976d78c1abc75a7cd34bb758d0a613 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:05:48 -0500 Subject: [PATCH 044/252] data manager clean up --- seqr/urls.py | 5 +- seqr/utils/search/elasticsearch/es_utils.py | 36 --- seqr/views/apis/data_manager_api.py | 22 -- seqr/views/apis/data_manager_api_tests.py | 291 +------------------- 4 files changed, 2 insertions(+), 352 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index a6196d2259..cf2d551d2b 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -120,8 +120,7 @@ update_user, \ forgot_password -from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ - update_rna_seq, proxy_to_kibana, load_phenotype_prioritization_data, \ +from seqr.views.apis.data_manager_api import update_rna_seq, proxy_to_kibana, load_phenotype_prioritization_data, \ validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ trigger_delete_project, trigger_delete_family from seqr.views.apis.report_api import \ @@ -329,8 +328,6 @@ 'report/gregor': gregor_export, 'report/seqr_stats': seqr_stats, - 'data_management/elasticsearch_status': elasticsearch_status, - 'data_management/delete_index': delete_index, 'data_management/get_all_users': get_all_users, 'data_management/update_rna_seq': update_rna_seq, 'data_management/load_phenotype_prioritization_data': load_phenotype_prioritization_data, diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index a9b9bff984..628fa04dd4 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -145,42 +145,6 @@ def _validate_index_metadata(index_metadata, elasticsearch_index, project=None, )) -def delete_es_index(index): - client = get_es_client() - client.indices.delete(index) - updated_indices, _ = _get_es_indices(client) - return updated_indices - - -def get_elasticsearch_status(): - client = get_es_client() - - disk_status = { - disk['node']: disk for disk in - _get_es_meta(client, 'allocation', ['node', 'shards', 'disk.avail', 'disk.used', 'disk.percent']) - } - - node_stats = {} - for node in _get_es_meta(client, 'nodes', ['name', 'heap.percent']): - if node['name'] in disk_status: - disk_status[node.pop('name')].update(node) - else: - node_stats[node['name']] = node - - indices, seqr_index_projects = _get_es_indices(client) - - errors = ['{} does not exist and is used by project(s) {}'.format( - index, ', '.join(['{} ({} samples)'.format(p.name, len(indivs)) for p, indivs in project_individuals.items()]) - ) for index, project_individuals in sorted(seqr_index_projects.items()) if project_individuals] - - return { - 'indices': indices, - 'diskStats': list(disk_status.values()), - 'nodeStats': list(node_stats.values()), - 'errors': errors, - } - - def _get_es_meta(client, meta_type, fields, filter_rows=None): return [{ _to_camel_case(field.replace('.', '_')): o[field] for field in fields diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 345d0bbf7d..5cd50904a8 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -14,8 +14,6 @@ from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search -from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index -from seqr.utils.search.utils import es_only, InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list @@ -34,26 +32,6 @@ logger = SeqrLogger(__name__) - -@data_manager_required -@es_only -def elasticsearch_status(request): - return create_json_response(get_elasticsearch_status()) - - -@data_manager_required -@es_only -def delete_index(request): - index = json.loads(request.body)['index'] - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=index) - if active_samples: - projects = set(active_samples.values_list('individual__family__project__name', flat=True)) - raise InvalidSearchException(f'"{index}" is still used by: {", ".join(projects)}') - - updated_indices = delete_es_index(index) - - return create_json_response({'indices': updated_indices}) - RNA = 'RNA' TISSUE_FIELD = 'TissueOfOrigin' AIRTABLE_TISSUE_TYPE_MAP = { diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 7530e20c2b..6bf9b5ae91 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -7,228 +7,16 @@ import responses from seqr.utils.communication_utils import _set_bulk_notification_stream -from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ - update_rna_seq, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ +from seqr.views.apis.data_manager_api import update_rna_seq, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ get_loaded_projects, load_data, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest -from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' NON_ANALYST_PROJECT_GUID = 'R0004_non_analyst_project' -ES_CAT_ALLOCATION=[{ - 'node': 'node-1', - 'shards': '113', - 'disk.used': '67.2gb', - 'disk.avail': '188.6gb', - 'disk.percent': '26' -}, - {'node': 'UNASSIGNED', - 'shards': '2', - 'disk.used': None, - 'disk.avail': None, - 'disk.percent': None - }] - -ES_CAT_NODES=[{ - 'name': 'node-1', - 'heap.percent': '57', -}, - {'name': 'no-disk-node', - 'heap.percent': '83', - }] - -EXPECTED_DISK_ALLOCATION = [{ - 'node': 'node-1', - 'shards': '113', - 'diskUsed': '67.2gb', - 'diskAvail': '188.6gb', - 'diskPercent': '26', - 'heapPercent': '57', -}, - {'node': 'UNASSIGNED', - 'shards': '2', - 'diskUsed': None, - 'diskAvail': None, - 'diskPercent': None - }] - -EXPECTED_NODE_STATS = [{'name': 'no-disk-node', 'heapPercent': '83'}] - -ES_CAT_INDICES = [{ - "index": "test_index", - "docs.count": "122674997", - "store.size": "14.9gb", - "creation.date.string": "2019-11-04T19:33:47.522Z" -}, - { - "index": "test_index_alias_1", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_alias_2", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_no_project", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_sv", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, - { - "index": "test_index_sv_wgs", - "docs.count": "672312", - "store.size": "233.4mb", - "creation.date.string": "2019-10-03T19:53:53.846Z" - }, -] - -ES_CAT_ALIAS = [ - { - "alias": "test_index_second", - "index": "test_index_alias_1" - }, - { - "alias": "test_index_second", - "index": "test_index_alias_2" - }] - -ES_INDEX_MAPPING = { - "test_index": { - "mappings": { - "_meta": { - "gencodeVersion": "25", - "genomeVersion": "38", - "sampleType": "WES", - "sourceFilePath": "test_index_file_path", - }, - "_all": { - "enabled": False - } - } - }, - "test_index_alias_1": { - "mappings": { - "_meta": { - "gencodeVersion": "25", - "hail_version": "0.2.24", - "genomeVersion": "37", - "sampleType": "WGS", - "sourceFilePath": "test_index_alias_1_path", - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_alias_2": { - "mappings": { - "_meta": { - "gencodeVersion": "19", - "genomeVersion": "37", - "sampleType": "WES", - "datasetType": "VARIANTS", - "sourceFilePath": "test_index_alias_2_path" - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_no_project": { - "mappings": { - "_meta": { - "gencodeVersion": "19", - "genomeVersion": "37", - "sampleType": "WGS", - "datasetType": "VARIANTS", - "sourceFilePath": "test_index_no_project_path" - }, - "_all": { - "enabled": False - }, - } - }, - "test_index_sv": { - "mappings": { - "_meta": { - "gencodeVersion": "29", - "genomeVersion": "38", - "sampleType": "WES", - "datasetType": "SV", - "sourceFilePath": "test_sv_index_path" - }, - } - }, - "test_index_sv_wgs": { - "mappings": { - "_meta": { - "gencodeVersion": "29", - "genomeVersion": "38", - "sampleType": "WGS", - "datasetType": "SV", - "sourceFilePath": "test_sv_wgs_index_path" - }, - } - }, -} - -TEST_INDEX_EXPECTED_DICT = { - "index": "test_index", - "sampleType": "WES", - "genomeVersion": "38", - "sourceFilePath": "test_index_file_path", - "docsCount": "122674997", - "storeSize": "14.9gb", - "creationDateString": "2019-11-04T19:33:47.522Z", - "gencodeVersion": "25", - "projects": [{'projectName': '1kg project n\xe5me with uni\xe7\xf8de', 'projectGuid': 'R0001_1kg'}] -} - -TEST_SV_INDEX_EXPECTED_DICT = { - "index": "test_index_sv", - "sampleType": "WES", - "genomeVersion": "38", - "sourceFilePath": "test_sv_index_path", - "docsCount": "672312", - "storeSize": "233.4mb", - "creationDateString": "2019-10-03T19:53:53.846Z", - "gencodeVersion": "29", - "datasetType": "SV", - "projects": [{'projectName': '1kg project n\xe5me with uni\xe7\xf8de', 'projectGuid': 'R0001_1kg'}] -} - -TEST_INDEX_NO_PROJECT_EXPECTED_DICT = { - "index": "test_index_no_project", - "sampleType": "WGS", - "genomeVersion": "37", - "sourceFilePath": "test_index_no_project_path", - "docsCount": "672312", - "storeSize": "233.4mb", - "creationDateString": "2019-10-03T19:53:53.846Z", - "datasetType": "VARIANTS", - "gencodeVersion": "19", - "projects": [] -} - -EXPECTED_ERRORS = [ - 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', - 'test_index_old does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', -] - RNA_TPM_MUSCLE_SAMPLE_GUID = 'RS000162_T_na19675_d2' RNA_OUTLIER_MUSCLE_SAMPLE_GUID = 'RS000172_E_na19675_d2' RNA_SPLICE_SAMPLE_GUID = 'RS000151_S_na19675_1' @@ -518,72 +306,6 @@ class DataManagerAPITest(AirtableTest): VCF_SAMPLES = VCF_SAMPLES SKIP_TDR = False - @urllib3_responses.activate - def test_elasticsearch_status(self): - url = reverse(elasticsearch_status) - self.check_data_manager_login(url) - - urllib3_responses.add_json( - '/_cat/allocation?format=json&h=node,shards,disk.avail,disk.used,disk.percent', ES_CAT_ALLOCATION) - urllib3_responses.add_json( - '/_cat/nodes?format=json&h=name,heap.percent', ES_CAT_NODES) - urllib3_responses.add_json( - '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) - urllib3_responses.add_json('/_cat/aliases?format=json&h=alias,index', ES_CAT_ALIAS) - urllib3_responses.add_json('/_all/_mapping', ES_INDEX_MAPPING) - - response = self.client.get(url) - self._assert_expected_es_status(response) - - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'indices', 'errors', 'diskStats', 'nodeStats'}) - - self.assertEqual(len(response_json['indices']), 6) - self.assertDictEqual(response_json['indices'][0], TEST_INDEX_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][3], TEST_INDEX_NO_PROJECT_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][4], TEST_SV_INDEX_EXPECTED_DICT) - - self.assertListEqual(response_json['errors'], EXPECTED_ERRORS) - - self.assertListEqual(response_json['diskStats'], EXPECTED_DISK_ALLOCATION) - self.assertListEqual(response_json['nodeStats'], EXPECTED_NODE_STATS) - - @urllib3_responses.activate - def test_delete_index(self): - url = reverse(delete_index) - self.check_data_manager_login(url) - - response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'test_index'})) - self._assert_expected_delete_active_index_response(response) - - urllib3_responses.add_json( - '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) - urllib3_responses.add_json('/_cat/aliases?format=json&h=alias,index', ES_CAT_ALIAS) - urllib3_responses.add_json('/_all/_mapping', ES_INDEX_MAPPING) - urllib3_responses.add(urllib3_responses.DELETE, '/unused_index') - - response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'unused_index'})) - self._assert_expected_delete_index_response(response) - - def _assert_expected_delete_active_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') - self.assertEqual(len(urllib3_responses.calls), 0) - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'indices'}) - self.assertEqual(len(response_json['indices']), 6) - self.assertDictEqual(response_json['indices'][0], TEST_INDEX_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][3], TEST_INDEX_NO_PROJECT_EXPECTED_DICT) - self.assertDictEqual(response_json['indices'][4], TEST_SV_INDEX_EXPECTED_DICT) - - self.assertEqual(urllib3_responses.calls[0].request.method, 'DELETE') - @mock.patch('seqr.views.apis.data_manager_api.KIBANA_ELASTICSEARCH_PASSWORD', 'abc123') @responses.activate def test_kibana_proxy(self): @@ -1750,17 +1472,6 @@ def _get_expected_read_file_subprocess_calls(self, file_name, sample_guid): (f'==> {gsutil_cat}', None), ] - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'elasticsearch_status is disabled without the elasticsearch backend') - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 500) - self.assertEqual(response.json()['error'], 'delete_index is disabled without the elasticsearch backend') - - def _assert_expected_delete_active_index_response(self, response): - self._assert_expected_delete_index_response(response) - def _assert_expected_get_projects_requests(self): pdo_filter = "OR(SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';')))" expected_filters = [ From 4066d0686a499ff277864dd11a0913b39445f9b3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:06:16 -0500 Subject: [PATCH 045/252] clean up --- seqr/utils/search/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 9f5de481a3..608687a8f0 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -52,14 +52,6 @@ class InvalidSearchException(Exception): MIN_MULTI_FAMILY_SEQR_AC = 5000 -def es_only(func): - def _wrapped(*args, **kwargs): - if not es_backend_enabled(): - raise ValueError(f'{func.__name__} is disabled without the elasticsearch backend') - return func(*args, **kwargs) - return _wrapped - - def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func From 10aa0af057323cbe20f390fa87c78e354a3c07dd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:28:55 -0500 Subject: [PATCH 046/252] clean up --- seqr/utils/search/elasticsearch/es_utils.py | 85 --------------------- 1 file changed, 85 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 628fa04dd4..5b9e848dd5 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -107,91 +107,6 @@ def get_index_metadata(index_name, client, include_fields=False, use_cache=True) return index_metadata -def _get_es_sample_ids(elasticsearch_index, sample_field, es_client): - s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index) - s = s.params(size=0) - s.aggs.bucket('sample_ids', elasticsearch_dsl.A('terms', field=sample_field, size=10000)) - response = s.execute() - return [agg['key'] for agg in response.aggregations.sample_ids.buckets] - - -def _get_samples_field(index_metadata): - return next((field for field in SAMPLE_FIELDS_LIST if field in index_metadata['fields'].keys())) - - -def _validate_index_metadata(index_metadata, elasticsearch_index, project=None, genome_version=None, - dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS): - metadata_fields = ['genomeVersion', 'sampleType', 'sourceFilePath'] - if any(field not in (index_metadata or {}) for field in metadata_fields): - raise ValueError("Index metadata must contain fields: {}".format(', '.join(metadata_fields))) - - sample_type = index_metadata['sampleType'] - if sample_type not in {choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES}: - raise ValueError("Sample type not supported: {}".format(sample_type)) - - if index_metadata['genomeVersion'] != (genome_version or project.genome_version): - raise ValueError('Index "{0}" has genome version {1} but this project uses version {2}'.format( - elasticsearch_index, index_metadata['genomeVersion'], project.genome_version - )) - - dataset_path = index_metadata['sourceFilePath'] - if not dataset_path.endswith(DATASET_FILE_EXTENSIONS): - raise ValueError("Variant call dataset path must end with {}".format(' or '.join(DATASET_FILE_EXTENSIONS))) - - index_dataset_type = EsSearch.get_index_metadata_dataset_type(index_metadata) - if index_dataset_type != dataset_type: - raise ValueError('Index "{0}" has dataset type {1} but expects {2}'.format( - elasticsearch_index, index_dataset_type, dataset_type - )) - - -def _get_es_meta(client, meta_type, fields, filter_rows=None): - return [{ - _to_camel_case(field.replace('.', '_')): o[field] for field in fields - } for o in getattr(client.cat, meta_type)(format="json", h=','.join(fields)) - if filter_rows is None or filter_rows(o)] - - -def _get_es_indices(client): - indices = _get_es_meta( - client, 'indices', ['index', 'docs.count', 'store.size', 'creation.date.string'], - filter_rows=lambda index: all( - not index['index'].startswith(omit_prefix) for omit_prefix in ['.', 'index_operations_log'])) - - aliases = defaultdict(list) - for alias in _get_es_meta(client, 'aliases', ['alias', 'index']): - aliases[alias['alias']].append(alias['index']) - - index_metadata = get_index_metadata('_all', client, use_cache=False) - - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index__isnull=False).select_related('individual__family__project') - - seqr_index_projects = defaultdict(lambda: defaultdict(set)) - es_projects = set() - for sample in active_samples: - for index_name in sample.elasticsearch_index.split(','): - project = sample.individual.family.project - es_projects.add(project) - if index_name in aliases: - for aliased_index_name in aliases[index_name]: - seqr_index_projects[aliased_index_name][project].add(sample.individual.guid) - else: - seqr_index_projects[index_name.rstrip('*')][project].add(sample.individual.guid) - - for index in indices: - index_name = index['index'] - index.update(index_metadata[index_name]) - - projects_for_index = [] - for index_prefix in list(seqr_index_projects.keys()): - if index_name.startswith(index_prefix): - projects_for_index += list(seqr_index_projects.pop(index_prefix).keys()) - index['projects'] = [ - {'projectGuid': project.guid, 'projectName': project.name} for project in projects_for_index] - - return indices, seqr_index_projects - - def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): variant_ids = sorted(set(variant_ids)) variants = EsSearch( From 1cb4928e7d786d37d8248435c023b9f4a3ea4fc2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:40:29 -0500 Subject: [PATCH 047/252] coverage --- .../commands/check_for_new_samples_from_pipeline.py | 1 + seqr/views/apis/data_manager_api_tests.py | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 2a413b2729..b663d689f2 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -438,6 +438,7 @@ def _update_projects_saved_variant_json(cls, families_by_project, dataset_type): for k, v in success.items(): if v > 0: logger.info(f' {k}: Updated {v} variants') + # TODO all un tested, refactor summary if skipped: logger.info(f'Skipped the following {len(skipped)} project with no saved variants: {", ".join(skipped)}') if len(error): diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 6bf9b5ae91..488f1110b1 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1464,14 +1464,6 @@ def _add_file_list_iter(self, file_list, stdout): self.mock_does_file_exist, self.mock_does_file_exist, self.mock_file_iter, self.mock_does_file_exist, self.mock_does_file_exist, self.mock_file_iter, ] - def _get_expected_read_file_subprocess_calls(self, file_name, sample_guid): - gsutil_cat = f'gsutil cat gs://seqr-scratch-temp/{file_name}/{sample_guid}.json.gz | gunzip -c -q - ' - self.mock_subprocess.assert_called_with(gsutil_cat, stdout=-1, stderr=-2, shell=True) # nosec - return [ - (f'==> gsutil ls gs://seqr-scratch-temp/{file_name}/{sample_guid}.json.gz', None), - (f'==> {gsutil_cat}', None), - ] - def _assert_expected_get_projects_requests(self): pdo_filter = "OR(SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';')))" expected_filters = [ From 0b3d86e62723d2938078a190b912ebcacdd1fecc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:44:57 -0500 Subject: [PATCH 048/252] clean up reloading logs --- .../check_for_new_samples_from_pipeline.py | 29 ++----------------- ...eck_for_new_samples_from_pipeline_tests.py | 3 -- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index b663d689f2..1043f6569c 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -416,35 +416,12 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, @classmethod def _update_projects_saved_variant_json(cls, families_by_project, dataset_type): - success = {} - skipped = {} - error = {} logger.info(f'Reloading saved variants in {len(families_by_project)} projects') for project, family_guids in families_by_project.items(): project_name = project.name - try: - updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, Sample.objects.filter(is_active=True)) - if updated_saved_variants is None: - skipped[project_name] = True - else: - success[project_name] = len(updated_saved_variants) - family_summary = f' in {len(family_guids)} families' if family_guids else '' - logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - except Exception as e: - logger.error(f'Error reloading variants in {project_name}: {e}') - error[project_name] = e - - logger.info('Reload Summary: ') - for k, v in success.items(): - if v > 0: - logger.info(f' {k}: Updated {v} variants') - # TODO all un tested, refactor summary - if skipped: - logger.info(f'Skipped the following {len(skipped)} project with no saved variants: {", ".join(skipped)}') - if len(error): - logger.info(f'{len(error)} failed projects') - for k, v in error.items(): - logger.info(f' {k}: {v}') + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, Sample.objects.filter(is_active=True)) + family_summary = f' in {len(family_guids)} families' if family_guids else '' + logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') @classmethod def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 91ad648dc3..d98808af85 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -369,8 +369,6 @@ class CheckNewSamplesTest(object): ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), ] def set_up(self): @@ -490,7 +488,6 @@ def _test_success_call(self, anvil_email_calls): ('Reloading saved variants in 2 projects', None), ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), - ('Reload Summary: ', None), ], }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', From 815768d5d4944a2f7551a13990b74f03b8d631d2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Feb 2026 17:49:33 -0500 Subject: [PATCH 049/252] codacy --- .../check_for_new_samples_from_pipeline.py | 16 +++++----------- seqr/utils/search/elasticsearch/es_utils.py | 3 --- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 1043f6569c..01a87b26a7 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -252,8 +252,11 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers except Exception as e: logger.error(f'Error updating individuals sample qc {run_version}: {e}') - # Reload saved variant JSON - cls._update_projects_saved_variant_json(families_by_project, clickhouse_dataset_type) + logger.info(f'Reloading saved variants in {len(families_by_project)} projects') + samples = Sample.objects.filter(is_active=True) + for project, family_guids in families_by_project.items(): + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) + logger.info(f'Updated {len(updated_saved_variants)} variants in {len(family_guids)} families for project {project.name}') @classmethod def _is_internal_project(cls, project): @@ -414,15 +417,6 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, updates.update({v.id: v for v in variants}) return updates - @classmethod - def _update_projects_saved_variant_json(cls, families_by_project, dataset_type): - logger.info(f'Reloading saved variants in {len(families_by_project)} projects') - for project, family_guids in families_by_project.items(): - project_name = project.name - updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, Sample.objects.filter(is_active=True)) - family_summary = f' in {len(family_guids)} families' if family_guids else '' - logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - @classmethod def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 5b9e848dd5..cd1d32c179 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -1,15 +1,12 @@ from collections import defaultdict from elasticsearch import Elasticsearch from elasticsearch.exceptions import ConnectionError as EsConnectionError, TransportError -import elasticsearch_dsl from urllib3.connectionpool import connection_from_url -from seqr.models import Sample from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page -from seqr.views.utils.json_utils import _to_camel_case from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER From 02148240398188726daf5af12f3198b34802628e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 10:52:38 -0500 Subject: [PATCH 050/252] add datsets reducer --- .../components/FamilyTable/IndividualRow.jsx | 28 +++++++++---------- ui/redux/selectors.js | 12 ++++++++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 3f796b1df5..2cf5870971 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -32,7 +32,7 @@ import { import { snakecaseToTitlecase } from 'shared/utils/stringUtils' import { updateIndividual } from 'redux/rootReducer' -import { getSamplesByGuid, getMmeSubmissionsByGuid, getIGVSamplesByFamilySampleIndividual } from 'redux/selectors' +import { getDatasetsByIndividual, getMmeSubmissionsByGuid, getIGVSamplesByFamilySampleIndividual } from 'redux/selectors' import { HPO_FORM_FIELDS } from '../HpoTerms' import { CASE_REVIEW_STATUS_MORE_INFO_NEEDED, CASE_REVIEW_STATUS_OPTIONS, CASE_REVIEW_TABLE_NAME, INDIVIDUAL_DETAIL_FIELDS, @@ -149,10 +149,10 @@ MmeStatusLabel.propTypes = { mmeSubmission: PropTypes.object, } -const DataDetails = React.memo(({ loadedSamples, individual, mmeSubmission }) => ( +const DataDetails = React.memo(({ loadedDatasets, individual, mmeSubmission }) => (
- {loadedSamples.map( - sample =>
, + {loadedDatasets.map( + dataset =>
, )} {individual.rnaSample && ( DataDetails.propTypes = { mmeSubmission: PropTypes.object, individual: PropTypes.object, - loadedSamples: PropTypes.arrayOf(PropTypes.object), + loadedDatasets: PropTypes.arrayOf(PropTypes.object), } const formatGene = gene => `${gene.gene} ${gene.comments ? ` (${gene.comments.trim()})` : ''}` @@ -548,7 +548,7 @@ class IndividualRow extends React.PureComponent { project: PropTypes.object.isRequired, individual: PropTypes.object.isRequired, mmeSubmission: PropTypes.object, - samplesByGuid: PropTypes.object.isRequired, + datasets: PropTypes.arrayOf(PropTypes.object).isRequired, alignmentSample: PropTypes.object, dispatchUpdateIndividual: PropTypes.func, dispatchUpdateIndividualIGV: PropTypes.func, @@ -575,17 +575,15 @@ class IndividualRow extends React.PureComponent { render() { const { - project, individual, mmeSubmission, samplesByGuid, tableName, updateIndividualPedigree, alignmentSample, + project, individual, mmeSubmission, datasets, tableName, updateIndividualPedigree, alignmentSample, dispatchUpdateIndividualIGV, } = this.props - const { displayName, sex, affected, createdDate, sampleGuids } = individual + const { displayName, sex, affected, createdDate } = individual - let loadedSamples = sampleGuids.map( - sampleGuid => samplesByGuid[sampleGuid], - ) - loadedSamples = orderBy(loadedSamples, [s => s.loadedDate], 'desc') // only show active or first/ last inactive samples - loadedSamples = loadedSamples.filter((sample, i) => sample.isActive || i === 0 || i === loadedSamples.length - 1) + const loadedDatasets = orderBy(datasets, [s => s.loadedDate], 'desc').filter( + ({ isActive }, i) => isActive || i === 0 || i === datasets.length - 1, + ) const leftContent = ( @@ -622,7 +620,7 @@ class IndividualRow extends React.PureComponent { const rightContent = editCaseReview ? : ( @@ -646,7 +644,7 @@ export { IndividualRow as IndividualRowComponent } const mapStateToProps = (state, ownProps) => ({ project: getCurrentProject(state), - samplesByGuid: getSamplesByGuid(state), + datasets: getDatasetsByIndividual(state)[ownProps.individual.individualGuid], mmeSubmission: getMmeSubmissionsByGuid(state)[ownProps.individual.mmeSubmissionGuid], alignmentSample: ( getIGVSamplesByFamilySampleIndividual(state)[ownProps.individual.familyGuid]?.alignment || {} diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 528efb37d7..7be8f1ee5f 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -67,6 +67,18 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { return acc }, {}) +export const getDatasetsByIndividual = createSelector( + getSamplesByGuid, + samplesByGuid => Object.values(samplesByGuid).reduce((acc, sample) => { + const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive, sampleType, datasetType, loadedDate: loadedDate.split('T')[0] }) + return acc + }, {}), +) + export const getNotesByFamilyType = createSelector( getFamilyNotesByGuid, notesByGuid => Object.values(notesByGuid).reduce((acc, note) => { From 16f86f91012313c02e084492e95a667d4acc0169 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:05:51 -0500 Subject: [PATCH 051/252] families datasets selector --- ui/pages/Project/selectors.js | 21 +++++---------------- ui/redux/selectors.js | 26 +++++++++++++++++++++----- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 2e36795d83..927d74f7df 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -17,11 +17,11 @@ import { import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' import { - getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getSamplesByGuid, getGenesById, getUser, + getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, - getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, + getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, } from 'redux/selectors' @@ -491,27 +491,16 @@ const getFamiliesExportData = createSelector( }], []), ) -const getSamplesByIndividual = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).reduce((acc, sample) => { - if (!acc[sample.individualGuid]) { - acc[sample.individualGuid] = [] - } - acc[sample.individualGuid].push(sample) - return acc - }, {}), -) - const getIndividualsExportData = createSelector( getVisibleFamiliesInSortedOrder, getSortedIndividualsByFamily, - getSamplesByIndividual, - (families, individualsByFamily, samplesByIndividual) => families.reduce((acc, family) => [ + getDatasetsByIndividual, + (families, individualsByFamily, datasetsByIndividual) => families.reduce((acc, family) => [ ...acc, ...(individualsByFamily[family.familyGuid] || []).map(individual => ({ ...individual, [FAMILY_FIELD_ID]: family.familyId, [INDIVIDUAL_HAS_DATA_FIELD]: ( - samplesByIndividual[individual.individualGuid] || []).some(({ isActive }) => isActive), + datasetsByIndividual[individual.individualGuid] || []).some(({ isActive }) => isActive), }))], []), ) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 7be8f1ee5f..ae2cbcadb1 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -11,7 +11,7 @@ export const getFamiliesByGuid = state => state.familiesByGuid export const getFamilyNotesByGuid = state => state.familyNotesByGuid export const getFamilyDetailsLoading = state => state.familyDetailsLoading export const getIndividualsByGuid = state => state.individualsByGuid -export const getSamplesByGuid = state => state.samplesByGuid +const getSamplesByGuid = state => state.samplesByGuid export const getIgvSamplesByGuid = state => state.igvSamplesByGuid export const getAnalysisGroupsByGuid = state => state.analysisGroupsByGuid export const getAnalysisGroupIsLoading = state => state.analysisGroupsLoading.isLoading @@ -57,6 +57,7 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce(( }, {}) export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid) export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid) +// TODO export const getSamplesGroupedByProjectGuid = createSelector(getSamplesByGuid, groupEntitiesByProjectGuid) const groupByFamilyGuid = objs => objs.reduce((acc, o) => { @@ -149,17 +150,32 @@ const getSortedSamples = createSelector( samplesByGuid => Object.values(samplesByGuid).sort((a, b) => a.loadedDate.localeCompare(b.loadedDate)), ) +// TODO export const getSamplesByFamily = createSelector( getSortedSamples, sortedSamples => groupByFamilyGuid(sortedSamples || []), ) +export const getDatasetsByFamily = createSelector( + getFamiliesByGuid, + getDatasetsByIndividual, + (familiesByGuid, datasetsByIndividual) => Object.entries(familiesByGuid).reduce( + (acc, [familyGuid, { individualGuids }]) => ({ + ...acc, + [familyGuid]: individualGuids.reduce( + (familyAcc, individualGuid) => [...familyAcc, ...(datasetsByIndividual[individualGuid] || [])], [], + ).sort((a, b) => a.loadedDate.localeCompare(b.loadedDate)), + }), + {}, + ), +) + export const getHasActiveSearchSampleByFamily = createSelector( - getSamplesByFamily, - samplesByFamily => Object.entries(samplesByFamily).reduce( - (acc, [familyGuid, familySamples]) => ({ + getDatasetsByFamily, + datasetsByFamily => Object.entries(datasetsByFamily).reduce( + (acc, [familyGuid, familyDatasets]) => ({ ...acc, - [familyGuid]: familySamples.some(({ isActive }) => isActive), + [familyGuid]: familyDatasets.some(({ isActive }) => isActive), }), {}, ), ) From 48803058d93137d7931feb64547968ae148a0e50 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:11:23 -0500 Subject: [PATCH 052/252] rename family filters --- ui/redux/selectors.js | 6 +++--- ui/shared/utils/constants.js | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index ae2cbcadb1..ccafafef78 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -457,8 +457,8 @@ const isAnalysedBy = (family, analysedByFilter, user, analysedByOptions) => { export const familyPassesFilters = createSelector( getUser, - getSamplesByFamily, - (user, samplesByFamily) => ( + getDatasetsByFamily, + (user, datasetsByFamily) => ( family, groupedFilters, analysedByOptions, categoryFilters = CATEGORY_FAMILY_FILTERS, ) => { if (groupedFilters.analysedBy && !isAnalysedBy(family, groupedFilters.analysedBy, user, analysedByOptions)) { @@ -468,7 +468,7 @@ export const familyPassesFilters = createSelector( const filters = categoryFilters[key]?.filter( opt => groupVals.includes(opt.value) && opt.createFilter, ).map(opt => opt.createFilter) - return !filters?.length || filters.some(filter => filter(family, user, samplesByFamily)) + return !filters?.length || filters.some(filter => filter(family, user, datasetsByFamily)) }) }, ) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 7e8ba0c9d2..1b54de2f0b 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -373,9 +373,9 @@ const SHOW_ANALYSED_BY_ME = 'SHOW_ANALYSED_BY_ME' const SHOW_ANALYSED = 'SHOW_ANALYSED' const SHOW_NOT_ANALYSED = 'SHOW_NOT_ANALYSED' -const hasMatchingSampleFilter = isMatchingSample => (family, user, samplesByFamily) => ( - (family.sampleTypes || samplesByFamily[family.familyGuid] || []).some( - sample => sample.isActive && isMatchingSample(sample), +const hasMatchingDatasetFilter = isMatchingDataset => (family, user, datasetsByFamily) => ( + (family.sampleTypes || datasetsByFamily[family.familyGuid] || []).some( + sample => sample.isActive && isMatchingDataset(sample), )) export const ASSIGNED_TO_ME_FILTER = { @@ -430,7 +430,7 @@ export const CATEGORY_FAMILY_FILTERS = { { value: SHOW_DATA_LOADED, name: 'Data Loaded', - createFilter: hasMatchingSampleFilter(() => true), + createFilter: hasMatchingDatasetFilter(() => true), }, { value: `${SHOW_DATA_LOADED}_RNA`, @@ -440,7 +440,7 @@ export const CATEGORY_FAMILY_FILTERS = { ...[DATASET_TYPE_SV_CALLS, DATASET_TYPE_MITO_CALLS].map(dataType => ({ value: `${SHOW_DATA_LOADED}_${dataType}`, name: `Data Loaded -${DATASET_TITLE_LOOKUP[dataType]}`, - createFilter: hasMatchingSampleFilter( + createFilter: hasMatchingDatasetFilter( ({ datasetType }) => datasetType === dataType, ), })), From a1b799fff484465b0753653024f2feb147784a44 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:17:08 -0500 Subject: [PATCH 053/252] shared usage --- ui/shared/components/panel/family/FamilyFields.jsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ui/shared/components/panel/family/FamilyFields.jsx b/ui/shared/components/panel/family/FamilyFields.jsx index 2f3834dbde..d9aafe3e27 100644 --- a/ui/shared/components/panel/family/FamilyFields.jsx +++ b/ui/shared/components/panel/family/FamilyFields.jsx @@ -7,7 +7,7 @@ import styled from 'styled-components' import { loadUserOptions, updateFamily } from 'redux/rootReducer' import { loadProjectAnalysisGroups } from 'redux/utils/reducerUtils' import { - getSamplesByFamily, + getDatasetsByFamily, getUserOptionsIsLoading, getHasActiveSearchSampleByFamily, getUserOptions, @@ -28,22 +28,22 @@ const NoWrap = styled.div` white-space: nowrap; ` -const BaseFirstSample = React.memo(({ firstFamilySample, compact, hasActiveVariantSample }) => ( +const BaseFirstSample = React.memo(({ firstFamilyDataset, compact, hasActiveVariantSample }) => ( )) BaseFirstSample.propTypes = { - firstFamilySample: PropTypes.object, + firstFamilyDataset: PropTypes.object, compact: PropTypes.bool, hasActiveVariantSample: PropTypes.bool, } const mapSampleDispatchToProps = (state, ownProps) => ({ - firstFamilySample: (getSamplesByFamily(state)[ownProps.familyGuid] || [])[0], + firstFamilyDataset: (getDatasetsByFamily(state)[ownProps.familyGuid] || [])[0], hasActiveVariantSample: getHasActiveSearchSampleByFamily(state)[ownProps.familyGuid], }) From 13e6747a0849a51821e10642a5206c764cc7aaac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:30:02 -0500 Subject: [PATCH 054/252] seperate actiuve dqataset selectors --- ui/pages/Project/selectors.js | 9 ++++---- ui/redux/selectors.js | 40 +++++++++++++++++++++++++++-------- ui/shared/utils/constants.js | 6 +++--- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 927d74f7df..73adc79756 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -21,7 +21,7 @@ import { getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, - getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getDatasetsByIndividual, + getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, } from 'redux/selectors' @@ -494,13 +494,12 @@ const getFamiliesExportData = createSelector( const getIndividualsExportData = createSelector( getVisibleFamiliesInSortedOrder, getSortedIndividualsByFamily, - getDatasetsByIndividual, - (families, individualsByFamily, datasetsByIndividual) => families.reduce((acc, family) => [ + getActiveDatasetsByIndividual, + (families, individualsByFamily, activeDatasetsByIndividual) => families.reduce((acc, family) => [ ...acc, ...(individualsByFamily[family.familyGuid] || []).map(individual => ({ ...individual, [FAMILY_FIELD_ID]: family.familyId, - [INDIVIDUAL_HAS_DATA_FIELD]: ( - datasetsByIndividual[individual.individualGuid] || []).some(({ isActive }) => isActive), + [INDIVIDUAL_HAS_DATA_FIELD]: (activeDatasetsByIndividual[individual.individualGuid] || []).length > 0, }))], []), ) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index ccafafef78..7628ec9a43 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -80,6 +80,16 @@ export const getDatasetsByIndividual = createSelector( }, {}), ) +export const getActiveDatasetsByIndividual = createSelector( + getDatasetsByIndividual, + datasetsByIndividual => Object.entries(datasetsByIndividual).reduce((acc, [individualGuid, datasets]) => ({ + ...acc, + [individualGuid]: datasets.filter(({ isActive }) => isActive).map( + ({ sampleType, datasetType, loadedDate }) => ({ sampleType, datasetType, loadedDate }), + ), + }), {}), +) + export const getNotesByFamilyType = createSelector( getFamilyNotesByGuid, notesByGuid => Object.values(notesByGuid).reduce((acc, note) => { @@ -156,6 +166,21 @@ export const getSamplesByFamily = createSelector( sortedSamples => groupByFamilyGuid(sortedSamples || []), ) +export const getActiveDatasetsByFamily = createSelector( + getFamiliesByGuid, + getActiveDatasetsByIndividual, + (familiesByGuid, activeDatasetsByIndividual) => Object.entries(familiesByGuid).reduce( + (acc, [familyGuid, { individualGuids }]) => ({ + ...acc, + [familyGuid]: individualGuids.reduce( + (familyAcc, individualGuid) => [...familyAcc, ...(activeDatasetsByIndividual[individualGuid] || [])], [], + ), + }), + {}, + ), +) + +// TODO current usage: firstFamilyDataset export const getDatasetsByFamily = createSelector( getFamiliesByGuid, getDatasetsByIndividual, @@ -171,12 +196,9 @@ export const getDatasetsByFamily = createSelector( ) export const getHasActiveSearchSampleByFamily = createSelector( - getDatasetsByFamily, - datasetsByFamily => Object.entries(datasetsByFamily).reduce( - (acc, [familyGuid, familyDatasets]) => ({ - ...acc, - [familyGuid]: familyDatasets.some(({ isActive }) => isActive), - }), {}, + getActiveDatasetsByFamily, + activeDatasetsByFamily => Object.entries(activeDatasetsByFamily).reduce( + (acc, [familyGuid, familyDatasets]) => ({ ...acc, [familyGuid]: familyDatasets.length > 0 }), {}, ), ) @@ -457,8 +479,8 @@ const isAnalysedBy = (family, analysedByFilter, user, analysedByOptions) => { export const familyPassesFilters = createSelector( getUser, - getDatasetsByFamily, - (user, datasetsByFamily) => ( + getActiveDatasetsByFamily, + (user, activeDatasetsByFamily) => ( family, groupedFilters, analysedByOptions, categoryFilters = CATEGORY_FAMILY_FILTERS, ) => { if (groupedFilters.analysedBy && !isAnalysedBy(family, groupedFilters.analysedBy, user, analysedByOptions)) { @@ -468,7 +490,7 @@ export const familyPassesFilters = createSelector( const filters = categoryFilters[key]?.filter( opt => groupVals.includes(opt.value) && opt.createFilter, ).map(opt => opt.createFilter) - return !filters?.length || filters.some(filter => filter(family, user, datasetsByFamily)) + return !filters?.length || filters.some(filter => filter(family, user, activeDatasetsByFamily)) }) }, ) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 1b54de2f0b..c6945bbe9a 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -373,9 +373,9 @@ const SHOW_ANALYSED_BY_ME = 'SHOW_ANALYSED_BY_ME' const SHOW_ANALYSED = 'SHOW_ANALYSED' const SHOW_NOT_ANALYSED = 'SHOW_NOT_ANALYSED' -const hasMatchingDatasetFilter = isMatchingDataset => (family, user, datasetsByFamily) => ( - (family.sampleTypes || datasetsByFamily[family.familyGuid] || []).some( - sample => sample.isActive && isMatchingDataset(sample), +const hasMatchingDatasetFilter = isMatchingDataset => (family, user, activeDatasetsByFamily) => ( + (family.sampleTypes || activeDatasetsByFamily[family.familyGuid] || []).some( + sample => isMatchingDataset(sample), )) export const ASSIGNED_TO_ME_FILTER = { From 09e6f78b437310b8873348f4af73397af11fb35a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:36:43 -0500 Subject: [PATCH 055/252] clean up family search selector --- ui/pages/Search/selectors.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 6b49def59a..b490f3af43 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -13,7 +13,7 @@ import { getGenesById, getSearchesByHash, getSamplesGroupedByProjectGuid, - getSamplesByFamily, + getActiveDatasetsByFamily, } from 'redux/selectors' import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' import { compareObjects } from 'shared/utils/sortUtils' @@ -196,8 +196,8 @@ export const getLocusListOptions = createListEqualSelector( }, ) -const getSampleDatasetTypes = samples => ([ - ...new Set((samples || []).filter(({ isActive }) => isActive).map(({ datasetType }) => datasetType)), +const getActiveDatasetTypes = datasets => ([ + ...new Set((datasets || []).map(({ datasetType }) => datasetType)), ]) export const getProjectDatasetTypes = createSelector( @@ -206,7 +206,10 @@ export const getProjectDatasetTypes = createSelector( (projectsByGuid, samplesByProjectGuid) => Object.values(projectsByGuid).reduce( (acc, { projectGuid, datasetTypes }) => ({ ...acc, - [projectGuid]: datasetTypes || getSampleDatasetTypes(Object.values(samplesByProjectGuid[projectGuid] || {})), + [projectGuid]: datasetTypes || getActiveDatasetTypes( + // TODO clean up + Object.values(samplesByProjectGuid[projectGuid] || {}).filter(({ isActive }) => isActive), + ), }), {}, ), ) @@ -214,10 +217,10 @@ export const getProjectDatasetTypes = createSelector( export const getDatasetTypes = createSelector( (state, props) => props.projectFamilies, getProjectDatasetTypes, - getSamplesByFamily, - (projectFamilies, projectDatasetTypes, samplesByFamily) => { + getActiveDatasetsByFamily, + (projectFamilies, projectDatasetTypes, datasetsByFamily) => { const isSingleFamily = (projectFamilies || []).length === 1 && projectFamilies[0].familyGuids?.length === 1 - const datasetTypes = isSingleFamily ? getSampleDatasetTypes(samplesByFamily[projectFamilies[0].familyGuids[0]]) : ( + const datasetTypes = isSingleFamily ? getActiveDatasetTypes(datasetsByFamily[projectFamilies[0].familyGuids[0]]) : ( projectFamilies || [] ).reduce((acc, { projectGuid }) => new Set([ ...acc, ...(projectDatasetTypes[projectGuid] || [])]), new Set()) From ed7d12ee9c2d16966979965c40dbcf34541a402d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:41:31 -0500 Subject: [PATCH 056/252] clean up histogram --- ui/pages/Project/selectors.js | 10 ++++------ ui/redux/selectors.js | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 73adc79756..96dcb3d7d9 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -20,7 +20,7 @@ import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, - getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, + getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, getActiveDatasetsByFamily, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, } from 'redux/selectors' @@ -134,11 +134,9 @@ export const getProjectAnalysisGroupFamilySizeHistogram = createSelector( export const getProjectAnalysisGroupDataLoadedFamilySizeHistogram = createSelector( getProjectAnalysisGroupFamiliesByGuid, - getSamplesByFamily, - (familiesByGuid, samplesByFamily) => getFamilySizeHistogram(Object.values(familiesByGuid).map(((family) => { - const sampleIndividuals = new Set((samplesByFamily[family.familyGuid] || []).filter( - sample => sample.isActive, - ).map(sample => sample.individualGuid)) + getActiveDatasetsByFamily, + (familiesByGuid, datasetsByFamily) => getFamilySizeHistogram(Object.values(datasetsByFamily).map(((family) => { + const sampleIndividuals = new Set((datasetsByFamily[family.familyGuid] || []).map(sample => sample.individualGuid)) const hasSampleParents = (family.parents || []).reduce( (acc, { individualGuid, maternalGuid, paternalGuid }) => { const hasSampleMaternal = sampleIndividuals.has(maternalGuid) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 7628ec9a43..a735114f1b 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -85,7 +85,7 @@ export const getActiveDatasetsByIndividual = createSelector( datasetsByIndividual => Object.entries(datasetsByIndividual).reduce((acc, [individualGuid, datasets]) => ({ ...acc, [individualGuid]: datasets.filter(({ isActive }) => isActive).map( - ({ sampleType, datasetType, loadedDate }) => ({ sampleType, datasetType, loadedDate }), + ({ sampleType, datasetType, loadedDate }) => ({ sampleType, datasetType, loadedDate, individualGuid }), ), }), {}), ) From 8e789a6f49f41c3ca7a72684e6f9b2061accea08 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 11:48:22 -0500 Subject: [PATCH 057/252] family export data --- ui/pages/Project/constants.js | 12 ++++++------ ui/pages/Project/selectors.js | 14 +++++++------- ui/redux/selectors.js | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ui/pages/Project/constants.js b/ui/pages/Project/constants.js index 4c2e4c11c6..4c6acae21c 100644 --- a/ui/pages/Project/constants.js +++ b/ui/pages/Project/constants.js @@ -367,17 +367,17 @@ export const FAMILY_SORT_OPTIONS = [ { value: SORT_BY_DATA_LOADED_DATE, name: 'Date Loaded', - createSortKeyGetter: (individualsByGuid, samplesByFamily) => (family) => { - const loadedSamples = samplesByFamily[family.familyGuid] || [] - return loadedSamples.length ? loadedSamples[loadedSamples.length - 1].loadedDate : '2000-01-01T01:00:00.000Z' + createSortKeyGetter: (individualsByGuid, datasetsByFamily) => (family) => { + const loadedDatasets = datasetsByFamily[family.familyGuid] || [] + return loadedDatasets.length ? loadedDatasets[loadedDatasets.length - 1].loadedDate : '2000-01-01T01:00:00.000Z' }, }, { value: SORT_BY_DATA_FIRST_LOADED_DATE, name: 'Date First Loaded', - createSortKeyGetter: (individualsByGuid, samplesByFamily) => (family) => { - const loadedSamples = samplesByFamily[family.familyGuid] || [] - return loadedSamples.length ? loadedSamples[0].loadedDate : '2000-01-01T01:00:00.000Z' + createSortKeyGetter: (individualsByGuid, datasetsByFamily) => (family) => { + const loadedDatasets = datasetsByFamily[family.familyGuid] || [] + return loadedDatasets.length ? loadedDatasets[0].loadedDate : '2000-01-01T01:00:00.000Z' }, }, { diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 96dcb3d7d9..f6415309b5 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -20,7 +20,7 @@ import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, - getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, getActiveDatasetsByFamily, + getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, getDatasetsByFamily, getActiveDatasetsByFamily, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, } from 'redux/selectors' @@ -454,16 +454,16 @@ export const getVisibleFamilies = createSelector( export const getVisibleFamiliesInSortedOrder = createSelector( getVisibleFamilies, getIndividualsByGuid, - getSamplesByFamily, + getDatasetsByFamily, getFamiliesSortOrder, getFamiliesSortDirection, - (visibleFamilies, individualsByGuid, samplesByFamily, familiesSortOrder, familiesSortDirection) => { + (visibleFamilies, individualsByGuid, datasetsByFamily, familiesSortOrder, familiesSortDirection) => { if (!familiesSortOrder || !FAMILY_SORT_LOOKUP[familiesSortOrder] || visibleFamilies.some(({ familyId }) => !familyId)) { // families have been loaded without any core fields return visibleFamilies } - const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, samplesByFamily) + const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, datasetsByFamily) return visibleFamilies.slice(0).sort((a, b) => getSortKey(a).localeCompare(getSortKey(b)) * familiesSortDirection) }, ) @@ -479,13 +479,13 @@ export const getEntityExportConfig = ({ projectName, tableName, fileName, fields const getFamiliesExportData = createSelector( getVisibleFamiliesInSortedOrder, - getSamplesByFamily, + getDatasetsByFamily, getNotesByFamilyType, - (visibleFamilies, samplesByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { + (visibleFamilies, datasetsByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { ...family, ...FAMILY_NOTES_FIELDS.reduce((noteAcc, { id, noteType }) => ( { ...noteAcc, [id]: (notesByFamilyType[family.familyGuid] || {})[noteType] }), {}), - [FAMILY_FIELD_FIRST_SAMPLE]: (samplesByFamily[family.familyGuid] || [])[0], + [FAMILY_FIELD_FIRST_SAMPLE]: (datasetsByFamily[family.familyGuid] || [])[0], }], []), ) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index a735114f1b..8a5e1ed310 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -180,7 +180,7 @@ export const getActiveDatasetsByFamily = createSelector( ), ) -// TODO current usage: firstFamilyDataset +// TODO current usage: firstFamilyDataset and last loaded date export const getDatasetsByFamily = createSelector( getFamiliesByGuid, getDatasetsByIndividual, From f3afb34067e2064989657af85d7af53bde95ad22 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 12:12:19 -0500 Subject: [PATCH 058/252] remove family samples selector --- ui/pages/Project/constants.js | 2 +- ui/pages/Project/selectors.js | 22 ++++++++++++---------- ui/redux/selectors.js | 11 ----------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/ui/pages/Project/constants.js b/ui/pages/Project/constants.js index 4c6acae21c..e4bbb3c15f 100644 --- a/ui/pages/Project/constants.js +++ b/ui/pages/Project/constants.js @@ -506,9 +506,9 @@ export const CASE_REVIEW_INDIVIDUAL_EXPORT_DATA = [ export const SAMPLE_EXPORT_DATA = [ { header: 'Family ID', field: 'familyId' }, { header: 'Individual ID', field: 'individualId' }, - { header: 'Sample ID', field: 'sampleId' }, { header: 'Loaded Date', field: 'loadedDate' }, { header: 'Sample Type', field: 'sampleType' }, + { header: 'Dataset Type', field: 'datasetType' }, ] export const TAG_FIELD_NAME = 'tags' diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index f6415309b5..53dad363a5 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -18,9 +18,9 @@ import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/str import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, - getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, + getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getDatasetsByIndividual, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, - getVariantTagsByGuid, getUserOptionsByUsername, getSamplesByFamily, getNotesByFamilyType, getDatasetsByFamily, getActiveDatasetsByFamily, + getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, getDatasetsByFamily, getActiveDatasetsByFamily, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, } from 'redux/selectors' @@ -503,14 +503,16 @@ const getIndividualsExportData = createSelector( const getSamplesExportData = createSelector( getVisibleFamiliesInSortedOrder, - getIndividualsByGuid, - getSamplesByFamily, - (visibleFamilies, individualsByGuid, samplesByFamily) => visibleFamilies.reduce((acc, family) => [ - ...acc, ...(samplesByFamily[family.familyGuid] || []).map(sample => ({ - ...sample, - [FAMILY_FIELD_ID]: family.familyId, - [INDIVIDUAL_FIELD_ID]: individualsByGuid[sample.individualGuid]?.individualId, - }))], []), + getSortedIndividualsByFamily, + getDatasetsByIndividual, + (visibleFamilies, individualsByFamily, datasetsByIndividual) => visibleFamilies.reduce((acc, family) => [ + ...acc, ...(individualsByFamily[family.familyGuid] || []).reduce((indivAcc, { individualGuid, individualId }) => ([ + ...indivAcc, ...(datasetsByIndividual[individualGuid] || []).map(sample => ({ + ...sample, + [FAMILY_FIELD_ID]: family.familyId, + [INDIVIDUAL_FIELD_ID]: individualId, + })), + ]), [])], []), ) export const getProjectExportUrls = createSelector( diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 8a5e1ed310..ec561dbd1b 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -155,17 +155,6 @@ export const getSortedIndividualsByFamily = createSelector( groupByFamilyGuid, ) -const getSortedSamples = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).sort((a, b) => a.loadedDate.localeCompare(b.loadedDate)), -) - -// TODO -export const getSamplesByFamily = createSelector( - getSortedSamples, - sortedSamples => groupByFamilyGuid(sortedSamples || []), -) - export const getActiveDatasetsByFamily = createSelector( getFamiliesByGuid, getActiveDatasetsByIndividual, From ba534086fac1fff5fc2d1a1897c0464ca7a11c39 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 12:42:58 -0500 Subject: [PATCH 059/252] scope down family samples selector --- .../components/FamilyTable/IndividualRow.jsx | 3 +-- ui/pages/Project/constants.js | 14 ++++++-------- ui/pages/Project/selectors.js | 17 +++++++++-------- ui/redux/selectors.js | 18 ++++++++++++------ .../components/panel/family/FamilyFields.jsx | 4 ++-- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 2cf5870971..16edcef0bf 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -5,7 +5,6 @@ import { connect } from 'react-redux' import { Link } from 'react-router-dom' import { Field } from 'react-final-form' import { Label, Popup, Form, Input, Loader } from 'semantic-ui-react' -import orderBy from 'lodash/orderBy' import { SearchInput, YearSelector, RadioButtonGroup, ButtonRadioGroup, Select } from 'shared/components/form/Inputs' import { helpLabel, validators } from 'shared/components/form/FormHelpers' @@ -581,7 +580,7 @@ class IndividualRow extends React.PureComponent { const { displayName, sex, affected, createdDate } = individual // only show active or first/ last inactive samples - const loadedDatasets = orderBy(datasets, [s => s.loadedDate], 'desc').filter( + const loadedDatasets = datasets.reverse().filter( ({ isActive }, i) => isActive || i === 0 || i === datasets.length - 1, ) diff --git a/ui/pages/Project/constants.js b/ui/pages/Project/constants.js index e4bbb3c15f..22854f020d 100644 --- a/ui/pages/Project/constants.js +++ b/ui/pages/Project/constants.js @@ -367,18 +367,16 @@ export const FAMILY_SORT_OPTIONS = [ { value: SORT_BY_DATA_LOADED_DATE, name: 'Date Loaded', - createSortKeyGetter: (individualsByGuid, datasetsByFamily) => (family) => { - const loadedDatasets = datasetsByFamily[family.familyGuid] || [] - return loadedDatasets.length ? loadedDatasets[loadedDatasets.length - 1].loadedDate : '2000-01-01T01:00:00.000Z' - }, + createSortKeyGetter: (individualsByGuid, minMaxDatasetsByFamily) => family => ( + minMaxDatasetsByFamily[family.familyGuid][1]?.loadedDate || '2000-01-01T01:00:00.000Z' + ), }, { value: SORT_BY_DATA_FIRST_LOADED_DATE, name: 'Date First Loaded', - createSortKeyGetter: (individualsByGuid, datasetsByFamily) => (family) => { - const loadedDatasets = datasetsByFamily[family.familyGuid] || [] - return loadedDatasets.length ? loadedDatasets[0].loadedDate : '2000-01-01T01:00:00.000Z' - }, + createSortKeyGetter: (individualsByGuid, minMaxDatasetsByFamily) => family => ( + minMaxDatasetsByFamily[family.familyGuid][0]?.loadedDate || '2000-01-01T01:00:00.000Z' + ), }, { value: SORT_BY_ANALYSIS_STATUS, diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 53dad363a5..172b8ad08b 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -18,11 +18,12 @@ import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/str import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, - getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getDatasetsByIndividual, + getAnalysisGroupsGroupedByProjectGuid, getSavedVariantsByGuid, getSortedIndividualsByFamily, getMmeResultsByGuid, getMmeSubmissionsByGuid, getHasActiveSearchSampleByFamily, getSelectableTagTypesByProject, - getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, getDatasetsByFamily, getActiveDatasetsByFamily, + getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, + getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, } from 'redux/selectors' import { @@ -454,16 +455,16 @@ export const getVisibleFamilies = createSelector( export const getVisibleFamiliesInSortedOrder = createSelector( getVisibleFamilies, getIndividualsByGuid, - getDatasetsByFamily, + getMinMaxDatasetsByFamily, getFamiliesSortOrder, getFamiliesSortDirection, - (visibleFamilies, individualsByGuid, datasetsByFamily, familiesSortOrder, familiesSortDirection) => { + (visibleFamilies, individualsByGuid, minMaxDatasetsByFamily, familiesSortOrder, familiesSortDirection) => { if (!familiesSortOrder || !FAMILY_SORT_LOOKUP[familiesSortOrder] || visibleFamilies.some(({ familyId }) => !familyId)) { // families have been loaded without any core fields return visibleFamilies } - const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, datasetsByFamily) + const getSortKey = FAMILY_SORT_LOOKUP[familiesSortOrder](individualsByGuid, minMaxDatasetsByFamily) return visibleFamilies.slice(0).sort((a, b) => getSortKey(a).localeCompare(getSortKey(b)) * familiesSortDirection) }, ) @@ -479,13 +480,13 @@ export const getEntityExportConfig = ({ projectName, tableName, fileName, fields const getFamiliesExportData = createSelector( getVisibleFamiliesInSortedOrder, - getDatasetsByFamily, + getMinMaxDatasetsByFamily, getNotesByFamilyType, - (visibleFamilies, datasetsByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { + (visibleFamilies, minMaxDatasetsByFamily, notesByFamilyType) => visibleFamilies.reduce((acc, family) => [...acc, { ...family, ...FAMILY_NOTES_FIELDS.reduce((noteAcc, { id, noteType }) => ( { ...noteAcc, [id]: (notesByFamilyType[family.familyGuid] || {})[noteType] }), {}), - [FAMILY_FIELD_FIRST_SAMPLE]: (datasetsByFamily[family.familyGuid] || [])[0], + [FAMILY_FIELD_FIRST_SAMPLE]: (minMaxDatasetsByFamily[family.familyGuid] || [])[0], }], []), ) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index ec561dbd1b..8466962b51 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -70,7 +70,9 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { export const getDatasetsByIndividual = createSelector( getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).reduce((acc, sample) => { + samplesByGuid => Object.values(samplesByGuid).sort( + (a, b) => a.loadedDate.localeCompare(b.loadedDate), + ).reduce((acc, sample) => { const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample if (!acc[individualGuid]) { acc[individualGuid] = [] @@ -169,16 +171,20 @@ export const getActiveDatasetsByFamily = createSelector( ), ) -// TODO current usage: firstFamilyDataset and last loaded date -export const getDatasetsByFamily = createSelector( +export const getMinMaxDatasetsByFamily = createSelector( getFamiliesByGuid, getDatasetsByIndividual, (familiesByGuid, datasetsByIndividual) => Object.entries(familiesByGuid).reduce( (acc, [familyGuid, { individualGuids }]) => ({ ...acc, - [familyGuid]: individualGuids.reduce( - (familyAcc, individualGuid) => [...familyAcc, ...(datasetsByIndividual[individualGuid] || [])], [], - ).sort((a, b) => a.loadedDate.localeCompare(b.loadedDate)), + [familyGuid]: individualGuids.reduce(([minLoaded, maxLoaded], individualGuid) => { + const individualDatasets = datasetsByIndividual[individualGuid] || [] + const endIndex = individualDatasets.length - 1 + return [ + minLoaded?.loadedDate < individualDatasets[0]?.loadedDate ? minLoaded : individualDatasets[0], + maxLoaded?.loadedDate > individualDatasets[endIndex]?.loadedDate ? maxLoaded : individualDatasets[endIndex], + ] + }, [null, null]), }), {}, ), diff --git a/ui/shared/components/panel/family/FamilyFields.jsx b/ui/shared/components/panel/family/FamilyFields.jsx index d9aafe3e27..a2a529f0c2 100644 --- a/ui/shared/components/panel/family/FamilyFields.jsx +++ b/ui/shared/components/panel/family/FamilyFields.jsx @@ -7,7 +7,7 @@ import styled from 'styled-components' import { loadUserOptions, updateFamily } from 'redux/rootReducer' import { loadProjectAnalysisGroups } from 'redux/utils/reducerUtils' import { - getDatasetsByFamily, + getMinMaxDatasetsByFamily, getUserOptionsIsLoading, getHasActiveSearchSampleByFamily, getUserOptions, @@ -43,7 +43,7 @@ BaseFirstSample.propTypes = { } const mapSampleDispatchToProps = (state, ownProps) => ({ - firstFamilyDataset: (getDatasetsByFamily(state)[ownProps.familyGuid] || [])[0], + firstFamilyDataset: (getMinMaxDatasetsByFamily(state)[ownProps.familyGuid] || [])[0], hasActiveVariantSample: getHasActiveSearchSampleByFamily(state)[ownProps.familyGuid], }) From bde77c4b17736f1d99f402e6726fb1c1c51dec0a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 15:35:53 -0500 Subject: [PATCH 060/252] remove final samples seelctor --- ui/pages/Search/selectors.js | 13 ++++++------- ui/redux/selectors.js | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index b490f3af43..fc75539a9d 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -12,7 +12,6 @@ import { getSearchFamiliesByHash, getGenesById, getSearchesByHash, - getSamplesGroupedByProjectGuid, getActiveDatasetsByFamily, } from 'redux/selectors' import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' @@ -202,14 +201,14 @@ const getActiveDatasetTypes = datasets => ([ export const getProjectDatasetTypes = createSelector( getProjectsByGuid, - getSamplesGroupedByProjectGuid, - (projectsByGuid, samplesByProjectGuid) => Object.values(projectsByGuid).reduce( + getFamiliesGroupedByProjectGuid, + getActiveDatasetsByFamily, + (projectsByGuid, familiesByProjectGuid, datasetsByFamily) => Object.values(projectsByGuid).reduce( (acc, { projectGuid, datasetTypes }) => ({ ...acc, - [projectGuid]: datasetTypes || getActiveDatasetTypes( - // TODO clean up - Object.values(samplesByProjectGuid[projectGuid] || {}).filter(({ isActive }) => isActive), - ), + [projectGuid]: datasetTypes || getActiveDatasetTypes(Object.keys(familiesByProjectGuid[projectGuid] || {}).reduce( + (projectAcc, familyGuid) => [...projectAcc, ...(datasetsByFamily[familyGuid] || [])], [], + )), }), {}, ), ) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 8466962b51..265e125698 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -57,8 +57,6 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce(( }, {}) export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid) export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid) -// TODO -export const getSamplesGroupedByProjectGuid = createSelector(getSamplesByGuid, groupEntitiesByProjectGuid) const groupByFamilyGuid = objs => objs.reduce((acc, o) => { if (!acc[o.familyGuid]) { From e7c8e7be83f9dd06360e3cc4e7298910bfc28db6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 15:50:09 -0500 Subject: [PATCH 061/252] fix bug with partial load families --- ui/pages/Project/selectors.test.js | 6 ++++-- ui/redux/selectors.js | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/pages/Project/selectors.test.js b/ui/pages/Project/selectors.test.js index 502fdc28f4..ce3ed0dc11 100644 --- a/ui/pages/Project/selectors.test.js +++ b/ui/pages/Project/selectors.test.js @@ -47,8 +47,10 @@ test('getProjectExportUrls', () => { expect(exportFamilies.length).toEqual(2) expect(exportFamilies[0].familyGuid).toEqual('F011652_2') expect(exportFamilies[1].familyGuid).toEqual('F011652_1') - expect(exportFamilies[0].firstSample.sampleGuid).toEqual('S2310656_wal_mc16200_mc16203') - expect(exportFamilies[1].firstSample).toEqual(undefined) + expect(exportFamilies[0].firstSample.datasetType).toEqual('SNV_INDEL') + expect(exportFamilies[0].firstSample.sampleType).toEqual('WES') + expect(exportFamilies[0].firstSample.loadedDate).toEqual('2018-03-13') + expect(exportFamilies[1].firstSample).toEqual(null) expect(exportFamilies[0].analysisNotes.length).toEqual(2) expect(exportFamilies[0].caseNotes).toEqual(undefined) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 265e125698..ad1f3fa955 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -177,6 +177,9 @@ export const getMinMaxDatasetsByFamily = createSelector( ...acc, [familyGuid]: individualGuids.reduce(([minLoaded, maxLoaded], individualGuid) => { const individualDatasets = datasetsByIndividual[individualGuid] || [] + if (individualDatasets.length === 0) { + return [minLoaded, maxLoaded] + } const endIndex = individualDatasets.length - 1 return [ minLoaded?.loadedDate < individualDatasets[0]?.loadedDate ? minLoaded : individualDatasets[0], From baacd0ad3b404ad913c79f3e9683ca4e90ef3dbb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Feb 2026 15:56:13 -0500 Subject: [PATCH 062/252] fix test --- ui/pages/Search/selectors.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/Search/selectors.test.js b/ui/pages/Search/selectors.test.js index 78b77f1bc7..b8dd139d22 100644 --- a/ui/pages/Search/selectors.test.js +++ b/ui/pages/Search/selectors.test.js @@ -64,7 +64,7 @@ test('getLocusListOptions', () => { test('getDatasetTypes', () => { expect(getDatasetTypes(STATE, { projectFamilies: [{ projectGuid: PROJECT_GUID }] })).toEqual('SNV_INDEL,SV') - expect(getDatasetTypes(STATE, EXPECTED_INITAL_SEARCH)).toEqual('SNV_INDEL') + expect(getDatasetTypes(STATE, EXPECTED_INITAL_SEARCH)).toEqual('SNV_INDEL,SV') }) test('getSearchGeneBreakdownValues', () => { From a8f88c5da14843de7ebd2a6a0dbd887c09eabbc9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Feb 2026 15:15:49 -0500 Subject: [PATCH 063/252] better selectors --- .../filters/ProjectFamiliesFilter.jsx | 3 ++- ui/pages/Search/selectors.js | 23 ++++--------------- ui/redux/selectors.js | 18 +++++++++++++++ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx b/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx index fce85b7b04..8790eaa5ee 100644 --- a/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx +++ b/ui/pages/Search/components/filters/ProjectFamiliesFilter.jsx @@ -6,10 +6,11 @@ import { Form, Button } from 'semantic-ui-react' import { getProjectsByGuid, getProjectAnalysisGroupFamilyGuidsByGuid, + getProjectDatasetTypes, } from 'redux/selectors' import { Multiselect, ButtonRadioGroup } from 'shared/components/form/Inputs' import { SOLVED_FAMILY_STATUS_OPTIONS } from 'shared/utils/constants' -import { getProjectFamilies, getSearchContextIsLoading, getFamilyOptions, getAnalysisGroupOptions, getProjectDatasetTypes } from '../../selectors' +import { getProjectFamilies, getSearchContextIsLoading, getFamilyOptions, getAnalysisGroupOptions } from '../../selectors' import { loadProjectFamiliesContext } from '../../reducers' import { ProjectFilter } from './ProjectsField' diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index fc75539a9d..0b646d900d 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -12,6 +12,7 @@ import { getSearchFamiliesByHash, getGenesById, getSearchesByHash, + getProjectDatasetTypes, getActiveDatasetsByFamily, } from 'redux/selectors' import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' @@ -195,31 +196,15 @@ export const getLocusListOptions = createListEqualSelector( }, ) -const getActiveDatasetTypes = datasets => ([ - ...new Set((datasets || []).map(({ datasetType }) => datasetType)), -]) - -export const getProjectDatasetTypes = createSelector( - getProjectsByGuid, - getFamiliesGroupedByProjectGuid, - getActiveDatasetsByFamily, - (projectsByGuid, familiesByProjectGuid, datasetsByFamily) => Object.values(projectsByGuid).reduce( - (acc, { projectGuid, datasetTypes }) => ({ - ...acc, - [projectGuid]: datasetTypes || getActiveDatasetTypes(Object.keys(familiesByProjectGuid[projectGuid] || {}).reduce( - (projectAcc, familyGuid) => [...projectAcc, ...(datasetsByFamily[familyGuid] || [])], [], - )), - }), {}, - ), -) - export const getDatasetTypes = createSelector( (state, props) => props.projectFamilies, getProjectDatasetTypes, getActiveDatasetsByFamily, (projectFamilies, projectDatasetTypes, datasetsByFamily) => { const isSingleFamily = (projectFamilies || []).length === 1 && projectFamilies[0].familyGuids?.length === 1 - const datasetTypes = isSingleFamily ? getActiveDatasetTypes(datasetsByFamily[projectFamilies[0].familyGuids[0]]) : ( + const datasetTypes = isSingleFamily ? [ + ...new Set((datasetsByFamily[projectFamilies[0].familyGuids[0]] || []).map(({ datasetType }) => datasetType)), + ] : ( projectFamilies || [] ).reduce((acc, { projectGuid }) => new Set([ ...acc, ...(projectDatasetTypes[projectGuid] || [])]), new Set()) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index ad1f3fa955..f97f91a3a2 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -66,6 +66,24 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { return acc }, {}) +export const getProjectDatasetTypes = createSelector( + getProjectsByGuid, + getSamplesByGuid, + (projectsByGuid, samplesByGuid) => Object.values(samplesByGuid).reduce( + (acc, { projectGuid, datasetType }) => { + const { datasetTypes } = projectsByGuid[projectGuid] || {} + if (datasetTypes) { + return { ...acc, [projectGuid]: datasetTypes } + } + if (!acc[projectGuid]) { + acc[projectGuid] = new Set() + } + acc[projectGuid].add()(datasetType) + return acc + }, {}, + ), +) + export const getDatasetsByIndividual = createSelector( getSamplesByGuid, samplesByGuid => Object.values(samplesByGuid).sort( From 86e40c517f2e706f34589e66b29e9dd6ef83973a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Feb 2026 15:22:09 -0500 Subject: [PATCH 064/252] properly handle project agg val --- ui/redux/selectors.js | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index f97f91a3a2..a6f9ec93aa 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -69,19 +69,24 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { export const getProjectDatasetTypes = createSelector( getProjectsByGuid, getSamplesByGuid, - (projectsByGuid, samplesByGuid) => Object.values(samplesByGuid).reduce( - (acc, { projectGuid, datasetType }) => { - const { datasetTypes } = projectsByGuid[projectGuid] || {} - if (datasetTypes) { - return { ...acc, [projectGuid]: datasetTypes } - } - if (!acc[projectGuid]) { - acc[projectGuid] = new Set() - } - acc[projectGuid].add()(datasetType) - return acc - }, {}, - ), + (projectsByGuid, samplesByGuid) => { + const projectDatasetTypes = Object.values(projectsByGuid).reduce( + (acc, { projectGuid, datasetTypes }) => ({ ...acc, [projectGuid]: datasetTypes }), {}, + ) + const sampleDatasetTypes = Object.values(samplesByGuid).reduce( + (acc, { projectGuid, datasetType, isActive }) => { + if (projectDatasetTypes[projectGuid] || !isActive) { + return acc + } + if (!acc[projectGuid]) { + acc[projectGuid] = new Set() + } + acc[projectGuid].add()(datasetType) + return acc + }, {}, + ) + return { ...projectDatasetTypes, ...sampleDatasetTypes } + }, ) export const getDatasetsByIndividual = createSelector( From 5c15bfcd8124a5dfe267306128ab76fbe8542392 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 26 Feb 2026 11:04:04 -0500 Subject: [PATCH 065/252] clean up --- seqr/views/utils/variant_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 03d7486c52..c361cde763 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -134,8 +134,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_data = _search_new_saved_variants(new_variant_keys, genome_version) else: new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} - # TODO fix call args - new_variant_data = _get_clickhouse_variant_keys(new_variant_data, primary_id_field, project=project, **kwargs) + new_variant_data = _get_clickhouse_variant_annotations(new_variant_data, primary_id_field, project=project, **kwargs) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): From 9950a567924afb98780136b9388d721c4cd1b3fd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 26 Feb 2026 12:05:03 -0500 Subject: [PATCH 066/252] fix tests --- .../fixtures/clickhouse_saved_variants.json | 34 +++---------------- seqr/views/apis/saved_variant_api_tests.py | 11 +++--- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index dc0247eb37..69197c9a56 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -95,54 +95,28 @@ ] } }, { - "model": "clickhouse_search.annotationsgrch37snvindel", + "model": "clickhouse_search.variantdetailsgrch37snvindel", "pk": 113, "fields": { "key": 113, - "xpos": 15062456358, - "chrom": "15", - "pos": 62456358, - "ref": "A", - "alt": "C", "variant_id": "15-62456358-A-C", "rsid": null, "caid": null, "lifted_over_chrom": null, "lifted_over_pos": null, - "hgmd": [null, null], - "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null], - "populations": [ - [0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ], - "sorted_transcript_consequences": [] + "transcripts": [] } }, { - "model": "clickhouse_search.annotationsgrch37snvindel", + "model": "clickhouse_search.variantdetailsgrch37snvindel", "pk": 114, "fields": { "key": 114, - "xpos": 15062456406, - "chrom": "15", - "pos": 62456406, - "ref": "G", - "alt": "A", "variant_id": "15-62456406-G-A", "rsid": null, "caid": null, "lifted_over_chrom": null, "lifted_over_pos": null, - "hgmd": [null, null], - "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null], - "populations": [ - [0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ], - "sorted_transcript_consequences": [] + "transcripts": [] } }, { "model": "clickhouse_search.entriesgrch37snvindel", diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 1d5f7c1fbf..7b106bc37b 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -40,7 +40,7 @@ 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', - 'mainTranscriptId': None, + 'mainTranscriptId': '', 'populations': {'callset': {'ac': 17, 'af': 0.607, 'an': 28}}, 'pos': 62456358, 'predictions': {'cadd': 12.34}, @@ -65,7 +65,7 @@ 'chrom': '15', 'genotypes': {}, 'genomeVersion': '37', - 'mainTranscriptId': None, + 'mainTranscriptId': '', 'populations': {'callset': {'ac': 1, 'af': 0.033, 'an': 8686}}, 'pos': 62456406, 'predictions': {'cadd': 13.56}, @@ -163,8 +163,7 @@ def test_saved_variant_data(self): variant = variants[VARIANT_GUID] fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), fields) - fields.update({'mainTranscriptId', 'mmeSubmissions'}) - self.assertSetEqual(set(variant.keys()), fields) + self.assertSetEqual(set(variant.keys()), {*fields, 'mmeSubmissions'}) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) self.assertSetEqual( @@ -372,9 +371,9 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), { - 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', *self.SAVED_VARIANT_38_DETAIL_FIELDS, + *fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): From afa8051156c08b160c91c396f45cce68202cfce9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 26 Feb 2026 14:37:32 -0500 Subject: [PATCH 067/252] helper clean up --- seqr/views/utils/variant_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index c361cde763..2a0e38df74 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -229,9 +229,7 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_ samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) search_variants_by_id = { - v['variantId']: v for v in _get_clickhouse_variants( - samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), family_variant_ids=family_variant_ids, user=user, genome_version=genome_version, - ) + v['variantId']: v for v in _get_clickhouse_variants(samples, families_by_id, family_variant_ids, genome_version) } new_variants = {} @@ -254,7 +252,7 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_ return new_variants -def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str = None, **kwargs) -> list[dict]: +def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: variant_data = _get_clickhouse_variant_annotations( {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, ) From 6eab1195ca06b29d1cd92db108238271ea7ef977 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 3 Mar 2026 13:53:54 -0500 Subject: [PATCH 068/252] cleanup shared helper --- .../commands/check_for_new_samples_from_pipeline.py | 7 +++---- .../commands/reload_saved_variant_genotypes.py | 11 +++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 7579f30e35..e98c02d7c4 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -253,9 +253,8 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') logger.info(f'Reloading saved variants in {len(families_by_project)} projects') - samples = Sample.objects.filter(is_active=True) for project, family_guids in families_by_project.items(): - updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples) + updated_saved_variants = cls._update_project_saved_variant_genotypes(project, family_guids, clickhouse_dataset_type) logger.info(f'Updated {len(updated_saved_variants)} variants in {len(family_guids)} families for project {project.name}') @classmethod @@ -391,7 +390,7 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) ) @staticmethod - def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, samples): + def _update_project_saved_variant_genotypes(project, family_guids, dataset_type): updates = {} for family_guid in family_guids: variant_models_by_key = { @@ -406,7 +405,7 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type, variants = [] genotypes_by_key = get_clickhouse_genotypes( project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), - samples, + samples=Sample.objects.filter(is_active=True), ) for key, genotypes in genotypes_by_key.items(): variant = variant_models_by_key[key] diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py index 8904d377bc..3cd146ac40 100644 --- a/seqr/management/commands/reload_saved_variant_genotypes.py +++ b/seqr/management/commands/reload_saved_variant_genotypes.py @@ -2,7 +2,6 @@ from django.core.management.base import BaseCommand from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes from seqr.models import Project, Family, Sample -from seqr.utils.search.utils import clickhouse_only logger = logging.getLogger(__name__) @@ -14,7 +13,6 @@ def add_arguments(self, parser): parser.add_argument('project') parser.add_argument('--family-guid', help='optional family to reload variants for') - @clickhouse_only def handle(self, *args, **options): project = Project.objects.get(guid=options['project']) family_guid = options['family_guid'] @@ -24,13 +22,10 @@ def handle(self, *args, **options): if family_guid: samples = samples.filter(individual__family__guid=family_guid) dataset_types = { - f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type: dataset_type + f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct() } - for clickhouse_dataset_type, dataset_type in sorted(dataset_types.items()): - update_project_saved_variant_genotypes( - project.id, project.genome_version, family_guids, project.guid, - samples=samples.filter(dataset_type=dataset_type), clickhouse_dataset_type=clickhouse_dataset_type, - ) + for dataset_type in sorted(dataset_types): + update_project_saved_variant_genotypes(project, family_guids, dataset_type) logger.info('Done') From eb2982b8dd2037537b0a6cba8605a9bad04fabb3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 3 Mar 2026 15:20:14 -0500 Subject: [PATCH 069/252] codacy --- seqr/views/apis/saved_variant_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 97071692a1..6bba030678 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -5,7 +5,7 @@ from django.urls.base import reverse from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict -from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Sample, Project +from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler From ea2cdbaeddf8116dc733b87d2245ad228b462b26 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 3 Mar 2026 15:59:45 -0500 Subject: [PATCH 070/252] fixx typo --- ui/redux/selectors.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index bae377c8f9..ea1daf9e45 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -80,7 +80,7 @@ export const getProjectDatasetTypes = createSelector( if (!acc[projectGuid]) { acc[projectGuid] = new Set() } - acc[projectGuid].add()(datasetType) + acc[projectGuid].add(datasetType) return acc }, {}, ) From 6b7ea895bbfcccb23f53dcc0f3884c4e87af718c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 4 Mar 2026 11:25:39 -0500 Subject: [PATCH 071/252] condense variant looku test --- clickhouse_search/search_tests.py | 61 ++++++++++++++++--------- seqr/utils/search/search_utils_tests.py | 41 ----------------- 2 files changed, 40 insertions(+), 62 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6ed311915b..9db43c915a 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -114,27 +114,28 @@ def _assert_expected_search(self, expected_results, gene_counts=None, inheritanc self.search_model.search['inheritance']['filter'] = inheritance_filter variants, total = query_variants(results_model, user=self.user, sort=sort) - encoded_variants = self._assert_expected_variants(variants, expected_results) - self.assertEqual(total, len(expected_results)) - self._assert_expected_search_cache(encoded_variants, total, cached_variant_fields, sort, results_model) + self._assert_expected_variants( + variants, expected_results, cache_key=f'search_results__{results_model.guid}__{sort}', sort=sort, + format_cached_variants=self._format_cached_variants, cached_variant_fields=cached_variant_fields, + ) if gene_counts: gene_counts_json = get_variant_query_gene_counts(results_model, self.user) self.assertDictEqual(gene_counts_json, gene_counts) - def _assert_expected_variants(self, variants, expected_results): + def _assert_expected_variants(self, variants, expected_results, cache_key, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) self.assertListEqual(encoded_variants, expected_results) - return encoded_variants + cached_variants = format_cached_variants(encoded_variants, **kwargs) if format_cached_variants else encoded_variants + self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) - def _assert_expected_search_cache(self, variants, total, cached_variant_fields, sort, results_model): + def _format_cached_variants(self, variants, cached_variant_fields=None): cached_variants = [ self._get_cached_variant(variant, (cached_variant_fields[i] if cached_variant_fields else None)) for i, variant in enumerate(variants) ] - results_cache = {'all_results': cached_variants, 'total_results': total} - self.assert_cached_results(results_cache, sort=sort, cache_key=f'search_results__{results_model.guid}__{sort}') + return {'all_results': cached_variants, 'total_results': len(variants)} @classmethod def _get_cached_variant(cls, variant, cached_variant_fields): @@ -769,12 +770,18 @@ def test_variant_lookup(self, mock_liftover): mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] variants = variant_lookup(self.user, '1-10439-AC-A', '38') - self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT]) + self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT], 'variant_lookup_results__1-10439-AC-A__38') with self.assertRaises(ObjectDoesNotExist) as cm: variant_lookup(self.user, '1-91511686-TCA-G', '38') self.assertEqual(str(cm.exception), 'Variant not present in seqr') + self.set_cache([VARIANT_LOOKUP_VARIANT]) + cached_variants = variant_lookup(self.user, '1-91511686-TCA-G', '38') + self.assertListEqual([VARIANT_LOOKUP_VARIANT], cached_variants) + self.mock_redis.get.assert_called_with('variant_lookup_results__1-91511686-TCA-G__38') + + self.set_cache(None) variants = variant_lookup(self.user, '7-143270172-A-G', '37') grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, @@ -782,14 +789,15 @@ def test_variant_lookup(self, mock_liftover): {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() ], key=lambda x: x['sampleId'], reverse=True)}, } - self._assert_expected_variants(variants, [grch37_lookup_variant]) + cache_key = 'variant_lookup_results__7-143270172-A-G__37' + self._assert_expected_variants(variants, [grch37_lookup_variant], cache_key) variants = variant_lookup(self.user, '7-143270172-A-G', '37', hom_only=True, affected_only=True) - self._assert_expected_variants(variants, [grch37_lookup_variant]) + self._assert_expected_variants(variants, [grch37_lookup_variant], f'{cache_key}__affected__hom') # Lookup works if variant is only present on a different build variants = variant_lookup(self.user, '7-143260172-A-G', '38') - self._assert_expected_variants(variants, [grch37_lookup_variant]) + self._assert_expected_variants(variants, [grch37_lookup_variant], 'variant_lookup_results__7-143260172-A-G__38') mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) @@ -802,7 +810,8 @@ def test_variant_lookup(self, mock_liftover): } del liftover_variant['liftedFamilyGuids'] variants = variant_lookup(self.user, '1-439-AC-A', '37') - self._assert_expected_variants(variants, [liftover_variant]) + cache_key = 'variant_lookup_results__1-439-AC-A__37' + self._assert_expected_variants(variants, [liftover_variant], cache_key) mock_liftover.assert_called_with('hg19', 'hg38') mock_convert_coordinate.assert_called_with('chr1', 439) @@ -814,9 +823,9 @@ def test_variant_lookup(self, mock_liftover): }, } variants = variant_lookup(self.user, '1-10439-AC-A', '38', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant]) + self._assert_expected_variants(variants, [hom_only_lookup_variant], 'variant_lookup_results__1-10439-AC-A__38__hom') variants = variant_lookup(self.user, '1-439-AC-A', '37', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant]) + self._assert_expected_variants(variants, [hom_only_lookup_variant], f'{cache_key}__hom') variants = variant_lookup(self.user, 'M-4429-G-A', '38') self._assert_expected_variants(variants, [{ @@ -824,14 +833,23 @@ def test_variant_lookup(self, mock_liftover): 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() ]}, - }]) + }], 'variant_lookup_results__M-4429-G-A__38') with self.assertRaises(ObjectDoesNotExist) as cm: variant_lookup(self.user, 'M-4429-G-A', '38', hom_only=True) self.assertEqual(str(cm.exception), 'Variant not present in seqr') + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') + self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') + + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') + self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) + cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' + self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT], cache_key) affected_only_lookup_variant = { **GCNV_LOOKUP_VARIANT, @@ -840,17 +858,18 @@ def test_variant_lookup(self, mock_liftover): }, } variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', affected_only=True) - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, affected_only_lookup_variant]) + self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, affected_only_lookup_variant], f'{cache_key}__affected') # reciprocal overlap does not meet the threshold for smaller events variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT]) + cache_key = 'variant_lookup_results__suffix_140608_DUP__38' + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT], cache_key) variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES', affected_only=True) - self._assert_expected_variants(variants, [affected_only_lookup_variant]) + self._assert_expected_variants(variants, [affected_only_lookup_variant], f'{cache_key}__affected') variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3]) + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3], 'variant_lookup_results__suffix_140593_DUP__38') def test_get_single_variant(self): self._set_single_family_search() diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 1c28c05cb3..59b05167c8 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -68,47 +68,6 @@ def setUp(self): ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_variant_lookup(self, mock_variant_lookup): - mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] - variants = variant_lookup(self.user, '1-10439-AC-A', '38', affected_only=True) - self.assertListEqual(variants, [VARIANT_LOOKUP_VARIANT]) - mock_variant_lookup.assert_called_with(self.user, '1-10439-AC-A', ('1', 10439, 'AC', 'A'), 'SNV_INDEL', None, '38', True, False) - cache_key = "variant_lookup_results__1-10439-AC-A__38" - self.assert_cached_results(variants, cache_key=f'{cache_key}__affected') - - mock_variant_lookup.reset_mock() - self.set_cache(variants) - cached_variant = variant_lookup(self.user, '1-10439-AC-A', '38') - self.assertListEqual(variants, cached_variant) - mock_variant_lookup.assert_not_called() - self.mock_redis.get.assert_called_with(cache_key) - - self.set_cache(None) - mock_variant_lookup.reset_mock() - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') - self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') - - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') - self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') - - mock_variant_lookup.return_value = [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT] - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', hom_only=True) - self.assertListEqual(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) - mock_variant_lookup.assert_called_with( - self.user, 'phase2_DEL_chr14_4640', None, 'SV', 'WGS', '38', False, True) - cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' - self.assert_cached_results(variants, cache_key=f'{cache_key}__hom') - - mock_variant_lookup.reset_mock() - self.set_cache(variants) - cached_variant = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self.assertListEqual(variants, cached_variant) - mock_variant_lookup.assert_not_called() - self.mock_redis.get.assert_called_with(cache_key) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') def test_get_single_variant(self, mock_get_variants_for_ids): mock_get_variants_for_ids.return_value = PARSED_VARIANTS[0] From ab2420ea4e7a54c2fcc0952944a5ed1cfd687010 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 4 Mar 2026 11:34:57 -0500 Subject: [PATCH 072/252] condense single search test --- clickhouse_search/search_tests.py | 10 ++++++--- seqr/utils/search/search_utils_tests.py | 28 ------------------------- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9db43c915a..bf1a4e2517 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -124,11 +124,15 @@ def _assert_expected_search(self, expected_results, gene_counts=None, inheritanc gene_counts_json = get_variant_query_gene_counts(results_model, self.user) self.assertDictEqual(gene_counts_json, gene_counts) - def _assert_expected_variants(self, variants, expected_results, cache_key, format_cached_variants=None, sort='xpos', **kwargs): + def _assert_expected_variants(self, variants, expected_results, cache_key=None, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) self.assertListEqual(encoded_variants, expected_results) - cached_variants = format_cached_variants(encoded_variants, **kwargs) if format_cached_variants else encoded_variants - self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) + if cache_key: + cached_variants = format_cached_variants(encoded_variants, **kwargs) if format_cached_variants else encoded_variants + self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) + else: + self.mock_redis.get.assert_not_called() + self.mock_redis.set.assert_not_called() def _format_cached_variants(self, variants, cached_variant_fields=None): cached_variants = [ diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 59b05167c8..d40cc583e7 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -68,34 +68,6 @@ def setUp(self): ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') - def test_get_single_variant(self, mock_get_variants_for_ids): - mock_get_variants_for_ids.return_value = PARSED_VARIANTS[0] - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self.assertDictEqual(variant, PARSED_VARIANTS[0]) - expected_samples = {s for s in self.affected_search_samples if s.guid not in NON_SNP_INDEL_SAMPLES and s.guid != FAMILY_3_SAMPLE} - self._assert_expected_get_single_variant_call( - mock_get_variants_for_ids, '2-103343353-GAGA-G', ('2', 103343353, 'GAGA', 'G'), expected_samples, - ) - - get_single_variant(family, 'prefix_19107_DEL', user=self.user) - expected_samples = { - s for s in self.search_samples if s.guid in ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] - } - self._assert_expected_get_single_variant_call( - mock_get_variants_for_ids, 'prefix_19107_DEL', None, expected_samples, dataset_type='SV', - ) - - mock_get_variants_for_ids.return_value = [] - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G') - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - def _assert_expected_get_single_variant_call(self, mock_call, variant_id, parsed_variant_id, expected_samples, dataset_type='SNV_INDEL'): - mock_call.assert_called_with(variant_id, parsed_variant_id, mock.ANY, '37', dataset_type) - self.assertSetEqual(set(mock_call.call_args.args[2]), expected_samples) - @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def _test_invalid_search_params(self, search_func): From cbd42aec8df9c106730495370986ec19415d1198 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 4 Mar 2026 14:11:08 -0500 Subject: [PATCH 073/252] clean up --- seqr/utils/search/search_utils_tests.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index d40cc583e7..52f270b88d 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -70,51 +70,51 @@ def setUp(self): @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) - def _test_invalid_search_params(self, search_func): + def test_invalid_search_params(self): with self.assertRaises(InvalidSearchException) as cm: query_variants(self.results_model, user=self.user, page=200) self.assertEqual(str(cm.exception), 'Unable to load more than 10000 variants (20000 requested)') self.search_model.search['locus'] = {'rawVariantItems': 'chr2-A-C'} with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid variants: chr2-A-C') self.search_model.search['locus']['rawVariantItems'] = 'rs9876,chr2-1234-A-C' with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid variants: rs9876') self.search_model.search['locus']['rawItems'] = 'chr27:1234-5678,2:40-400000000, ENSG00012345' with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345') self.search_model.search['locus']['rawItems'] = '1:1-1000, 2:2000-3000, 3:4000-5000' with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Too many genes/intervals') build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' self.search_model.search['locus']['rawItems'] = build_specific_genes with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') self.search_model.search['exclude'] = self.search_model.search['locus'] with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Cannot specify both Location and Excluded Genes/Intervals') self.search_model.search['locus'] = {} with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') self.search_model.search['pathogenicity'] = {'clinvar': ['pathogenic', 'vus']} self.search_model.search['exclude'] = {'clinvar': ['benign', 'vus']} with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'ClinVar pathogenicity vus is both included and excluded') self.search_model.search['exclude'] = {} @@ -190,7 +190,7 @@ def _test_invalid_search_params(self, search_func): self.results_model.families.set(Family.objects.filter(guid='F000014_14')) self.search_model.search['locus']['rawItems'] = build_specific_genes with self.assertRaises(InvalidSearchException) as cm: - search_func(self.results_model, user=self.user) + query_variants(self.results_model, user=self.user) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') def _test_invalid_no_location_search_params(self): @@ -217,7 +217,6 @@ def test_invalid_search_query_variants(self): query_variants(self.results_model, page=1, num_results=2, load_all=True) self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') - self._test_invalid_search_params(query_variants) def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', @@ -502,9 +501,6 @@ def _assert_expected_cached_variants(self, variants, num_results): self.PARSED_CACHED_VARIANTS[:num_results], ) - def test_invalid_search_get_variant_query_gene_counts(self): - self._test_invalid_search_params(get_variant_query_gene_counts) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') def test_get_variant_query_gene_counts(self, mock_get_variants): gene_agg_all_results = self.CACHED_VARIANTS + [format_cached_variant(PROJECT_2_VARIANT2)] From 2a6d5095efea3d43f81e3bcd0aac21b9a4534f2b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 13:02:01 -0500 Subject: [PATCH 074/252] move coe invalid search tests; --- clickhouse_search/search_tests.py | 33 +++++++++++++++++++- seqr/utils/search/search_utils_tests.py | 41 +------------------------ seqr/utils/search/utils.py | 6 +--- 3 files changed, 34 insertions(+), 46 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index bf1a4e2517..bcd4a5f43b 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -752,10 +752,41 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) + @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) + @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([]) + self._assert_expected_search([], locus={'rawVariantItems': 'chr2-A-C'}) + self.assertEqual(str(cm.exception), 'Invalid variants: chr2-A-C') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawVariantItems': 'rs9876,chr2-1234-A-C'}) + self.assertEqual(str(cm.exception), 'Invalid variants: rs9876') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': 'chr27:1234-5678,2:40-400000000, ENSG00012345'}) + self.assertEqual(str(cm.exception), 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': '1:1-1000, 2:2000-3000, 3:4000-5000'}) + self.assertEqual(str(cm.exception), 'Too many genes/intervals') + + build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': build_specific_genes}) + self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], exclude={'rawItems': build_specific_genes}) + self.assertEqual(str(cm.exception), 'Cannot specify both Location and Excluded Genes/Intervals') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus=None) + self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], exclude=None) self.assertEqual(str(cm.exception),'This search returned too many results') Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 52f270b88d..6b59482b17 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -71,45 +71,6 @@ def setUp(self): @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def test_invalid_search_params(self): - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user, page=200) - self.assertEqual(str(cm.exception), 'Unable to load more than 10000 variants (20000 requested)') - - self.search_model.search['locus'] = {'rawVariantItems': 'chr2-A-C'} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid variants: chr2-A-C') - - self.search_model.search['locus']['rawVariantItems'] = 'rs9876,chr2-1234-A-C' - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid variants: rs9876') - - self.search_model.search['locus']['rawItems'] = 'chr27:1234-5678,2:40-400000000, ENSG00012345' - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345') - - self.search_model.search['locus']['rawItems'] = '1:1-1000, 2:2000-3000, 3:4000-5000' - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Too many genes/intervals') - - build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' - self.search_model.search['locus']['rawItems'] = build_specific_genes - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') - - self.search_model.search['exclude'] = self.search_model.search['locus'] - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Cannot specify both Location and Excluded Genes/Intervals') - - self.search_model.search['locus'] = {} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') self.search_model.search['pathogenicity'] = {'clinvar': ['pathogenic', 'vus']} self.search_model.search['exclude'] = {'clinvar': ['benign', 'vus']} @@ -191,7 +152,7 @@ def test_invalid_search_params(self): self.search_model.search['locus']['rawItems'] = build_specific_genes with self.assertRaises(InvalidSearchException) as cm: query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') def _test_invalid_no_location_search_params(self): self.search_model.search['inheritance'] = {} diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0f8c4c2a45..2b3aeb3b4e 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -10,8 +10,7 @@ from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ - MAX_VARIANTS + MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ es_backend_enabled, ping_kibana from seqr.utils.gene_utils import parse_locus_list_items @@ -197,9 +196,6 @@ def _get_previous_search_results(search_model, sort, page, num_results, load_all if total_results is not None: end_index = min(end_index, total_results) - if end_index > MAX_VARIANTS: - raise InvalidSearchException(f'Unable to load more than {MAX_VARIANTS} variants ({end_index} requested)') - cached_page = None loaded_results = previous_search_results.get('all_results') or [] if len(loaded_results) >= end_index: From f6e0aecc782cd4a147d091baccf370175f4a2c84 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 13:59:09 -0500 Subject: [PATCH 075/252] move some invalid search tests --- clickhouse_search/search_tests.py | 12 ++++++++++++ seqr/utils/search/search_utils_tests.py | 19 ------------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index bcd4a5f43b..50527e1ab6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -789,6 +789,18 @@ def test_invalid_search(self): self._assert_expected_search([], exclude=None) self.assertEqual(str(cm.exception),'This search returned too many results') + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}) + self.assertEqual(str(cm.exception),'ClinVar pathogenicity vus is both included and excluded') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], exclude=None, inheritance_mode='recessive') + self.assertEqual(str(cm.exception),'Annotations must be specified to search for compound heterozygous variants') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive') + self.assertEqual(str(cm.exception),'Location must be specified to search for compound heterozygous variants across many families') + Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 6b59482b17..da0686ac16 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -72,25 +72,6 @@ def setUp(self): @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def test_invalid_search_params(self): - self.search_model.search['pathogenicity'] = {'clinvar': ['pathogenic', 'vus']} - self.search_model.search['exclude'] = {'clinvar': ['benign', 'vus']} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'ClinVar pathogenicity vus is both included and excluded') - - self.search_model.search['exclude'] = {} - self.search_model.search['inheritance'] = {'mode': 'recessive'} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Annotations must be specified to search for compound heterozygous variants') - - self.search_model.search['annotations'] = {'frameshift': ['frameshift_variant']} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Location must be specified to search for compound heterozygous variants across many families') - self.results_model.families.set([family for family in self.families if family.guid == 'F000005_5']) with self.assertRaises(InvalidSearchException) as cm: query_variants(self.results_model) From c7f5e685469457e7b55a7f7abd7138e9ee3e1b6a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 14:24:14 -0500 Subject: [PATCH 076/252] delete es code --- ...eck_for_new_samples_from_pipeline_tests.py | 2 - seqr/utils/search/elasticsearch/__init__.py | 0 seqr/utils/search/elasticsearch/constants.py | 418 ----- seqr/utils/search/elasticsearch/es_search.py | 358 ----- seqr/utils/search/elasticsearch/es_utils.py | 175 --- .../search/elasticsearch/es_utils_tests.py | 1343 ----------------- seqr/views/apis/family_api_tests.py | 3 +- seqr/views/utils/test_utils.py | 5 - 8 files changed, 1 insertion(+), 2303 deletions(-) delete mode 100644 seqr/utils/search/elasticsearch/__init__.py delete mode 100644 seqr/utils/search/elasticsearch/constants.py delete mode 100644 seqr/utils/search/elasticsearch/es_search.py delete mode 100644 seqr/utils/search/elasticsearch/es_utils.py delete mode 100644 seqr/utils/search/elasticsearch/es_utils_tests.py diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 55b8cd3d8e..55e19a2553 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -735,8 +735,6 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] databases = '__all__' - ES_HOSTNAME = '' - MOCK_DATA_DIR = '/seqr/seqr-hail-search-data' PROJECT_EMAIL_TEXT = TEXT_EMAIL_TEMPLATE.format(1, 'WES', 'Non-Analyst Project') PROJECT_EMAIL_HTML = HTML_EMAIL_TEMAPLTE.format(1, 'WES', EXTERNAL_PROJECT_GUID, 'Non-Analyst Project') diff --git a/seqr/utils/search/elasticsearch/__init__.py b/seqr/utils/search/elasticsearch/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/seqr/utils/search/elasticsearch/constants.py b/seqr/utils/search/elasticsearch/constants.py deleted file mode 100644 index 250d2f3b40..0000000000 --- a/seqr/utils/search/elasticsearch/constants.py +++ /dev/null @@ -1,418 +0,0 @@ -from django.db.models import Min - -from reference_data.models import Omim, GeneConstraint -from seqr.models import Sample, PhenotypePrioritization -from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, \ - PATHOGENICTY_HGMD_SORT_KEY, PRIORITIZED_GENE_SORT - - -MAX_COMPOUND_HET_GENES = 1000 -MAX_INDEX_NAME_LENGTH = 4000 -MAX_SEARCH_CLAUSES = 1024 -MAX_INDEX_SEARCHES = 75 -PREFILTER_SEARCH_SIZE = 200 - -GENOTYPE_QUERY_MAP = { - REF_REF: {'not_allowed_num_alt': ['samples_no_call', 'samples_num_alt_1', 'samples_num_alt_2', 'samples']}, - REF_ALT: {'allowed_num_alt': ['samples_num_alt_1', 'samples']}, - ALT_ALT: {'allowed_num_alt': ['samples_num_alt_2', 'samples_cn_0', 'samples_cn_2', 'samples_cn_gte_4']}, - HAS_ALT: {'allowed_num_alt': ['samples_num_alt_1', 'samples_num_alt_2', 'samples']}, - HAS_REF: { - 'not_allowed_num_alt': ['samples_no_call', 'samples_num_alt_2', 'samples_cn_0', 'samples_cn_gte_4'], - }, -} - -PATH_FREQ_OVERRIDE_CUTOFF = 0.05 - -CLINVAR_PATH_SIGNIFICANCES = {'pathogenic', 'likely_pathogenic'} - -HGMD_CLASS_MAP = { - 'disease_causing': ['DM'], - 'likely_disease_causing': ['DM?'], - 'hgmd_other': ['DP', 'DFP', 'FP', 'FTV'], -} - -POPULATIONS = { - 'sv_callset': { - 'AF': 'sf', - 'filter_AF': [], - 'AC': 'sc', - 'AN': 'sn', - }, - 'callset': { - 'AF': 'AF', - 'filter_AF': [], - 'AC': 'AC', - 'AN': 'AN', - 'Hom': 'homozygote_count', - }, - 'topmed': { - 'filter_AF': [], - 'Het': None, - }, - 'exac': { - 'filter_AF': ['exac_AF_POPMAX'], - 'AC': 'exac_AC_Adj', - 'AN': 'exac_AN_Adj', - 'Hom': 'exac_AC_Hom', - 'Hemi': 'exac_AC_Hemi', - }, - 'gnomad_exomes': { - 'filter_AF': ['gnomad_exomes_AF_POPMAX_OR_GLOBAL'], - }, - 'gnomad_genomes': { - 'filter_AF': ['gnomad_genomes_AF_POPMAX_OR_GLOBAL'], - }, - 'gnomad_svs': {}, - 'callset_heteroplasmy': { - 'AN': 'AN', - 'AC': 'AC_het', - 'AF': 'AF_het', - }, - 'gnomad_mito': {'max_hl': None}, - 'gnomad_mito_heteroplasmy': { - 'AN': 'gnomad_mito_AN', - 'AC': 'gnomad_mito_AC_het', - 'AF': 'gnomad_mito_AF_het', - 'max_hl': 'gnomad_mito_max_hl' - }, - 'helix': {'max_hl': None}, - 'helix_heteroplasmy': { - 'AN': 'helix_AN', - 'AC': 'helix_AC_het', - 'AF': 'helix_AF_het', - 'max_hl': 'helix_max_hl', - } -} - -POPULATION_FIELD_CONFIGS = { - 'AF': {'format_value': float}, - 'filter_AF': {'format_value': lambda val: float(val) if val is not None else None, 'default_value': None}, - 'AC': {}, - 'AN': {}, - 'Hom': {}, - 'Hemi': {}, - 'Het': {}, - 'ID': {'format_value': str, 'default_value': None}, - 'max_hl': {'format_value': float}, -} -for population, pop_config in POPULATIONS.items(): - for freq_field in POPULATION_FIELD_CONFIGS.keys(): - if freq_field not in pop_config: - freq_suffix = freq_field - pop_config[freq_field] = '{}_{}'.format(population, freq_suffix) - -DEFAULT_POP_FIELD_CONFIG = { - 'format_value': int, - 'default_value': 0, -} -POPULATION_RESPONSE_FIELD_CONFIGS = {k: dict(DEFAULT_POP_FIELD_CONFIG, **v) for k, v in POPULATION_FIELD_CONFIGS.items()} - -CLINVAR_SORT = { - '_script': { - 'type': 'number', - 'script': { - 'source': """ - if (!doc.containsKey('clinvar_clinical_significance') || doc['clinvar_clinical_significance'].empty ) { - return 2; - } - String clinsig = doc['clinvar_clinical_significance'].value; - if (clinsig.indexOf('Pathogenic') >= 0 || clinsig.indexOf('Likely_pathogenic') >= 0) { - return 0; - } else if (clinsig.indexOf('Benign') >= 0 || clinsig.indexOf('Likely_benign') >= 0) { - return 3; - } - return 1; - """ - } - } -} - - -def _get_phenotype_priority_ranks_by_gene(samples, *args): - families = {s.individual.family for s in samples} - family_ranks = PhenotypePrioritization.objects.filter( - individual__family=list(families)[0], rank__lte=100).values('gene_id').annotate(min_rank=Min('rank')) - return {agg['gene_id']: agg['min_rank'] for agg in family_ranks} - - -SORT_FIELDS = { - PATHOGENICTY_SORT_KEY: [CLINVAR_SORT], - PATHOGENICTY_HGMD_SORT_KEY: [CLINVAR_SORT, { - '_script': { - 'type': 'number', - 'script': { - 'source': "(doc.containsKey('hgmd_class') && !doc['hgmd_class'].empty && doc['hgmd_class'].value == 'DM') ? 0 : 1" - } - } - }], - 'in_omim': [{ - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'omim_gene_ids': lambda *args: [omim.gene.gene_id for omim in Omim.objects.filter( - phenotype_mim_number__isnull=False, gene__isnull=False).only('gene__gene_id')] - }, - 'source': "(doc.containsKey('mainTranscript_gene_id') && !doc['mainTranscript_gene_id'].empty && params.omim_gene_ids.contains(doc['mainTranscript_gene_id'].value)) ? 0 : 1", - } - } - }, { - '_script': { - 'type': 'number', - 'script': { - 'source': """ - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.omim_gene_ids.contains(doc['geneIds'][i])) { - return 0; - } - } - return 1 - """ - } - } - }], - PRIORITIZED_GENE_SORT: [{ - '_script': { - 'type': 'number', - 'script': { - 'params': { - 'prioritized_ranks_by_gene': _get_phenotype_priority_ranks_by_gene, - }, - 'source': """ - int min_rank = 1000000; - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.prioritized_ranks_by_gene.getOrDefault(doc['geneIds'][i], 1000000) < min_rank) { - min_rank = params.prioritized_ranks_by_gene.get(doc['geneIds'][i]) - } - } - return min_rank; - """ - } - } - }], - 'protein_consequence': [{ - '_script': { - 'type': 'number', - 'script': { - 'source': "doc.containsKey('svType') ? 4.5 : (doc['mainTranscript_major_consequence_rank'].empty ? 1000 : doc['mainTranscript_major_consequence_rank'].value)" - } - } - }], - 'constraint': [{ - '_script': { - 'order': 'asc', - 'type': 'number', - 'script': { - 'params': { - 'constraint_ranks_by_gene': lambda *args: { - constraint.gene.gene_id: constraint.mis_z_rank + constraint.pLI_rank - for constraint in GeneConstraint.objects.all().only('gene__gene_id', 'mis_z_rank', 'pLI_rank')} - }, - 'source': """ - int min = 1000000000; - for (int i = 0; i < doc['geneIds'].length; ++i) { - if (params.constraint_ranks_by_gene.getOrDefault(doc['geneIds'][i], 1000000000) < min) { - min = params.constraint_ranks_by_gene.get(doc['geneIds'][i]) - } - } - return min - """ - } - }, - }], - 'size': [{ - '_script': { - 'type': 'number', - 'script': { - 'source': "(doc.containsKey('svType') && (doc['svType'].value == 'BND' || doc['svType'].value == 'CTX')) ? -50 : doc['start'].value - doc['end'].value" - } - } - }], - XPOS_SORT_KEY: ['xpos'], -} -POPULATION_SORTS = { - sort: [{ - '_script': { - 'type': 'number', - 'script': { - 'params': {'field': POPULATIONS[pop_key]['AF']}, - 'source': "doc.containsKey(params.field) ? (doc[params.field].empty ? 0 : doc[params.field].value) : 1" - } - } - }] for sort, pop_key in {'gnomad': 'gnomad_genomes', 'gnomad_exomes': 'gnomad_exomes', 'callset_af': 'callset'}.items()} -SORT_FIELDS.update(POPULATION_SORTS) -PREDICTOR_SORT_FIELDS = { - 'cadd': 'cadd_PHRED', - 'revel': 'dbnsfp_REVEL_score', - 'eigen': 'eigen_Eigen_phred', - 'mpc': 'mpc_MPC', - 'splice_ai': 'splice_ai_delta_score', - 'primate_ai': 'primate_ai_score', -} -SORT_FIELDS.update({ - sort: [{sort_field: {'order': 'desc', 'unmapped_type': 'double', 'numeric_type': 'double'}}] - for sort, sort_field in PREDICTOR_SORT_FIELDS.items() -}) - -SCREEN_KEY = 'SCREEN' -CLINVAR_KEY = 'clinvar' -CLINVAR_FIELDS = ['clinical_significance', 'variation_id', 'allele_id', 'gold_stars'] -HGMD_KEY = 'hgmd' -HGMD_FIELDS = ['accession', 'class'] -GENOTYPES_FIELD_KEY = 'genotypes' -HAS_ALT_FIELD_KEYS = ['samples_num_alt_1', 'samples_num_alt_2', 'samples'] -SORTED_TRANSCRIPTS_FIELD_KEY = 'sortedTranscriptConsequences' -CANONICAL_TRANSCRIPT_FILTER = 'non_coding_transcript_exon_variant__canonical' -NESTED_FIELDS = { - field_name: {field: {} for field in fields} for field_name, fields in { - CLINVAR_KEY: CLINVAR_FIELDS, - HGMD_KEY: HGMD_FIELDS, - }.items() -} - -GRCH38_LOCUS_FIELD = 'rg37_locus' -XSTOP_FIELD = 'xstop' -SPLICE_AI_FIELD = 'splice_ai' -CORE_FIELDS_CONFIG = { - 'alt': {}, - 'contig': {'response_key': 'chrom'}, - 'end': {'format_value': int}, - 'filters': {'response_key': 'genotypeFilters', 'format_value': ','.join, 'default_value': ''}, - 'num_exon': {'response_key': 'numExon'}, - 'originalAltAlleles': {'format_value': lambda alleles: [a.split('-')[-1] for a in alleles], 'default_value': []}, - 'ref': {}, - 'rsid': {}, - 'screen_region_type': {'response_key': 'screenRegionType', 'format_value': lambda types: types[0] if types else None}, - 'start': {'response_key': 'pos', 'format_value': int}, - 'svType': {}, - 'variantId': {}, - 'xpos': {'format_value': int}, - XSTOP_FIELD: {'format_value': int}, - 'rg37_locus_end': {'response_key': 'rg37LocusEnd', 'format_value': lambda locus: locus.to_dict()}, - 'sv_type_detail': {'response_key': 'svTypeDetail'}, - 'cpx_intervals': { - 'response_key': 'cpxIntervals', - 'format_value': lambda intervals: [interval.to_dict() for interval in (intervals or [])], - }, - 'algorithms': {'format_value': ', '.join}, - 'bothsides_support': {'response_key': 'bothsidesSupport'}, -} -MITO_CORE_FIELDS_CONFIG = { - 'common_low_heteroplasmy': {'response_key': 'commonLowHeteroplasmy'}, - 'high_constraint_region': {'response_key': 'highConstraintRegion'}, - 'mitomap_pathogenic': {'response_key': 'mitomapPathogenic'}, -} -CORE_FIELDS_CONFIG.update(MITO_CORE_FIELDS_CONFIG) -PREDICTION_FIELDS_CONFIG = { - 'cadd_PHRED': {'response_key': 'cadd'}, - 'dbnsfp_DANN_score': {}, - 'eigen_Eigen_phred': {}, - 'dbnsfp_VEST4_score': { - 'response_key': 'vest', - 'format_value': lambda x: x and next((v for v in x.split(';') if v != '.'), None), - }, - 'dbnsfp_MutPred_score': {'response_key': 'mut_pred', 'format_value': lambda x: None if x == '-' else x}, - 'mpc_MPC': {}, - 'dbnsfp_MutationTaster_pred': {'response_key': 'mut_taster'}, - 'dbnsfp_Polyphen2_HVAR_pred': {'response_key': 'polyphen'}, - 'gnomad_non_coding_constraint_z_score': {'response_key': 'gnomad_noncoding'}, - 'primate_ai_score': {'response_key': 'primate_ai'}, - 'splice_ai_delta_score': {'response_key': SPLICE_AI_FIELD}, - 'splice_ai_splice_consequence': {'response_key': 'splice_ai_consequence'}, - 'dbnsfp_REVEL_score': {}, - 'dbnsfp_SIFT_pred': {}, - 'StrVCTVRE_score': {'response_key': 'strvctvre'}, -} -MITO_PREDICTION_FIELDS_CONFIG = { - 'mitimpact_apogee': {}, - 'hap_defining_variant': {'response_key': 'haplogroup_defining', 'format_value': lambda k: 'Y' if k else None}, - 'mitotip_mitoTIP': {}, - 'hmtvar_hmtVar': {}, -} -PREDICTION_FIELDS_CONFIG.update(MITO_PREDICTION_FIELDS_CONFIG) - -def get_prediction_response_key(key): - return key.split('_')[1].lower() - -PREDICTION_FIELD_LOOKUP = { - field_config.get('response_key', get_prediction_response_key(field)): field - for field, field_config in PREDICTION_FIELDS_CONFIG.items() -} -MULTI_FIELD_PREDICTORS = { - 'fathmm': ['dbnsfp_fathmm_MKL_coding_pred', 'dbnsfp_FATHMM_pred'] -} -PREDICTION_FIELDS_RESPONSE_CONFIG = {k: {'response_key': k} for k, v in MULTI_FIELD_PREDICTORS.items()} -PREDICTION_FIELDS_RESPONSE_CONFIG.update(PREDICTION_FIELDS_CONFIG) - -QUALITY_QUERY_FIELDS = {'gq_sv': 5} -SHARED_QUALITY_FIELDS = {'gq': 5} -SNP_QUALITY_FIELDS = {'ab': 5} -SNP_QUALITY_FIELDS.update(SHARED_QUALITY_FIELDS) -SV_QUALITY_FIELDS = {'qs': 10} -SV_QUALITY_FIELDS.update(SHARED_QUALITY_FIELDS) -QUALITY_QUERY_FIELDS.update(SNP_QUALITY_FIELDS) -QUALITY_QUERY_FIELDS.update(SV_QUALITY_FIELDS) -BASE_GENOTYPE_FIELDS_CONFIG = { - 'sample_id': {}, - 'sample_type': {}, - 'num_alt': {'format_value': int, 'default_value': -1}, -} - -GENOTYPE_FIELDS_CONFIG = { - 'ad': {}, - 'dp': {}, - 'pl': {}, -} -GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SNP_QUALITY_FIELDS.keys()}) -MITO_GENOTYPE_FIELDS_CONFIG = { - 'dp': {}, - 'hl': {}, - 'mito_cn': {}, - 'contamination': {}, -} -MITO_GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -MITO_GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SHARED_QUALITY_FIELDS.keys()}) -SV_GENOTYPE_FIELDS_CONFIG = { - 'cn': {'format_value': int, 'default_value': -1}, - 'end': {}, - 'start': {}, - 'num_exon': {}, - 'geneIds': {'response_key': 'geneIds', 'format_value': list}, - 'defragged': {'format_value': bool}, - 'prev_call': {'format_value': bool}, - 'prev_overlap': {'format_value': bool}, - 'new_call': {'format_value': bool}, - 'prev_num_alt': {'format_value': lambda i: None if i is None else int(i)}, -} -SV_GENOTYPE_FIELDS_CONFIG.update(BASE_GENOTYPE_FIELDS_CONFIG) -SV_GENOTYPE_FIELDS_CONFIG.update({field: {} for field in SV_QUALITY_FIELDS.keys()}) - -GENOTYPE_FIELDS = { - Sample.DATASET_TYPE_VARIANT_CALLS: GENOTYPE_FIELDS_CONFIG, - Sample.DATASET_TYPE_SV_CALLS: SV_GENOTYPE_FIELDS_CONFIG, - Sample.DATASET_TYPE_MITO_CALLS: MITO_GENOTYPE_FIELDS_CONFIG, -} - -QUERY_FIELD_NAMES = list(CORE_FIELDS_CONFIG.keys()) + list(PREDICTION_FIELDS_CONFIG.keys()) + \ - [field for fields in MULTI_FIELD_PREDICTORS.values() for field in fields] + \ - [SORTED_TRANSCRIPTS_FIELD_KEY, GENOTYPES_FIELD_KEY, GRCH38_LOCUS_FIELD] + HAS_ALT_FIELD_KEYS -for field_name, fields in NESTED_FIELDS.items(): - QUERY_FIELD_NAMES += ['{}_{}'.format(field_name, field) for field in fields.keys()] -for pop_config in POPULATIONS.values(): - for pop_field in pop_config.values(): - if isinstance(pop_field, list): - QUERY_FIELD_NAMES += pop_field - elif pop_field is not None: - QUERY_FIELD_NAMES.append(pop_field) - -SV_SAMPLE_OVERRIDE_FIELD_CONFIGS = { - 'pos': {'select_val': min, 'genotype_field': 'start'}, - 'end': {'select_val': max}, - 'numExon':{'select_val': max}, - 'geneIds': { - 'select_val': lambda gene_lists: set([gene_id for gene_list in gene_lists for gene_id in (gene_list or [])]), - 'equal': lambda a, b: set(a or []) == set(b or []) - }, -} diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py deleted file mode 100644 index 59503053f1..0000000000 --- a/seqr/utils/search/elasticsearch/es_search.py +++ /dev/null @@ -1,358 +0,0 @@ -from collections import defaultdict -from copy import deepcopy -import elasticsearch -from elasticsearch_dsl import Search, Q, MultiSearch -import hashlib -import json -from sys import maxsize -from itertools import combinations - -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Individual -from seqr.utils.search.constants import XPOS_SORT_KEY, COMPOUND_HET, RECESSIVE, NEW_SV_FIELD, ALL_DATA_TYPES, X_LINKED_RECESSIVE, MAX_VARIANTS, \ - INHERITANCE_FILTERS, REF_REF, ANY_AFFECTED, AFFECTED, UNAFFECTED, HAS_ALT -from seqr.utils.search.elasticsearch.constants import \ - HAS_ALT_FIELD_KEYS, GENOTYPES_FIELD_KEY, POPULATION_RESPONSE_FIELD_CONFIGS, POPULATIONS, \ - SORTED_TRANSCRIPTS_FIELD_KEY, CORE_FIELDS_CONFIG, NESTED_FIELDS, PREDICTION_FIELDS_RESPONSE_CONFIG, \ - QUERY_FIELD_NAMES, GENOTYPE_QUERY_MAP, HGMD_CLASS_MAP, \ - SORT_FIELDS, MAX_COMPOUND_HET_GENES, MAX_INDEX_NAME_LENGTH, QUALITY_QUERY_FIELDS, \ - GRCH38_LOCUS_FIELD, MAX_SEARCH_CLAUSES, SV_SAMPLE_OVERRIDE_FIELD_CONFIGS, \ - PREDICTION_FIELD_LOOKUP, MULTI_FIELD_PREDICTORS, SPLICE_AI_FIELD, CLINVAR_KEY, HGMD_KEY, CLINVAR_PATH_SIGNIFICANCES, \ - PATH_FREQ_OVERRIDE_CUTOFF, CANONICAL_TRANSCRIPT_FILTER, \ - get_prediction_response_key, XSTOP_FIELD, GENOTYPE_FIELDS, SCREEN_KEY, MAX_INDEX_SEARCHES, PREFILTER_SEARCH_SIZE -from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json -from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS, get_chrom_pos -from seqr.views.utils.json_utils import _to_camel_case - -logger = SeqrLogger(__name__) - -class EsSearch(object): - - AGGREGATION_NAME = 'compound het' - CACHED_COUNTS_KEY = 'loaded_variant_counts' - - def __init__(self, samples, genome_version, previous_search_results=None, return_all_queried_families=False, user=None, sort=None, skipped_samples=None): - from seqr.utils.search.utils import InvalidSearchException - from seqr.utils.search.elasticsearch.es_utils import get_es_client, InvalidIndexException - self._client = get_es_client() - - self.samples_by_family_index = defaultdict(lambda: defaultdict(dict)) - for s in samples.select_related('individual__family'): - self.samples_by_family_index[s.elasticsearch_index][s.individual.family.guid][s.sample_id] = s - - self._set_indices(sorted(list(self.samples_by_family_index.keys()))) - self._set_index_metadata() - - invalid_genome_indices = [ - f"{index} ({meta['genomeVersion']})" for index, meta in self.index_metadata.items() - if meta['genomeVersion'] != genome_version - ] - self._genome_version = genome_version - self._skipped_samples = skipped_samples - - self.indices_by_dataset_type = defaultdict(list) - for index in self._indices: - dataset_type = self._get_index_dataset_type(index) - self.indices_by_dataset_type[dataset_type].append(index) - - self.previous_search_results = {} if previous_search_results is None else previous_search_results - self._return_all_queried_families = return_all_queried_families - self._user = user - - self._search = Search() - self._index_searches = defaultdict(list) - self._family_individual_affected_status = {} - self._allowed_consequences = None - self._allowed_consequences_secondary = None - self._consequence_overrides = {} - self._filtered_gene_ids = None - self._paired_index_comp_het = False - self._no_sample_filters = False - self._any_affected_sample_filters = False - - self._sort = deepcopy(SORT_FIELDS.get(sort, [])) if sort else None - if self._sort: - self._sort_variants(samples) - - @staticmethod - def _parse_xstop(result): - xstop = result.pop(XSTOP_FIELD, None) - - def _get_index_dataset_type(self, index): - return self.get_index_metadata_dataset_type(self.index_metadata[index]) - - @staticmethod - def get_index_metadata_dataset_type(index_metadata): - data_type = index_metadata.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) - return data_type - - def _set_indices(self, indices): - self._indices = indices - self._set_index_name() - - def _set_index_name(self): - self.index_name = ','.join(sorted(self._indices)) - - def _set_index_metadata(self): - from seqr.utils.search.elasticsearch.es_utils import get_index_metadata - self.index_metadata = get_index_metadata(self.index_name, self._client, include_fields=True) - - def _sort_variants(self, sample_data): - main_sort_dict = self._sort[0] if len(self._sort) and isinstance(self._sort[0], dict) else None - - # always final sort on variant ID to keep different variants at the same position grouped properly - if 'variantId' not in self._sort: - self._sort.append('variantId') - - self._search = self._search.sort(*self._sort) - - def _filter(self, new_filter): - self._search = self._search.filter(new_filter) - return self - - def filter_by_variant_ids(self, variant_ids): - self._filter(Q('terms', variantId=variant_ids)) - return self - - def search(self, page=1, num_results=100): - indices = self._indices - - logger.info('Searching in elasticsearch indices: {}'.format(', '.join(indices)), self._user) - - is_single_search, search_kwargs = self._should_execute_single_search(page=page, num_results=num_results) - - if is_single_search: - return self._execute_single_search(**search_kwargs) - - def _is_single_search(self): - return len(self._indices) == 1 and len(self._index_searches) < 2 and \ - len(self._index_searches.get(self._indices[0], [])) <= 1 - - def _should_execute_single_search(self, page=1, num_results=100): - is_single_search = self._is_single_search() - num_loaded = len(self.previous_search_results.get('all_results', [])) - - if not self._index_searches: - # If doing all project-families all inheritance search, do it as a single query - # Load all variants, do not skip pages - num_loaded += self.previous_search_results.get('duplicate_doc_count', 0) - if num_loaded >= (page - 1) * num_results: - start_index = num_loaded - return True, {'page': page, 'num_results': num_results, 'start_index': start_index, 'deduplicate': True} - - def _execute_single_search(self, page=1, num_results=100, start_index=None, deduplicate=False, **kwargs): - num_results_for_search = num_results * len(self._indices) if deduplicate else num_results - searches, log_messages = self._get_paginated_searches( - self.index_name, page=page, num_results=num_results_for_search, start_index=start_index - ) - logger.info(log_messages[0], self._user) - search = searches[0] - response = self._execute_search(search) - parsed_response = self._parse_response(response) - return self._process_single_search_response( - parsed_response, page=page, num_results=num_results, deduplicate=deduplicate, **kwargs) - - def _process_single_search_response(self, parsed_response, page=1, num_results=100, deduplicate=False, **kwargs): - variant_results, total_results, is_compound_het, _ = parsed_response - self.previous_search_results['total_results'] = total_results - - results_start_index = (page - 1) * num_results - - if deduplicate: - variant_results = self._deduplicate_results(variant_results) - - # Only save contiguous pages of results: - previous_all_results = self.previous_search_results.get('all_results', []) - if len(previous_all_results) >= results_start_index: - self.previous_search_results['all_results'] = self.previous_search_results.get('all_results', []) + variant_results - variant_results = self.previous_search_results['all_results'][results_start_index:] - - return variant_results[:num_results] - - def _parse_response(self, response): - index_name = response.hits[0].meta.index if response.hits else None - - response_total = response.hits.total['value'] - logger.info('Total hits: {} ({} seconds)'.format(response_total, response.took / 1000.0), self._user) - return [self._parse_hit(hit) for hit in response], response_total, False, index_name - - def _parse_hit(self, raw_hit): - hit = {k: raw_hit[k] for k in QUERY_FIELD_NAMES if k in raw_hit} - index_name = raw_hit.meta.index - index_family_samples = self.samples_by_family_index[index_name] - data_type = self._get_index_dataset_type(index_name) - - family_guids, genotypes = self._parse_genotypes(raw_hit, hit, index_family_samples, data_type) - - result = _get_field_values(hit, CORE_FIELDS_CONFIG, format_response_key=str) - result.update({ - field_name: _get_field_values(hit, fields, lookup_field_prefix=field_name) - for field_name, fields in NESTED_FIELDS.items() - }) - if hasattr(raw_hit.meta, 'sort'): - result['_sort'] = [_parse_es_sort(sort, self._sort[i]) for i, sort in enumerate(raw_hit.meta.sort)] - - result['genomeVersion'] = self._genome_version - self._parse_xstop(result) - result[CLINVAR_KEY]['version'] = self.index_metadata[index_name].get('clinvar_version') - - populations = { - population: _get_field_values( - hit, POPULATION_RESPONSE_FIELD_CONFIGS, format_response_key=lambda key: key.lower(), - lookup_field_prefix=population, - existing_fields=self.index_metadata[index_name]['fields'], - get_addl_fields=lambda field: pop_config[field] if isinstance(pop_config[field], list) else [pop_config[field]], - skip_fields=[field for field, val in pop_config.items() if val is None], - ) - for population, pop_config in POPULATIONS.items() - } - - sorted_transcripts = [ - {_to_camel_case(k): v for k, v in transcript.to_dict().items()} - for transcript in hit[SORTED_TRANSCRIPTS_FIELD_KEY] or [] - ] - - transcripts = defaultdict(list) - for transcript in sorted_transcripts: - if transcript['geneId']: - transcripts[transcript['geneId']].append(transcript) - gene_ids = result.pop('geneIds', None) - main_transcript_id, selected_main_transcript_id = self._get_main_transcript(sorted_transcripts) - - result.update({ - 'familyGuids': sorted(family_guids), - 'genotypes': genotypes, - 'mainTranscriptId': main_transcript_id, - 'selectedMainTranscriptId': selected_main_transcript_id, - 'populations': populations, - 'predictions': _get_field_values( - hit, PREDICTION_FIELDS_RESPONSE_CONFIG, format_response_key=get_prediction_response_key, - get_addl_fields=lambda field: MULTI_FIELD_PREDICTORS.get(field, []) - ), - 'transcripts': dict(transcripts), - }) - return result - - def _parse_genotypes(self, raw_hit, hit, index_family_samples, data_type): - if hasattr(raw_hit.meta, 'matched_queries'): - family_guids = list(raw_hit.meta.matched_queries) - else: - # Searches for all inheritance and all families do not filter on inheritance so there are no matched_queries - alt_allele_samples = set() - for alt_samples_field in HAS_ALT_FIELD_KEYS: - if alt_samples_field in hit: - alt_allele_samples.update(hit[alt_samples_field]) - - if self._any_affected_sample_filters: - pass - else: - _is_matched_sample = lambda *args: True - - family_guids = [family_guid for family_guid, samples_by_id in index_family_samples.items() - if any(sample_id in alt_allele_samples and _is_matched_sample(family_guid, sample) - for sample_id, sample in samples_by_id.items())] - - genotypes = {} - genotype_fields_config = GENOTYPE_FIELDS[data_type] - for family_guid in family_guids: - samples_by_id = index_family_samples[family_guid] - for genotype_hit in hit[GENOTYPES_FIELD_KEY]: - sample = samples_by_id.get(genotype_hit['sample_id']) - if sample: - genotype_hit['sample_type'] = sample.sample_type - genotypes[sample.individual.guid] = _get_field_values(genotype_hit, genotype_fields_config) - - return family_guids, genotypes - - def _get_main_transcript(self, sorted_transcripts): - main_transcript_id = sorted_transcripts[0]['transcriptId'] \ - if len(sorted_transcripts) and 'transcriptRank' in sorted_transcripts[0] else None - - selected_main_transcript_id = None - - return main_transcript_id, selected_main_transcript_id - - def _deduplicate_results(self, sorted_new_results): - original_result_count = len(sorted_new_results) - variant_results = [] - for variant in sorted_new_results: - if variant_results and variant_results[-1]['variantId'] == variant['variantId']: - self._merge_duplicate_variants(variant_results[-1], variant) - else: - variant_results.append(variant) - - previous_duplicates = self.previous_search_results.get('duplicate_doc_count', 0) - new_duplicates = original_result_count - len(variant_results) - self.previous_search_results['duplicate_doc_count'] = previous_duplicates + new_duplicates - - self.previous_search_results['total_results'] -= self.previous_search_results['duplicate_doc_count'] - - return variant_results - - def _get_paginated_searches(self, index_name, page=1, num_results=100, start_index=None): - searches = [] - log_messages = [] - for search in self._index_searches.get(index_name, [self._search]): - search = search.index(index_name.split(',')) - - if search.aggs.to_dict(): - log_messages.append('Loading {}s for {}'.format(self.AGGREGATION_NAME, index_name)) - else: - end_index = page * num_results - - search = search[start_index:end_index] - search = search.source(QUERY_FIELD_NAMES) - log_messages.append('Loading {} records {}-{}'.format(index_name, start_index, end_index)) - - searches.append(search) - return searches, log_messages - - def _execute_search(self, search): - logger.debug(json.dumps(search.to_dict(), indent=2), self._user) - try: - return search.using(self._client).execute() - except elasticsearch.exceptions.ConnectionTimeout as e: - raise e - - -def _get_family_affected_status(samples_by_id, inheritance_filter): - individual_affected_status = inheritance_filter.get('affected') or {} - affected_status = {} - for sample in samples_by_id.values(): - indiv = sample.individual - affected_status[indiv.guid] = individual_affected_status.get(indiv.guid) or indiv.affected - - return affected_status - - -VUS_FILTER = 'vus_or_conflicting' -VUS_REGEX = 'Conflicting_interpretations_of_pathogenicity.*|~((.*[Bb]enign.*)|(.*[Pp]athogenic.*))' - - -def _parse_es_sort(sort, sort_config): - if sort in {'Infinity', '-Infinity', None}: - # ES returns these values for sort when a sort field is missing, using the correct value for the given direction - sort = maxsize - - return sort - - -def _get_field_values(hit, field_configs, format_response_key=_to_camel_case, get_addl_fields=None, lookup_field_prefix='', existing_fields=None, skip_fields=None): - return { - field_config.get('response_key') or format_response_key(field): _value_if_has_key( - hit, - (get_addl_fields(field) if get_addl_fields else []) + - ['{}_{}'.format(lookup_field_prefix, field) if lookup_field_prefix else field], - existing_fields=existing_fields, - **field_config - ) if field not in (skip_fields or []) else None - for field, field_config in field_configs.items() - } - - -def _value_if_has_key(hit, keys, format_value=None, default_value=None, existing_fields=None, **kwargs): - for key in keys: - if key in hit: - return format_value(default_value if hit[key] is None else hit[key]) if format_value else hit[key] - return default_value if not existing_fields or any(key in existing_fields for key in keys) else None diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py deleted file mode 100644 index cd1d32c179..0000000000 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ /dev/null @@ -1,175 +0,0 @@ -from collections import defaultdict -from elasticsearch import Elasticsearch -from elasticsearch.exceptions import ConnectionError as EsConnectionError, TransportError -from urllib3.connectionpool import connection_from_url - -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json -from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY -from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch -from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page -from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ - ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER - - -class InvalidIndexException(Exception): - pass - - -ES_EXCEPTION_ERROR_MAP = { - InvalidIndexException: 400, - EsConnectionError: 504, - TransportError: lambda e: int(e.status_code) if e.status_code != 'N/A' else 400, -} -ES_EXCEPTION_MESSAGE_MAP = { - EsConnectionError: str, - TransportError: lambda e: '{}: {} - {} - {}'.format(e.__class__.__name__, e.status_code, repr(e.error), _get_transport_error_type(e.info)), -} -ES_ERROR_LOG_EXCEPTIONS = {InvalidIndexException} - -MAX_VARIANTS_FETCH = 1000 - - -def _get_transport_error_type(error): - error_type = 'no detail' - if isinstance(error, dict): - root_cause = error.get('root_cause') - error_info = error.get('error') - if (not root_cause) and isinstance(error_info, dict): - root_cause = error_info.get('root_cause') - - if root_cause: - error_type = root_cause[0].get('type') or root_cause[0].get('reason') - elif error_info and not isinstance(error_info, dict): - error_type = repr(error_info) - return error_type - - -def es_backend_enabled(): - return bool(ELASTICSEARCH_SERVICE_HOSTNAME) - - -def get_es_client(timeout=60, **kwargs): - client_kwargs = { - 'hosts': [{'host': ELASTICSEARCH_SERVICE_HOSTNAME, 'port': ELASTICSEARCH_SERVICE_PORT}], - 'timeout': timeout, - } - if ELASTICSEARCH_CREDENTIALS: - client_kwargs['http_auth'] = ELASTICSEARCH_CREDENTIALS - if ELASTICSEARCH_PROTOCOL: - client_kwargs['scheme'] = ELASTICSEARCH_PROTOCOL - if ES_SSL_CONTEXT: - client_kwargs['ssl_context'] = ES_SSL_CONTEXT - return Elasticsearch(**client_kwargs, **kwargs) - - -def ping_elasticsearch(): - if not get_es_client(timeout=3, max_retries=0).ping(): - raise ValueError('No response from elasticsearch ping') - - -def ping_kibana(): - resp = connection_from_url('http://{}'.format(KIBANA_SERVER)).urlopen('HEAD', '/status', timeout=3, retries=3) - if resp.status >= 400: - raise ValueError('Kibana Error {}: {}'.format(resp.status, resp.reason)) - - -SAMPLE_FIELDS_LIST = ['samples', 'samples_num_alt_1'] -# support .bgz instead of requiring .vcf.bgz due to issues with DSP delivery of large callsets -DATASET_FILE_EXTENSIONS = VCF_FILE_EXTENSIONS[:-1] + ('.bgz', '.bed', '.mt') - - -def get_index_metadata(index_name, client, include_fields=False, use_cache=True): - if use_cache: - cache_key = 'index_metadata__{}'.format(index_name) - cached_metadata = safe_redis_get_json(cache_key) - if cached_metadata: - return cached_metadata - - try: - mappings = client.indices.get_mapping(index=index_name) - except Exception as e: - raise InvalidIndexException('{} - Error accessing index: {}'.format( - index_name, e.error if hasattr(e, 'error') else str(e))) - index_metadata = {} - for index_name, mapping in mappings.items(): - variant_mapping = mapping['mappings'] - index_metadata[index_name] = variant_mapping.get('_meta', {}) - if include_fields: - index_metadata[index_name]['fields'] = { - field: field_props.get('type') for field, field_props in variant_mapping['properties'].items() - } - if use_cache and include_fields: - # Only cache metadata with fields - safe_redis_set_json(cache_key, index_metadata) - return index_metadata - - -def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): - variant_ids = sorted(set(variant_ids)) - variants = EsSearch( - samples, genome_version, user=user, sort=XPOS_SORT_KEY, - ).filter_by_variant_ids(variant_ids) - return variants.search(num_results=len(variant_ids)) - - -def get_es_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=None, num_results=None, - gene_agg=False, skip_genotype_filter=False): - es_search_cls = EsGeneAggSearch if gene_agg else EsSearch - - es_search = es_search_cls( - samples, - genome_version, - previous_search_results=previous_search_results, - user=user, - sort=sort, - skipped_samples=search.get('skipped_samples'), - ) - - es_search.filter_variants( - inheritance_mode=search.get('inheritance_mode'), inheritance_filter=search.get('inheritance_filter'), - frequencies=search.get('freqs'), pathogenicity=search.get('pathogenicity'), - annotations=search.get('annotations'), annotations_secondary=search.get('annotations_secondary'), - in_silico=search.get('in_silico'), quality_filter=search.get('qualityFilter'), - custom_query=search.get('customQuery'), locus=search.get('locus'), skip_genotype_filter=skip_genotype_filter, - dataset_type=search.get('dataset_type'), secondary_dataset_type=search.get('secondary_dataset_type'), - genes=search.get('genes'), intervals=search.get('intervals'), rs_ids=search.get('rs_ids'), - variant_ids=search.get('variant_ids'), exclude_locations=search.get('exclude_locations'), - ) - - return es_search.search(page=page, num_results=num_results) - - -def process_es_previously_loaded_results(previous_search_results, start_index, end_index): - grouped_results = previous_search_results.get('grouped_results') - results = None - if grouped_results: - results = get_compound_het_page(grouped_results, start_index, end_index) - - return results - - -def process_es_previously_loaded_gene_aggs(previous_search_results): - total_results = previous_search_results.get('total_results') - if total_results is None or 'all_results' in previous_search_results or 'grouped_results' not in previous_search_results: - return None - - loaded = sum(counts.get('loaded', 0) for counts in previous_search_results.get('loaded_variant_counts', {}).values()) - if loaded != total_results: - return None - - gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - - for group in previous_search_results['grouped_results']: - variants = next(iter(group.values())) - gene_id = next(iter(group)) - if not gene_id or gene_id == 'null': - gene_id = next(( - gene_id for gene_id, transcripts in variants[0]['transcripts'].items() - if any(t['transcriptId'] == variants[0]['mainTranscriptId'] for t in transcripts) - ), None) if variants[0]['mainTranscriptId'] else None - if gene_id: - gene_aggs[gene_id]['total'] += len(variants) - for family_guid in variants[0]['familyGuids']: - gene_aggs[gene_id]['families'][family_guid] += len(variants) - return gene_aggs - diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py deleted file mode 100644 index 503136fa2a..0000000000 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ /dev/null @@ -1,1343 +0,0 @@ -from copy import deepcopy -import mock -import jmespath -import json -import re -from collections import defaultdict -from datetime import timedelta -from django.test import TestCase -from sys import maxsize -from urllib3_mock import Responses - -from seqr.models import Family, Sample -from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status -from seqr.views.utils.test_utils import PARSED_VARIANTS, PARSED_SV_VARIANT, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT - - -# The responses library for mocking requests does not work with urllib3 (which is used by elasticsearch) -# The urllib3_mock library works for those requests, but it has limited functionality, so this extension adds helper -# methods for easier usage -class Urllib3Responses(Responses): - def add_json(self, url, json_response, method=None, match_querystring=True, **kwargs): - if not method: - method = self.GET - body = json.dumps(json_response) - self.add(method, url, match_querystring=match_querystring, content_type='application/json', body=body, **kwargs) - - def replace_json(self, url, *args, **kwargs): - existing_index = next(i for i, match in enumerate(self._urls) if match['url'] == url) - self.add_json(url, *args, **kwargs) - self._urls[existing_index] = self._urls.pop() - - def call_request_json(self, index=-1): - return json.loads(self.calls[index].request.body) - - def activate(self, func): - def wrapper(*args, **kwargs): - with self: - return func(*args, **kwargs) - return wrapper - -urllib3_responses = Urllib3Responses() - - -INDEX_NAME = 'test_index' -SECOND_INDEX_NAME = 'test_index_second' -HG38_INDEX_NAME = 'test_index_hg38' -SV_INDEX_NAME = 'test_index_sv' -SV_WGS_INDEX_NAME = 'test_index_sv_wgs' -MITO_WGS_INDEX_NAME = 'test_index_mito_wgs' -INDEX_ALIAS = '377e97bd791cf92a78296bc184e0976a' -ALIAS_MAP = {INDEX_ALIAS: ','.join([INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME, MITO_WGS_INDEX_NAME])} -SUB_INDICES = ['sub_index_1', 'sub_index_2'] -SECOND_SUB_INDICES = ['sub_index_a', 'sub_index_b'] - -ES_VARIANTS = [ - { - '_source': { - 'homozygote_count': 3, - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '1-248367227-TC-T' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': 0, - 'cadd_PHRED': '25.9', - 'splice_ai_delta_score': 0.75, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': 125568, - 'g1k_AN': None, - 'topmed_AF': 0.00016724, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'TC', - 'exac_AC_Hom': 0, - 'topmed_AC': 21, - 'dbnsfp_REVEL_score': None, - 'dbnsfp_VEST4_score': '0.335;0.341;0.38', - 'primate_ai_score': None, - 'variantId': '1-248367227-TC-T', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'nonsense_mediated_decay', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Gene3D:1', - ], - 'canonical': None, - 'transcript_rank': 1, - 'cdna_end': 143, - 'lof_filter': None, - 'hgvs': 'ENSP00000413641.1:p.Leu48del', - 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', - 'cdna_start': 141, - 'transcript_id': 'ENST00000428085', - 'protein_id': 'ENSP00000413641', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000413641.1:p.Leu48del', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant', - 'inframe_deletion', - 'NMD_transcript_variant' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'screen_region_type' : [ - 'dELS', - 'CTCF-bound' - ], - 'gnomad_non_coding_constraint_z_score': 1.01272, - 'hgmd_class': 'DM', - 'AC': 2, - 'exac_AN_Adj': 121308, - 'mpc_MPC': None, - 'AF': 0.063, - 'alt': 'T', - 'clinvar_clinical_significance': 'Pathogenic/Likely_pathogenic', - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': 0.0004590314436538903, - 'gnomad_genomes_FAF_AF': 0.000437, - 'exac_AF': 0.00006589, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 8, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': 0, - 'gnomad_genomes_AN': 30946, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.0009151523074911753, - 'gnomad_genomes_Hemi': None, - 'xpos': 1248367227, - 'start': 248367227, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245930, - 'contig': '1', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.0006726888333653661, - 'gnomad_exomes_AC': 16, - 'dbnsfp_FATHMM_pred': 'T', - 'dbnsfp_fathmm_MKL_coding_pred': 'D', - 'gnomad_exomes_AF': 0.00006505916317651364, - 'gnomad_genomes_AF': 0.00012925741614425127, - 'gnomad_genomes_AC': 4, - 'genotypes': [ - { - 'num_alt': 2, - 'ab': 1, - 'dp': 74, - 'gq': 99, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 88, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - ], - 'samples_num_alt_1': ['NA20885'], - 'samples_num_alt_2': ['NA20870'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3'], SECOND_INDEX_NAME: ['F000011_11']}, - }, - { - '_source': { - 'gnomad_exomes_Hemi': None, - 'originalAltAlleles': [ - '2-103343353-GAGA-G' - ], - 'hgmd_accession': None, - 'g1k_AF': None, - 'gnomad_genomes_Hom': None, - 'cadd_PHRED': None, - 'exac_AC_Hemi': None, - 'g1k_AC': None, - 'topmed_AN': None, - 'g1k_AN': None, - 'topmed_AF': None, - 'dbnsfp_MutationTaster_pred': None, - 'ref': 'GAGA', - 'exac_AC_Hom': 0, - 'topmed_AC': None, - 'dbnsfp_REVEL_score': None, - 'primate_ai_score': 1, - 'variantId': '2-103343353-GAGA-G', - 'sortedTranscriptConsequences': [ - { - 'amino_acids': 'LL/L', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 10, - 'codons': 'ctTCTc/ctc', - 'gene_symbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'PROSITE_profiles:PS50850', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 421, - 'lof_filter': None, - 'hgvs': 'ENSP00000258436.5:p.Leu126del', - 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', - 'cdna_start': 419, - 'transcript_id': 'ENST00000258436', - 'protein_id': 'ENSP00000258436', - 'category': 'missense', - 'gene_id': 'ENSG00000135953', - 'hgvsp': 'ENSP00000258436.5:p.Leu126del', - 'major_consequence': 'inframe_deletion', - 'consequence_terms': [ - 'inframe_deletion' - ] - }, - { - 'amino_acids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lof_flags': None, - 'major_consequence_rank': 4, - 'codons': 'Ccc/cc', - 'gene_symbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcript_rank': 0, - 'cdna_end': 897, - 'lof_filter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdna_start': 897, - 'transcript_id': 'ENST00000456743', - 'protein_id': 'ENSP00000389625', - 'category': 'lof', - 'gene_id': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'major_consequence': 'frameshift_variant', - 'consequence_terms': [ - 'frameshift_variant' - ] - } - ], - 'hgmd_class': None, - 'AC': 1, - 'exac_AN_Adj': 121336, - 'mpc_MPC': None, - 'AF': 0.031, - 'alt': 'G', - 'clinvar_clinical_significance': None, - 'rsid': None, - 'dbnsfp_DANN_score': None, - 'AN': 32, - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL': None, - 'gnomad_genomes_FAF_AF': None, - 'exac_AF': 0.00004942, - 'dbnsfp_GERP_RS': None, - 'dbnsfp_SIFT_pred': None, - 'exac_AC_Adj': 6, - 'g1k_POPMAX_AF': None, - 'topmed_Hom': None, - 'gnomad_genomes_AN': None, - 'dbnsfp_MetaSVM_pred': None, - 'dbnsfp_Polyphen2_HVAR_pred': None, - 'dbnsfp_VEST4_score': '.;.;.;.', - 'clinvar_allele_id': None, - 'gnomad_exomes_Hom': 0, - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL': 0.00016269686320447742, - 'gnomad_genomes_Hemi': None, - 'xpos': 2103343353, - 'start': 103343353, - 'filters': [], - 'dbnsfp_phastCons100way_vertebrate': None, - 'gnomad_exomes_AN': 245714, - 'contig': '2', - 'clinvar_gold_stars': None, - 'eigen_Eigen_phred': None, - 'exac_AF_POPMAX': 0.000242306760358614, - 'gnomad_exomes_AC': 6, - 'dbnsfp_FATHMM_pred': None, - 'gnomad_exomes_AF': 0.000024418633044922146, - 'gnomad_genomes_AF': None, - 'gnomad_genomes_AC': None, - 'genotypes': [ - { - 'num_alt': 1, - 'ab': 0.70212764, - 'dp': 50, - 'gq': 46, - 'sample_id': 'NA20870', - }, - { - 'num_alt': 1, - 'ab': 0.631, - 'dp': 50, - 'gq': 99, - 'sample_id': 'NA20885', - }, - { - 'num_alt': 2, - 'ab': 0, - 'dp': 67, - 'gq': 99, - 'sample_id': 'HG00731', - }, - { - 'num_alt': 1, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00732', - }, - { - 'num_alt': 0, - 'ab': 0, - 'dp': 42, - 'gq': 96, - 'sample_id': 'HG00733', - } - ], - 'samples_num_alt_1': ['NA20870', 'HG00733', 'NA20885'], - 'samples_num_alt_2': ['HG00732'], - }, - 'matched_queries': {INDEX_NAME: ['F000003_3', 'F000002_2'], SECOND_INDEX_NAME: ['F000011_11']}, - }, -] -BUILD_38_NO_LIFTOVER_ES_VARIANT = deepcopy(ES_VARIANTS[1]) -BUILD_38_NO_LIFTOVER_ES_VARIANT['_source'].update({ - 'start': 103343363, - 'xpos': 2103343363, - 'variantId': '2-103343363-GAGA-G' -}) -BUILD_38_ES_VARIANT = deepcopy(BUILD_38_NO_LIFTOVER_ES_VARIANT) -BUILD_38_ES_VARIANT['_source']['rg37_locus'] = {'contig': '2', 'position': 103343353} - -ES_SV_VARIANT = { - '_source': { - 'genotypes': [ - { - 'qs': 33, - 'cn': 1, - 'defragged': False, - 'sample_id': 'HG00731', - 'num_exon': 2, - 'start': 49045487, - 'end': 49045899, - 'geneIds': ['ENSG00000228198'], - 'prev_call': False, - 'prev_overlap': False, - 'new_call': True, - }, - { - 'qs': 80, - 'cn': 2, - 'defragged': False, - 'sample_id': 'HG00733', - 'num_exon': 1, - 'start': 49045987, - 'end': 49045890, - 'geneIds': ['ENSG00000228198', 'ENSG00000135953'], - 'prev_call': False, - 'prev_overlap': True, - 'new_call': False, - } - ], - 'xpos': 1049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 1049045587, - 'num_exon': 1, - 'pos': 49045487, - 'StrVCTVRE_score': 0.374, - 'svType': 'INS', - 'xstop': 9049045898, - 'variantId': 'prefix_19107_DEL', - 'samples': ['HG00731'], - 'sc': 7, - 'contig': '1', - 'bothsides_support': True, - 'sortedTranscriptConsequences': [ - { - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_id': 'ENSG00000135953' - }, - { - 'gene_id': 'ENSG00000186092' - }, - ], - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_INDEX_NAME: ['F000002_2']}, - } - -ES_SV_WGS_VARIANT = { - '_source': { - 'genotypes': [ - { - 'gq': 33, - 'sample_id': 'NA21234', - 'num_alt': 1, - 'prev_num_alt': 2, - } - ], - 'xpos': 2049045387, - 'end': 49045898, - 'start': 49045387, - 'xstart': 2049045387, - 'pos': 49045387, - 'svType': 'CPX', - 'xstop': 20012345678, - 'variantId': 'prefix_19107_CPX', - 'algorithms': ['wham', 'manta'], - 'sc': 7, - 'contig': '2', - 'rg37_locus': {'contig': '2', 'position': 49272526}, - 'rg37_locus_end': {'contig': '20', 'position': 12326326}, - 'sortedTranscriptConsequences': [ - { - 'gene_symbol': 'OR4F5', - 'major_consequence': 'DUP_PARTIAL', - 'gene_id': 'ENSG00000228198' - }, - { - 'gene_symbol': 'FBXO28', - 'major_consequence': 'MSV_EXON_OVR', - 'gene_id': 'ENSG00000228199' - }, - { - 'gene_symbol': 'FAM131C', - 'major_consequence': 'DUP_LOF', - 'gene_id': 'ENSG00000228201' - }, - { - "gene_symbol": "H3-2", - "gene_id": None, - "major_consequence": "NEAREST_TSS" - }, - ], - 'cpx_intervals': [{'type': 'DUP', 'chrom': '2', 'start': 1000, 'end': 3000}, - {'type': 'INV', 'chrom': '20', 'start': 11000, 'end': 13000}], - 'sv_type_detail': 'dupINV', - 'gnomad_svs_ID': 'gnomAD-SV_v2.1_BND_1_1', - 'gnomad_svs_AF': 0.00679, - 'gnomad_svs_AC': 22, - 'gnomad_svs_AN': 3240, - 'geneIds': ['ENSG00000228198'], - 'sf': 0.000693825, - 'sn': 10088 - }, - 'matched_queries': {SV_WGS_INDEX_NAME: ['F000014_14']}, - } - -ES_MITO_WGS_VARIANT = { - "_source" : { - "genotypes" : [ - { - "num_alt": 2, - "gq": 60.0, - "hl": 1.0, - "mito_cn": 319.03225806451616, - "contamination": 0.0, - "dp": 5139.0, - "sample_id": "HG00731" - }, - ], - "samples_gq_60_to_65" : [ - "HG00731" - ], - "samples_num_alt_2" : [ "HG00731" ], - "AC" : 0, - "AC_het" : 1, - "AF" : 0.0, - "AF_het" : 3.968253968253968E-4, - "alt" : "A", - "AN" : 2520, - "clinvar_allele_id" : None, - "clinvar_clinical_significance" : "Likely_pathogenic", - "clinvar_gold_stars" : None, - "codingGeneIds" : [ - "ENSG00000198840" - ], - "common_low_heteroplasmy" : False, - "contig" : "M", - "dbnsfp_SIFT_pred" : "D", - "dbnsfp_MutationTaster_pred" : "N", - "dbnsfp_FATHMM_pred" : "T", - "dbnsfp_GERP_RS" : "5.07", - "dbnsfp_phastCons100way_vertebrate" : "0.958000", - "docId" : "M-10195-C-A", - "domains" : [ - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Gene3D:1", - "PANTHER:PTHR11058", - "Pfam:PF00507" - ], - "end" : 10195, - "filters" : [ ], - "geneIds" : [ - "ENSG00000198840" - ], - "gnomad_mito_AN": 56421, - "gnomad_mito_AC": 1368, - "gnomad_mito_AC_het": 3, - "gnomad_mito_AF": 0.024246292, - "gnomad_mito_AF_het": 5.317169E-5, - "gnomad_mito_max_hl": 1.0, - "hap_defining_variant" : False, - "helix_AC": 1312, - "helix_AC_het": 5, - "helix_AF": 0.0033268193, - "helix_AF_het": 4.081987E-5, - "helix_max_hl": 0.90441, - "high_constraint_region" : True, - "hmtvar_hmtVar" : 0.71, - "mainTranscript_biotype" : "protein_coding", - "mainTranscript_canonical" : 1, - "mainTranscript_category" : "missense", - "mainTranscript_cdna_start" : 137, - "mainTranscript_cdna_end" : 137, - "mainTranscript_codons" : "cCc/cAc", - "mainTranscript_gene_id" : "ENSG00000198840", - "mainTranscript_gene_symbol" : "MT-ND3", - "mainTranscript_hgvs" : "p.Pro46His", - "mainTranscript_hgvsc" : "ENST00000361227.2:c.137C>A", - "mainTranscript_major_consequence" : "missense_variant", - "mainTranscript_major_consequence_rank" : 11, - "mainTranscript_transcript_id" : "ENST00000361227", - "mainTranscript_amino_acids" : "P/H", - "mainTranscript_domains" : "Gene3D:1,ENSP_mappings:5xtc,PANTHER:PTHR11058,ENSP_mappings:5xtd,Pfam:PF00507", - "mainTranscript_hgvsp" : "ENSP00000355206.2:p.Pro46His", - "mainTranscript_polyphen_prediction" : "probably_damaging", - "mainTranscript_protein_id" : "ENSP00000355206", - "mainTranscript_sift_prediction" : "deleterious_low_confidence", - "mitimpact_apogee" : 0.42, - "mitomap_pathogenic" : True, - "pos" : 10195, - "ref" : "C", - "rg37_locus" : { - "contig" : "MT", - "position" : 10195 - }, - "rsid" : None, - "sortedTranscriptConsequences" : [ - { - "biotype" : "protein_coding", - "canonical" : 1, - "cdna_start" : 137, - "cdna_end" : 137, - "codons" : "cCc/cAc", - "gene_id" : "ENSG00000198840", - "gene_symbol" : "MT-ND3", - "hgvsc" : "ENST00000361227.2:c.137C>A", - "hgvsp" : "ENSP00000355206.2:p.Pro46His", - "transcript_id" : "ENST00000361227", - "amino_acids" : "P/H", - "polyphen_prediction" : "probably_damaging", - "protein_id" : "ENSP00000355206", - "protein_start" : 46, - "sift_prediction" : "deleterious_low_confidence", - "consequence_terms" : [ - "missense_variant" - ], - "domains" : [ - "Gene3D:1", - "ENSP_mappings:5xtc", - "ENSP_mappings:5xtd", - "Pfam:PF00507", - "PANTHER:PTHR11058", - "PANTHER:PTHR11058" - ], - "major_consequence" : "missense_variant", - "category" : "missense", - "hgvs" : "p.Pro46His", - "major_consequence_rank" : 11, - "transcript_rank" : 0 - } - ], - "start" : 10195, - "transcriptConsequenceTerms" : [ - "missense_variant" - ], - "transcriptIds" : [ - "ENST00000361227" - ], - "variantId" : "M-10195-C-A", - "xpos" : 25000010195, - "xstart" : 25000010195, - "xstop" : 25000010195 - }, - 'matched_queries': {MITO_WGS_INDEX_NAME: ['F000002_2']}, -} - -OR2M3_COMPOUND_HET_ES_VARIANTS = deepcopy(ES_VARIANTS) -transcripts = OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] -transcripts[0]['major_consequence'] = 'frameshift_variant' -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['sortedTranscriptConsequences'] = [transcripts[1], transcripts[0]] -OR2M3_COMPOUND_HET_ES_VARIANTS[0]['_source']['rg37_locus'] = {'contig': '1', 'position': 248367217} -OR2M3_COMPOUND_HET_ES_VARIANTS[1]['_source']['rg37_locus'] = {'contig': '2', 'position': 103343343} -MFSD9_COMPOUND_HET_ES_VARIANTS = deepcopy(OR2M3_COMPOUND_HET_ES_VARIANTS) -for var in MFSD9_COMPOUND_HET_ES_VARIANTS: - var['_source']['variantId'] = '{}-het'.format(var['_source']['variantId']) -EXTRA_FAMILY_ES_VARIANTS = deepcopy(ES_VARIANTS) + [deepcopy(ES_VARIANTS[0])] -EXTRA_FAMILY_ES_VARIANTS[2]['matched_queries'][INDEX_NAME] = ['F000005_5'] -MISSING_SAMPLE_ES_VARIANTS = deepcopy(ES_VARIANTS) -MISSING_SAMPLE_ES_VARIANTS[1]['_source']['samples_num_alt_1'] = [] - -ES_SV_COMP_HET_VARIANT = deepcopy(ES_SV_VARIANT) -ES_SV_COMP_HET_VARIANT['_source']['xpos'] = 2101343374 -ES_SV_COMP_HET_VARIANT['_source']['start'] = 101343374 -ES_SV_COMP_HET_VARIANT['_source']['xstop'] = 1104943628 -ES_SV_COMP_HET_VARIANT['_source']['end'] = 104943628 -ES_SV_COMP_HET_VARIANT['_source']['num_exon'] = 2 -ES_SV_COMP_HET_VARIANT['_source']['variantId'] = 'prefix_191011_DEL' -ES_SV_COMP_HET_VARIANT['_source']['svType'] = 'DEL' -for gen in ES_SV_COMP_HET_VARIANT['_source']['genotypes']: - gen.update({'start': None, 'end': None, 'num_exon': None}) - gen.pop('geneIds') - -COMPOUND_HET_INDEX_VARIANTS = { - INDEX_NAME: { - 'ENSG00000135953': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228198': EXTRA_FAMILY_ES_VARIANTS, - 'ENSG00000228199': EXTRA_FAMILY_ES_VARIANTS, - }, - SECOND_INDEX_NAME: { - 'ENSG00000135953': MFSD9_COMPOUND_HET_ES_VARIANTS, 'ENSG00000228198': OR2M3_COMPOUND_HET_ES_VARIANTS, - }, - '{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME): {'ENSG00000135953': MISSING_SAMPLE_ES_VARIANTS}, - SV_INDEX_NAME: {'ENSG00000228198': [ES_SV_VARIANT], 'ENSG00000135953': []}, - '{},{}'.format(INDEX_NAME, SV_INDEX_NAME): {'ENSG00000228198': [ES_SV_VARIANT, ES_SV_COMP_HET_VARIANT, ES_VARIANTS[1]], 'ENSG00000135953': []}, -} - -INDEX_ES_VARIANTS = { - INDEX_NAME: ES_VARIANTS, - SECOND_INDEX_NAME: [ES_VARIANTS[1]], - SV_INDEX_NAME: [ES_SV_VARIANT], - SV_WGS_INDEX_NAME: [ES_SV_WGS_VARIANT], - HG38_INDEX_NAME: [BUILD_38_ES_VARIANT, BUILD_38_NO_LIFTOVER_ES_VARIANT], - MITO_WGS_INDEX_NAME: [ES_MITO_WGS_VARIANT], -} -INDEX_ES_VARIANTS.update({k: ES_VARIANTS for k in SUB_INDICES + SECOND_SUB_INDICES}) - -PARSED_ANY_AFFECTED_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_ANY_AFFECTED_VARIANTS[1]['familyGuids'] = ['F000003_3'] -PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes'] = {'I000007_na20870': PARSED_ANY_AFFECTED_VARIANTS[1]['genotypes']['I000007_na20870']} - -PARSED_COMPOUND_HET_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_COMPOUND_HET_VARIANTS[0]['_sort'] = [1248367327] -PARSED_COMPOUND_HET_VARIANTS[1]['_sort'] = [2103343453] -PARSED_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000003_3'] - -PARSED_SV_COMPOUND_HET_VARIANTS = [deepcopy(PARSED_SV_VARIANT), deepcopy(PARSED_COMPOUND_HET_VARIANTS[1])] -PARSED_SV_COMPOUND_HET_VARIANTS[0].update({ - '_sort': [2101343474], - 'xpos': 2101343374, - 'pos': 101343374, - 'end': 104943628, - 'variantId': 'prefix_191011_DEL', - 'svType': 'DEL', -}) -del PARSED_SV_COMPOUND_HET_VARIANTS[0]['svSourceDetail'] -PARSED_SV_COMPOUND_HET_VARIANTS[0]['transcripts']['ENSG00000186092'] = [{'geneId': 'ENSG00000186092'}] -for gen in PARSED_SV_COMPOUND_HET_VARIANTS[0]['genotypes'].values(): - gen.update({'start': None, 'end': None, 'numExon': None, 'geneIds': None}) -PARSED_SV_COMPOUND_HET_VARIANTS[1]['familyGuids'] = ['F000002_2'] - -PARSED_COMPOUND_HET_VARIANTS_PROJECT_2 = deepcopy(PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT) -for variant in PARSED_COMPOUND_HET_VARIANTS_PROJECT_2: - variant.update({ - 'variantId': '{}-het'.format(variant['variantId']), - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': variant['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '37', - 'selectedMainTranscriptId': None, - }) - variant['clinvar']['version'] = None - -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS = deepcopy(PARSED_VARIANTS) -PARSED_NO_CONSEQUENCE_FILTER_VARIANTS[1]['selectedMainTranscriptId'] = None - -PARSED_CADD_VARIANTS = deepcopy(PARSED_NO_CONSEQUENCE_FILTER_VARIANTS) -PARSED_CADD_VARIANTS[0]['_sort'][0] = -25.9 -PARSED_CADD_VARIANTS[1]['_sort'][0] = maxsize - - -PARSED_MULTI_INDEX_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_MULTI_INDEX_VARIANT['familyGuids'].append('F000011_11') -PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'] = { - 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, 'sampleType': 'WES', -} - -PARSED_HG38_VARIANT = deepcopy(PARSED_VARIANTS[1]) -PARSED_HG38_VARIANT.update({ - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': PARSED_MULTI_INDEX_VARIANT['genotypes']['I000015_na20885'], - }, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverPos': PARSED_MULTI_INDEX_VARIANT['pos'], - 'liftedOverChrom': PARSED_MULTI_INDEX_VARIANT['chrom'], - 'pos': PARSED_MULTI_INDEX_VARIANT['pos'] + 10, - 'xpos': PARSED_MULTI_INDEX_VARIANT['xpos'] + 10, - 'variantId': PARSED_MULTI_INDEX_VARIANT['variantId'].replace( - str(PARSED_MULTI_INDEX_VARIANT['pos']), str(PARSED_MULTI_INDEX_VARIANT['pos'] + 10) - ), - '_sort': [PARSED_MULTI_INDEX_VARIANT['_sort'][0] + 10], -}) -PARSED_HG38_VARIANT['clinvar']['version'] = None - -PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -for guid, genotype in PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_MULTI_INDEX_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT = deepcopy(PARSED_VARIANTS[1]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_VARIANT_0 = deepcopy(PARSED_VARIANTS[0]) -for guid, genotype in PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'].items(): - PARSED_MULTI_SAMPLE_VARIANT_0['genotypes'][guid] = dict(otherSample=genotype, **genotype) - -PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS = deepcopy(PARSED_COMPOUND_HET_VARIANTS) -for variant in PARSED_MULTI_SAMPLE_COMPOUND_HET_VARIANTS: - for guid, genotype in variant['genotypes'].items(): - variant['genotypes'][guid] = dict(otherSample=genotype, **genotype) - - -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT = deepcopy(PARSED_MULTI_INDEX_VARIANT) -PARSED_ANY_AFFECTED_MULTI_INDEX_VERSION_VARIANT.update({ - 'familyGuids': ['F000003_3', 'F000011_11'], - 'genotypes': { - ind_guid: PARSED_MULTI_INDEX_VARIANT['genotypes'][ind_guid] - for ind_guid in ['I000007_na20870', 'I000015_na20885'] - }, -}) -MAPPING_FIELDS = [ - 'start', - 'end', - 'rsid', - 'originalAltAlleles', - 'filters', - 'xpos', - 'alt', - 'ref', - 'contig', - 'variantId', - 'dbnsfp_MutationTaster_pred', - 'mpc_MPC', - 'dbnsfp_DANN_score', - 'eigen_Eigen_phred', - 'dbnsfp_REVEL_score', - 'splice_ai_delta_score', - 'splice_ai_splice_consequence', - 'dbnsfp_FATHMM_pred', - 'dbnsfp_fathmm_MKL_coding_pred', - 'dbnsfp_MutPred_score', - 'dbnsfp_VEST4_score', - 'primate_ai_score', - 'dbnsfp_SIFT_pred', - 'dbnsfp_Polyphen2_HVAR_pred', - 'cadd_PHRED', - 'gnomad_non_coding_constraint_z_score', - 'sortedTranscriptConsequences', - 'screen_region_type', - 'genotypes', - 'samples_no_call', - 'samples_num_alt_1', - 'samples_num_alt_2', - 'clinvar_clinical_significance', - 'clinvar_allele_id', - 'clinvar_variation_id', - 'clinvar_gold_stars', - 'hgmd_accession', - 'hgmd_class', - 'AC', - 'AF', - 'AN', - 'gnomad_genomes_AC', - 'gnomad_genomes_Hom', - 'gnomad_genomes_Hemi', - 'gnomad_genomes_AF', - 'gnomad_genomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_genomes_AN', - 'gnomad_genomes_Het', - 'gnomad_genomes_ID', - 'gnomad_exomes_AC', - 'gnomad_exomes_Hom', - 'gnomad_exomes_Hemi', - 'gnomad_exomes_AF', - 'gnomad_exomes_AF_POPMAX_OR_GLOBAL', - 'gnomad_exomes_AN', - 'gnomad_exomes_Het', - 'gnomad_exomes_ID', - 'exac_AC_Adj', - 'exac_AC_Hom', - 'exac_AC_Hemi', - 'exac_AF_POPMAX', - 'exac_AF', - 'exac_AN_Adj', - 'exac_Het', - 'exac_ID', - 'topmed_AC', - 'topmed_Hom', - 'topmed_Hemi', - 'topmed_Het', - 'topmed_AF', - 'topmed_AN', - 'topmed_ID', - 'gnomad_genomes_FAF_AF', - 'rg37_locus', - 'rg37_locus_end', - 'xstop', - 'bothsides_support', -] - -SV_MAPPING_FIELDS = [ - 'start', - 'end', - 'xpos', - 'contig', - 'variantId', - 'sortedTranscriptConsequences', - 'genotypes', - 'samples', - 'samples_cn_0', - 'samples_cn_1', - 'samples_cn_2', - 'samples_cn_3', - 'samples_cn_gte_4', - 'sf', - 'sc', - 'sn', - 'num_exon', - 'svType', - 'StrVCTVRE_score', - 'sv_type_detail', - 'cpx_intervals', - 'gnomad_svs_AC', - 'gnomad_svs_Hom', - 'gnomad_svs_Hemi', - 'gnomad_svs_AF', - 'gnomad_svs_AN', - 'gnomad_svs_filter_AF', - 'gnomad_svs_Het', - 'gnomad_svs_ID', - 'bothsides_support', -] - -MITO_MAPPING_FIELDS = [ - "AC", - "AC_het", - "AF", - "AF_het", - "AN", - "alt", - "clinvar_allele_id", - "clinvar_clinical_significance", - "clinvar_gold_stars", - "common_low_heteroplasmy", - "contig", - "dbnsfp_FATHMM_pred", - "dbnsfp_MutationTaster_pred", - "dbnsfp_Polyphen2_HVAR_pred", - "dbnsfp_REVEL_score", - "dbnsfp_SIFT_pred", - "end", - "filters", - "genotypes", - "gnomad_mito_AC", - "gnomad_mito_AC_het", - "gnomad_mito_AF", - "gnomad_mito_AF_het", - "gnomad_mito_AN", - "gnomad_mito_max_hl", - "hap_defining_variant", - "helix_AC", - "helix_AC_het", - "helix_AF", - "helix_AF_het", - "helix_max_hl", - "high_constraint_region", - "hmtvar_hmtVar", - "mitimpact_apogee", - "mitomap_pathogenic", - "mitotip_mitoTIP", - "ref", - "rg37_locus", - "rsid", - "samples_num_alt_1", - "samples_num_alt_2", - "sortedTranscriptConsequences", - "start", - "variantId", - "xpos", - "xstop", -] - -MITO_SOURCE_ONLY_FIELDS = [ - 'callset_max_hl', - 'exac_max_hl', - 'gnomad_exomes_max_hl', - 'gnomad_genomes_max_hl', - 'gnomad_svs_max_hl', - 'sv_callset_max_hl', - 'topmed_max_hl', - 'helix_ID', - 'helix_Hemi', - 'helix_AN', - 'helix_filter_AF', - 'helix_Het', - 'helix_Hom', - 'gnomad_mito_filter_AF', - 'gnomad_mito_Hom', - 'gnomad_mito_ID', - 'gnomad_mito_Het', - 'gnomad_mito_Hemi', - 'callset_heteroplasmy_Het', - 'callset_heteroplasmy_ID', - 'callset_heteroplasmy_Hemi', - 'callset_heteroplasmy_filter_AF', - 'callset_heteroplasmy_Hom', - 'callset_heteroplasmy_max_hl', - 'gnomad_mito_heteroplasmy_filter_AF', - 'gnomad_mito_heteroplasmy_Hemi', - 'gnomad_mito_heteroplasmy_Hom', - 'gnomad_mito_heteroplasmy_ID', - 'gnomad_mito_heteroplasmy_Het', - 'helix_heteroplasmy_Het', - 'helix_heteroplasmy_ID', - 'helix_heteroplasmy_Hemi', - 'helix_heteroplasmy_filter_AF', - 'helix_heteroplasmy_Hom', -] - -SOURCE_FIELDS = { - 'homozygote_count', 'callset_Hemi', 'callset_Het', 'callset_ID', 'sv_callset_Hemi', - 'sv_callset_Hom', 'sv_callset_Het', 'sv_callset_ID', 'algorithms', -} -SOURCE_FIELDS.update(MAPPING_FIELDS) -SOURCE_FIELDS.update(SV_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_MAPPING_FIELDS) -SOURCE_FIELDS.update(MITO_SOURCE_ONLY_FIELDS) -SOURCE_FIELDS -= { - 'samples_no_call', 'samples_cn_0', 'samples_cn_1', 'samples_cn_2', 'samples_cn_3', 'samples_cn_gte_4', 'topmed_Het', - 'gnomad_genomes_FAF_AF', -} - -FIELD_TYPE_MAP = { - 'cadd_PHRED': {'type': 'keyword'}, - 'primate_ai_score': {'type': 'float'}, - 'rg37_locus': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}}, - 'rg37_locus_end': {'properties': {'contig': {'type': 'keyword'}, 'position': {'type': 'integer'}}} -} -MAPPING_PROPERTIES = {field: FIELD_TYPE_MAP.get(field, {'type': 'keyword'}) for field in MAPPING_FIELDS} - -CORE_INDEX_METADATA = { - INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'clinvar_version': '2023-03-05'}, - 'properties': MAPPING_PROPERTIES, - }, - SECOND_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'VARIANTS'}, - 'properties': MAPPING_PROPERTIES, - }, - SV_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'SV'}, - 'properties': {field: {'type': 'keyword'} for field in SV_MAPPING_FIELDS}, - }, - MITO_WGS_INDEX_NAME: { - '_meta': {'genomeVersion': '37', 'datasetType': 'MITO'}, - 'properties': {field: {'type': 'keyword'} for field in MITO_MAPPING_FIELDS}, - }, -} -INDEX_METADATA = { - HG38_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SECOND_INDEX_NAME]), - SV_WGS_INDEX_NAME: deepcopy(CORE_INDEX_METADATA[SV_INDEX_NAME]), -} -for meta in INDEX_METADATA.values(): - meta['_meta']['genomeVersion'] = '38' -INDEX_METADATA.update(CORE_INDEX_METADATA) - -ALL_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples_num_alt_2': ['HG00731', 'HG00732', 'HG00733']}}, - {'terms': {'samples': ['HG00731', 'HG00732', 'HG00733']}}, - ]}} - ], - '_name': 'F000002_2' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20870']}}, - {'terms': {'samples_num_alt_2': ['NA20870']}}, - {'terms': {'samples': ['NA20870']}}, - ]}} - ], - '_name': 'F000003_3' - }}, - {'bool': { - 'must': [ - {'bool': {'should': [ - {'terms': {'samples_num_alt_1': ['NA20874']}}, - {'terms': {'samples_num_alt_2': ['NA20874']}}, - {'terms': {'samples': ['NA20874']}}, - ]}} - ], - '_name': 'F000005_5' - }} - ] - } -} - -COMPOUND_HET_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_1': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_1': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -COMPOUND_HET_PATH_INHERITANCE_QUERY = deepcopy(COMPOUND_HET_INHERITANCE_QUERY) -for fam_q in COMPOUND_HET_PATH_INHERITANCE_QUERY['bool']['should']: - fam_quality_q = fam_q['bool']['must'][1] - fam_quality_q['bool'] = {'should': [ - deepcopy(fam_quality_q), - {'regexp': {'clinvar_clinical_significance': '.*Pathogenic.*'}} - ]} - -RECESSIVE_INHERITANCE_QUERY = { - 'bool': { - 'should': [ - {'bool': { - '_name': 'F000002_2', - 'must': [ - {'bool': { - 'must_not': [ - {'term': {'samples_no_call': 'HG00732'}}, - {'term': {'samples_num_alt_2': 'HG00732'}}, - {'term': {'samples_no_call': 'HG00733'}}, - {'term': {'samples_num_alt_2': 'HG00733'}} - ], - 'must': [{'term': {'samples_num_alt_2': 'HG00731'}}] - }}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}}, - ] - }}, - {'bool': { - '_name': 'F000003_3', - 'must': [ - {'term': {'samples_num_alt_2': 'NA20870'}}, - {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'NA20870'}}, - {'term': {'samples_gq_5_to_10': 'NA20870'}} - ]}} - ] - }}, - ] - } -} - -ANNOTATION_QUERY = {'terms': {'transcriptConsequenceTerms': ['frameshift_variant']}} - -REDIS_CACHE = {} -def _set_cache(k, v): - REDIS_CACHE[k] = v -MOCK_REDIS = mock.MagicMock() -MOCK_REDIS.get.side_effect = REDIS_CACHE.get -MOCK_REDIS.set.side_effect =_set_cache - -def mock_hits(hits, increment_sort=False, include_matched_queries=True, sort=None, index=INDEX_NAME): - parsed_hits = deepcopy(hits) - for hit in parsed_hits: - hit.update({ - '_index': index, - '_id': hit['_source']['variantId'], - '_type': 'structural_variant' if SV_INDEX_NAME in index or SV_WGS_INDEX_NAME in index else 'variant', - }) - matched_queries = hit.pop('matched_queries') - - if sort or increment_sort: - sort_key = sort[0] if sort else 'xpos' - sort_value = jmespath.search(sort_key, hit['_source']) - hit['_sort'] = [sort_value] - return parsed_hits - - -def create_mock_response(search, index=INDEX_NAME): - index = ALIAS_MAP.get(index, index) - indices = index.split(',') - include_matched_queries = False - variant_id_filters = None - gene_ids_filters = set() - if 'query' in search: - for search_filter in search['query']['bool']['filter']: - if not variant_id_filters: - variant_id_filters = search_filter.get('terms', {}).get('variantId') - if not gene_ids_filters and not search.get('aggs'): - gene_ids_filters.update(search_filter.get('terms', {}).get('geneIds') or []) - - response_dict = { - 'took': 1, - 'hits': {'total': {'value': 5}, 'hits': []} - } - for index_name in sorted(indices): - index_hits = mock_hits( - INDEX_ES_VARIANTS[index_name], include_matched_queries=include_matched_queries, sort=search.get('sort'), - index=index_name) - if variant_id_filters: - index_hits = [hit for hit in index_hits if hit['_id'] in variant_id_filters] - response_dict['hits']['hits'] += index_hits - - try: - response_dict['hits']['hits'] = sorted(response_dict['hits']['hits'], key=lambda v: v['_sort']) - except (KeyError, TypeError): - pass - - return response_dict - -def get_indices_from_url(url): - return url.split('/')[1] - -def get_metadata_callback(request): - indices = get_indices_from_url(request.url).split(',') - response = {index: {'mappings': INDEX_METADATA[index]} for index in indices} - return 200, {}, json.dumps(response) - -def get_search_callback(request): - body = json.loads(request.body) - response = create_mock_response(body, get_indices_from_url(request.url)) - return 200, {}, json.dumps(response) - -def get_msearch_callback(request): - body = parse_msearch_body(request.body) - response = { - 'responses': [ - create_mock_response(exec_search, index=','.join(body[i - 1]['index'])) - for i, exec_search in enumerate(body) if not exec_search.get('index')] - } - return 200, {}, json.dumps(response) - -def setup_search_responses(): - urllib3_responses.add_callback( - urllib3_responses.POST, '/_msearch', callback=get_msearch_callback, - content_type='application/json', match_querystring=True) - urllib3_responses.add_callback( - urllib3_responses.POST, re.compile('^/[,\w]+/_search$'), callback=get_search_callback, - content_type='application/json', match_querystring=True) - -def setup_responses(): - urllib3_responses.add_callback( - urllib3_responses.GET, re.compile('^/[,\w]+/_mapping$'), callback=get_metadata_callback, - content_type='application/json', match_querystring=True) - setup_search_responses() - - -@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') -@mock.patch('seqr.utils.redis_utils.redis.StrictRedis', lambda **kwargs: MOCK_REDIS) -class EsUtilsTest(TestCase): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - Sample.objects.filter(sample_id='NA19678').update(is_active=False) - Sample.objects.filter(sample_id='NA20885').update(sample_type='WES') - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - - def assertExecutedSearch(self, filters=None, start_index=0, size=2, index=INDEX_NAME, expected_source_fields=SOURCE_FIELDS, call_index=-1, **kwargs): - executed_search = urllib3_responses.call_request_json(index=call_index) - searched_indices = get_indices_from_url(urllib3_responses.calls[call_index].request.url) - self.assertListEqual(sorted(searched_indices.split(',')), sorted(index.split(','))) - self.assertSameSearch( - executed_search, - dict(filters=filters, start_index=start_index, size=size, **kwargs), - expected_source_fields=expected_source_fields, - ) - - def assertSameSearch(self, executed_search, expected_search_params, expected_source_fields=SOURCE_FIELDS): - expected_search = { - 'from': expected_search_params['start_index'], - 'size': expected_search_params['size'] - } - - if expected_search_params['filters']: - expected_search['query'] = { - 'bool': { - 'filter': expected_search_params['filters'] - } - } - - if not expected_search_params.get('unsorted'): - expected_search['sort'] = expected_search_params.get('sort') or ['xpos', 'variantId'] - - if expected_search_params.get('gene_aggs'): - expected_search['aggs'] = { - 'genes': {'terms': {'field': 'geneIds', 'min_doc_count': 2, 'size': 1001}, 'aggs': { - 'vars_by_gene': { - 'top_hits': {'sort': expected_search['sort'], '_source': mock.ANY, 'size': 100} - } - }}} - else: - expected_search['_source'] = mock.ANY - self.assertDictEqual(executed_search, expected_search) - - if not expected_search_params.get('gene_count_aggs'): - source = executed_search['aggs']['genes']['aggs']['vars_by_gene']['top_hits']['_source'] \ - if expected_search_params.get('gene_aggs') else executed_search['_source'] - self.assertSetEqual(expected_source_fields, set(source)) - - @urllib3_responses.activate - def test_get_es_variants_for_variant_ids(self): - setup_responses() - samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') - get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) - self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], - size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), - ) - - def test_get_family_affected_status(self): - samples_by_id = { - sample_id: Sample.objects.get(sample_id=sample_id, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - for sample_id in ['HG00731', 'HG00732', 'HG00733'] - } - custom_affected = {'I000004_hg00731': 'N', 'I000005_hg00732': 'A'} - custom_multi_affected = {'I000005_hg00732': 'A'} - - self.assertDictEqual(_get_family_affected_status(samples_by_id, {}), { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'N', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) - - custom_affected_status = _get_family_affected_status(samples_by_id, {'affected': custom_multi_affected}) - self.assertDictEqual(custom_affected_status, { - 'I000004_hg00731': 'A', 'I000005_hg00732': 'A', 'I000006_hg00733': 'N'}) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 3dd6e69d91..4d2ad4eac0 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -291,7 +291,6 @@ def test_edit_families_handler(self, mock_pm_group): 'families': [{'familyGuid': 'F000012_12'}]})) self.assertEqual(response.status_code, 200) - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') @mock.patch('seqr.views.utils.permissions_utils.INTERNAL_NAMESPACES') @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): @@ -491,7 +490,7 @@ def test_update_family_fields(self): self.assertEqual(response_json['F000014_14']['displayName'], expected_id) def _anvil_enabled(self): - return not self.ES_HOSTNAME + return bool(self.MOCK_AIRTABLE_KEY) @mock.patch('seqr.views.utils.file_utils.anvil_enabled', lambda: False) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 8aa57fc9b8..4af77065fb 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -59,7 +59,6 @@ class AuthenticationTestMixin(object): AUTHENTICATED_USER = 'authenticated' NO_POLICY_USER = 'no_policy' - ES_HOSTNAME = 'testhost' MOCK_AIRTABLE_KEY = '' CLICKHOUSE_HOSTNAME = '' @@ -74,9 +73,6 @@ class AuthenticationTestMixin(object): no_policy_user = None def set_up_test(self): - patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) - patcher.start() - self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_API_KEY', self.MOCK_AIRTABLE_KEY) patcher.start() self.addCleanup(patcher.stop) @@ -552,7 +548,6 @@ def get_group_members_side_effect(user, group, use_sa_credentials=False): class AnvilAuthenticationTestMixin(AuthenticationTestMixin): - ES_HOSTNAME = '' CLICKHOUSE_HOSTNAME = 'testhost' MOCK_AIRTABLE_KEY = 'airflow_access' SKIP_RESET_VARIANT_JSON = False From 6a668af60527c0fabe12f302d11a5d9d30696a88 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 14:28:33 -0500 Subject: [PATCH 077/252] improve coverage --- seqr/utils/search/elasticsearch/es_utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index cd1d32c179..280c44ea45 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -62,17 +62,6 @@ def get_es_client(timeout=60, **kwargs): return Elasticsearch(**client_kwargs, **kwargs) -def ping_elasticsearch(): - if not get_es_client(timeout=3, max_retries=0).ping(): - raise ValueError('No response from elasticsearch ping') - - -def ping_kibana(): - resp = connection_from_url('http://{}'.format(KIBANA_SERVER)).urlopen('HEAD', '/status', timeout=3, retries=3) - if resp.status >= 400: - raise ValueError('Kibana Error {}: {}'.format(resp.status, resp.reason)) - - SAMPLE_FIELDS_LIST = ['samples', 'samples_num_alt_1'] # support .bgz instead of requiring .vcf.bgz due to issues with DSP delivery of large callsets DATASET_FILE_EXTENSIONS = VCF_FILE_EXTENSIONS[:-1] + ('.bgz', '.bed', '.mt') From 41988c94cb8a3ce5f7ed012d738d09126f44a392 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 14:31:19 -0500 Subject: [PATCH 078/252] remove unused import --- seqr/utils/search/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 690deef6f5..66ceb9f7b6 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -10,8 +10,7 @@ from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ - MAX_VARIANTS + MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom From d9859304bc33f24a6c2373e7b107cabd666f2707 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 14:37:26 -0500 Subject: [PATCH 079/252] move some invalid search tests --- clickhouse_search/search_tests.py | 15 +++++++++++++++ seqr/utils/search/search_utils_tests.py | 22 ---------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 50527e1ab6..e5c894ba08 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -801,6 +801,21 @@ def test_invalid_search(self): self._assert_expected_search([], annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive') self.assertEqual(str(cm.exception),'Location must be specified to search for compound heterozygous variants across many families') + self.results_model.families.set(Family.objects.filter(guid='F000005_5')) + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], inheritance_mode='recessive') + self.assertEqual(str(cm.exception),'Inheritance based search is disabled in families with no data loaded for affected individuals') + + self.results_model.families.set(Family.objects.filter(guid='F000003_3')) + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], pathogenicity={}, annotations={'structural': ['DEL']}) + self.assertEqual(str(cm.exception), 'Unable to search against dataset type "SV"') + + self.search_model.search['annotations_secondary'] = {'frameshift': ['frameshift_variant']} + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], inheritance_mode='recessive', annotations_secondary={'frameshift': ['frameshift_variant']}) + self.assertEqual(str(cm.exception),'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals') + Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index da0686ac16..483bc9b8f7 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -72,28 +72,6 @@ def setUp(self): @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def test_invalid_search_params(self): - self.results_model.families.set([family for family in self.families if family.guid == 'F000005_5']) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Inheritance based search is disabled in families with no data loaded for affected individuals', - ) - - self.results_model.families.set([family for family in self.families if family.guid == 'F000003_3']) - self.search_model.search['annotations'] = {'structural': ['DEL']} - self.search_model.search['pathogenicity'] = {} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Unable to search against dataset type "SV"') - - self.search_model.search['annotations_secondary'] = {'frameshift': ['frameshift_variant']} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals', - ) self.search_model.search['inheritance']['filter'] = {'affected': {'I000007_na20870': 'N'}} with self.assertRaises(InvalidSearchException) as cm: From 3fe0f3b2fbed8432779c8575911df0466c0577d6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 14:55:52 -0500 Subject: [PATCH 080/252] settings and kibana clean up --- seqr/urls.py | 13 +-------- seqr/views/apis/data_manager_api.py | 35 +++++------------------ seqr/views/apis/data_manager_api_tests.py | 30 +++---------------- settings.py | 23 --------------- 4 files changed, 12 insertions(+), 89 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index cf2d551d2b..8204a9f8f4 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -120,7 +120,7 @@ update_user, \ forgot_password -from seqr.views.apis.data_manager_api import update_rna_seq, proxy_to_kibana, load_phenotype_prioritization_data, \ +from seqr.views.apis.data_manager_api import update_rna_seq, load_phenotype_prioritization_data, \ validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ trigger_delete_project, trigger_delete_family from seqr.views.apis.report_api import \ @@ -389,17 +389,6 @@ handler401 = 'seqr.views.apis.auth_api.app_login_required_error' -kibana_urls = '^(?:{})'.format('|'.join([ - 'app', '\d+/built_assets', '\d+/bundles', 'bundles', 'elasticsearch', 'es_admin', 'node_modules/@kbn', 'internal', - 'plugins', 'translations', 'ui', 'api/apm', 'api/console', 'api/core', 'api/index_management', 'api/index_patterns', - 'api/kibana', 'api/licensing', 'api/monitoring', 'api/reporting', 'api/saved_objects', 'api/telemetry', - 'api/timelion', 'api/ui_metric', 'api/xpack', 'bootstrap', -])) - -urlpatterns += [ - re_path(kibana_urls, proxy_to_kibana, name='proxy_to_kibana'), -] - urlpatterns += [ re_path(r'^admin/login/$', RedirectView.as_view(url=LOGIN_URL, permanent=True, query_string=True)), re_path(r'^admin/', admin.site.urls), diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index f3f19295c1..4c5b5543ab 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -1,4 +1,3 @@ -import base64 from collections import defaultdict import json import os @@ -8,7 +7,6 @@ from django.core.exceptions import PermissionDenied from django.db.models import Max, F, Q from django.http.response import HttpResponse -from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError from clickhouse_search.search import delete_clickhouse_project @@ -27,8 +25,7 @@ from seqr.models import Sample, RnaSample, Individual, Project, PhenotypePrioritization -from settings import KIBANA_SERVER, KIBANA_ELASTICSEARCH_PASSWORD, KIBANA_ELASTICSEARCH_USER, \ - LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT +from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT logger = SeqrLogger(__name__) @@ -426,25 +423,16 @@ def trigger_delete_family(request): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - @data_manager_required -@csrf_exempt -def proxy_to_kibana(request): - headers = {} - if KIBANA_ELASTICSEARCH_PASSWORD: - token = base64.b64encode('{}:{}'.format(KIBANA_ELASTICSEARCH_USER, KIBANA_ELASTICSEARCH_PASSWORD).encode('utf-8')) - headers['Authorization'] = 'Basic {}'.format(token.decode('utf-8')) - return _proxy_iframe_page(request, 'Kibana', KIBANA_SERVER, additional_headers=headers) - +def proxy_to_luigi(request): + if not LUIGI_UI_SERVICE_HOSTNAME: + return HttpResponse('Loading Pipeline UI is not configured', status=404) -def _proxy_iframe_page(request, page_name, host, additional_headers=None, path_prefix=None): headers = convert_django_meta_to_http_headers(request) + host = f'{LUIGI_UI_SERVICE_HOSTNAME}:{LUIGI_UI_SERVICE_PORT}' headers['Host'] = host - headers.update(additional_headers or {}) - path = request.get_full_path() - if path_prefix: - path = path.replace(path_prefix, '') + path = request.get_full_path().replace('/luigi_ui', '') url = f'http://{host}{path}' request_method = getattr(requests.Session(), request.method.lower()) @@ -472,13 +460,4 @@ def _proxy_iframe_page(request, page_name, host, additional_headers=None, path_p return proxy_response except (ConnectionError, RequestConnectionError) as e: logger.error(str(e), request.user) - return HttpResponse(f'Error: Unable to connect to {page_name} {e}', status=400) - - -@data_manager_required -def proxy_to_luigi(request): - if not LUIGI_UI_SERVICE_HOSTNAME: - return HttpResponse('Loading Pipeline UI is not configured', status=404) - return _proxy_iframe_page( - request, 'Luigi UI', f'{LUIGI_UI_SERVICE_HOSTNAME}:{LUIGI_UI_SERVICE_PORT}', path_prefix='/luigi_ui', - ) + return HttpResponse(f'Error: Unable to connect to Luigi UI {e}', status=400) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 7cb8e69b8c..b07ad525d8 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -347,30 +347,14 @@ class DataManagerAPITest(AirtableTest): VCF_SAMPLES = VCF_SAMPLES SKIP_TDR = False - @mock.patch('seqr.views.apis.data_manager_api.KIBANA_ELASTICSEARCH_PASSWORD', 'abc123') - @responses.activate - def test_kibana_proxy(self): - url = '/api/kibana/random/path' - self.check_data_manager_login(url) - - self._test_request_proxy('localhost:5601', url, auth_header='Basic a2liYW5hOmFiYzEyMw==') - - # Test with error response - response = self.client.get('{}/bad_response'.format(url)) - self.assertEqual(response.status_code, 500) - - # Test with connection error - response = self.client.get('{}/bad_path'.format(url)) - self.assertContains(response, 'Error: Unable to connect to Kibana', status_code=400) - - def _test_request_proxy(self, host, url, auth_header=None, proxy_path=None): + def _test_request_proxy(self, host, url, proxy_path): response_args = { 'stream': True, 'body': 'Test response', 'content_type': 'text/custom', 'headers': {'x-test-header': 'test', 'keep-alive': 'true'}, } - proxy_url = f'http://{host}{proxy_path or url}' + proxy_url = f'http://{host}{proxy_path}' responses.add(responses.GET, proxy_url, status=200, **response_args) responses.add(responses.POST, proxy_url, status=201, **response_args) responses.add(responses.GET, '{}/bad_response'.format(proxy_url), body=HTTPError()) @@ -391,20 +375,14 @@ def _test_request_proxy(self, host, url, auth_header=None, proxy_path=None): get_request = responses.calls[0].request self.assertEqual(get_request.headers['Host'], host) self.assertEqual(get_request.headers['Test-Header'], 'some/value') - if auth_header: - self.assertEqual(get_request.headers['Authorization'], auth_header) - else: - self.assertFalse('Authorization' in get_request.headers) + self.assertFalse('Authorization' in get_request.headers) post_request = responses.calls[1].request self.assertEqual(post_request.headers['Host'], host) self.assertEqual(post_request.headers['Content-Type'], 'application/json') self.assertEqual(post_request.headers['Content-Length'], '26') self.assertEqual(post_request.body, data.encode('utf-8')) - if auth_header: - self.assertEqual(get_request.headers['Authorization'], auth_header) - else: - self.assertFalse('Authorization' in get_request.headers) + self.assertFalse('Authorization' in get_request.headers) @mock.patch('seqr.views.apis.data_manager_api.LUIGI_UI_SERVICE_HOSTNAME') @responses.activate diff --git a/settings.py b/settings.py index f0b0166431..94fc51a8b3 100644 --- a/settings.py +++ b/settings.py @@ -358,29 +358,6 @@ IS_ANVIL_LOADING_DELAY = bool(os.environ.get('IS_ANVIL_LOADING_DELAY')) # External service settings -ELASTICSEARCH_SERVICE_HOSTNAME = os.environ.get('ELASTICSEARCH_SERVICE_HOSTNAME') -ELASTICSEARCH_SERVICE_PORT = os.environ.get('ELASTICSEARCH_SERVICE_PORT', '9200') -ELASTICSEARCH_SERVER = '{host}:{port}'.format( - host=ELASTICSEARCH_SERVICE_HOSTNAME, port=ELASTICSEARCH_SERVICE_PORT) - -SEQR_ELASTICSEARCH_PASSWORD = os.environ.get('SEQR_ES_PASSWORD') -ELASTICSEARCH_USER = os.environ.get('ELASTICSEARCH_USER', 'seqr') -ELASTICSEARCH_CREDENTIALS = (ELASTICSEARCH_USER, SEQR_ELASTICSEARCH_PASSWORD) if SEQR_ELASTICSEARCH_PASSWORD else None -ELASTICSEARCH_PROTOCOL = os.environ.get('ELASTICSEARCH_PROTOCOL', 'http') -ELASTICSEARCH_CA_PATH = os.environ.get('ELASTICSEARCH_CA_PATH') -# if we have a custom CA certificate for elasticsearch, add it to the verification path for connections -if ELASTICSEARCH_CA_PATH: - ES_SSL_CONTEXT = create_default_context(cafile=ELASTICSEARCH_CA_PATH) -else: - ES_SSL_CONTEXT = None - -KIBANA_SERVER = '{host}:{port}'.format( - host=os.environ.get('KIBANA_SERVICE_HOSTNAME', 'localhost'), - port=os.environ.get('KIBANA_SERVICE_PORT', 5601) -) -KIBANA_ELASTICSEARCH_USER = os.environ.get('KIBANA_ELASTICSEARCH_USER', 'kibana') -KIBANA_ELASTICSEARCH_PASSWORD = os.environ.get('KIBANA_ES_PASSWORD') - REDIS_SERVICE_HOSTNAME = os.environ.get('REDIS_SERVICE_HOSTNAME', 'localhost') REDIS_SERVICE_PORT = int(os.environ.get('REDIS_SERVICE_PORT', '6379')) From a00805554363461551bbc6ac84849290ebc397df Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 15:08:43 -0500 Subject: [PATCH 081/252] remove es python packages --- requirements.in | 2 -- requirements.txt | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/requirements.in b/requirements.in index b416c0bd3b..0039bcce8c 100644 --- a/requirements.in +++ b/requirements.in @@ -9,8 +9,6 @@ django-cors-headers # allows CORS requests for client-side develop django-storages[google] # alternative GCS storage backend for the django media_root social-auth-app-django>5.0.0 # the package for Django to authenticate users with social medieas social-auth-core # the Python social authentication package. Required by social-auth-app-django -elasticsearch==7.9.1 # elasticsearch client -elasticsearch-dsl==7.2.1 # elasticsearch query utilities gunicorn # web server jmespath openpyxl # library for reading/writing Excel files diff --git a/requirements.txt b/requirements.txt index 53b74e6f0f..2ec95bc2af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ cachetools==5.3.0 # via google-auth certifi==2024.7.4 # via - # elasticsearch # requests cffi==1.15.1 # via cryptography @@ -60,12 +59,6 @@ django-notifications-hq==1.8.3 # via -r requirements.in django-storages[google]==1.11.1 # via -r requirements.in -elasticsearch==7.9.1 - # via - # -r requirements.in - # elasticsearch-dsl -elasticsearch-dsl==7.2.1 - # via -r requirements.in et-xmlfile==1.1.0 # via openpyxl feedparser==6.0.10 @@ -127,8 +120,6 @@ pyjwt==2.8.0 # via social-auth-core pyliftover==0.4 # via -r requirements.in -python-dateutil==2.8.2 - # via elasticsearch-dsl python3-openid==3.2.0 # via social-auth-core pytz==2022.7.1 @@ -157,10 +148,8 @@ sgmllib3k==1.0.0 # via feedparser six==1.16.0 # via - # elasticsearch-dsl # google-auth # markdownify - # python-dateutil slacker==0.14.0 # via -r requirements.in slugify==0.0.1 @@ -187,7 +176,6 @@ tzlocal==5.3.1 # via clickhouse-driver urllib3==2.6.3 # via - # elasticsearch # requests whitenoise==6.3.0 # via -r requirements.in From 0453e25335ec604baa85066ae61464ec81eb770a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 15:49:31 -0500 Subject: [PATCH 082/252] handle invalid tissue and report all errors per sample --- seqr/views/apis/data_manager_api.py | 19 +++++++++++-------- seqr/views/apis/data_manager_api_tests.py | 11 ++++++----- seqr/views/utils/dataset_utils.py | 5 +++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 81ce98e2b2..b5074f3cbb 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -79,21 +79,24 @@ def update_rna_seq(request): ], ) sample_metadata_mapping = {} - misconfigured_samples = {} + misconfigured_samples = defaultdict(list) for sample in airtable_samples: sample_ids = [ sample[sample_id_field] for sample_id_field in ['sample_id', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'] if sample.get(sample_id_field) ] - error_message = None + error_messages = [] if len(sample.get(TISSUE_FIELD, [])) != 1: - error_message = 'no tissue specified' if not sample.get(TISSUE_FIELD) else 'multiple tissues specified' - elif len(sample['pdos']) != 1: - error_message = 'multiple conflicting PDOs' + error_messages.append('no tissue specified' if not sample.get(TISSUE_FIELD) else 'multiple tissues specified') + elif sample[TISSUE_FIELD][0] not in TISSUE_TYPE_MAP: + error_messages.append('invalid tissue specified') + if len(sample['pdos']) != 1: + error_messages.append('multiple conflicting PDOs') elif sample['pdos'][0]['project_guid'] is None: - error_message = 'no project specified' - if error_message: - misconfigured_samples.update({sample_id: error_message for sample_id in sample_ids}) + error_messages.append('no project specified') + if error_messages: + for sample_id in sample_ids: + misconfigured_samples[sample_id].extend(error_messages) else: metadata = { 'tissue': TISSUE_TYPE_MAP[sample[TISSUE_FIELD][0]], diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3f47f30e63..4064964665 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -505,9 +505,8 @@ 'CollaboratorSampleID': 'NA12346', 'SeqrProject': [ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', - 'https://seqr.broadinstitute.org/project/R0004_non_analyst_project/project_page', ], - 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], + 'PDOStatus': ['RNA ready to load'], } }, { @@ -528,7 +527,7 @@ 'CollaboratorSampleID': 'NA12348', 'SeqrProject': ['R0003_test'], 'PDOStatus': ['RNA ready to load'], - 'TissueOfOrigin': ['Muscle'], + 'TissueOfOrigin': ['Custom Tissue'], } }, { @@ -937,10 +936,11 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to load the following samples from Airtable with no corresponding seqr ID: NA11111', + 'Unable to load the following samples that are improperly configured in Airtable with invalid tissue specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345, NA12347', 'Unable to load the following samples that are improperly configured in Airtable with multiple tissues specified: NA12345', - 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', - 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12347', 'Unable to load the following samples that are improperly configured in Airtable with no project specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', ]) _set_gzip_file_iter_stdout(mock_files[f'tmp/temp_uploads/{file_path}/muscle_samples.tsv.gz'], [header, loaded_data_row]) @@ -1016,6 +1016,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s self.reset_logs() body.update({'ignoreExtraSamples': True, 'file': RNA_FILE_ID}) warnings = [ + 'Skipped loading for the following 1 samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345', 'Skipped loading for the following 1 samples that are improperly configured in Airtable with multiple tissues specified: NA12345', f'Skipped loading for the following {len(params["skipped_samples"].split(","))} ' f'samples with no match: {params["skipped_samples"]}'] diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 88f62eb1a8..fc7de272b6 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -465,9 +465,10 @@ def _process_rna_errors(gene_ids, missing_required_fields, unmatched_samples, ig ] misconfigured = defaultdict(set) for sample_id in sorted(unmatched_samples.intersection(set((misconfigured_samples or {}).keys()))): - misconfigured[misconfigured_samples[sample_id]].add(sample_id) + for error in misconfigured_samples[sample_id]: + misconfigured[error].add(sample_id) unmatched += [ - (samples, f'that are improperly configured in Airtable with {error}') for error, samples in misconfigured.items() + (samples, f'that are improperly configured in Airtable with {error}') for error, samples in sorted(misconfigured.items()) ] for samples, _ in unmatched: unmatched_samples -= samples From 01e98a4d7bff9961166d7e0fedf4d51b1c658a14 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 16:15:34 -0500 Subject: [PATCH 083/252] move invalid search tests --- clickhouse_search/search_tests.py | 49 +++++++++++++++++- seqr/utils/search/search_utils_tests.py | 69 ------------------------- 2 files changed, 47 insertions(+), 71 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e5c894ba08..c0841bd6db 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -801,6 +801,14 @@ def test_invalid_search(self): self._assert_expected_search([], annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive') self.assertEqual(str(cm.exception),'Location must be specified to search for compound heterozygous variants across many families') + with self.assertRaises(InvalidSearchException) as se: + query_variants(self.results_model, sort='prioritized_gene', num_results=2) + self.assertEqual(str(se.exception), 'Phenotype sort is only supported for single-family search.') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], freqs={}) + self.assertEqual(str(cm.exception),'seqr AC frequency of at least 5000 must be specified to search across multiple families') + self.results_model.families.set(Family.objects.filter(guid='F000005_5')) with self.assertRaises(InvalidSearchException) as cm: self._assert_expected_search([], inheritance_mode='recessive') @@ -816,16 +824,53 @@ def test_invalid_search(self): self._assert_expected_search([], inheritance_mode='recessive', annotations_secondary={'frameshift': ['frameshift_variant']}) self.assertEqual(str(cm.exception),'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals') - Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], inheritance_mode='recessive', inheritance_filter={'affected': {'I000007_na20870': 'N'}}) + self.assertEqual(str(cm.exception),'Inheritance based search is disabled in families with no data loaded for affected individuals') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([]) + self.assertEqual(str(cm.exception), 'Inheritance must be specified if custom affected status is set') + + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref'}}) + self.assertEqual(str(cm.exception), 'Invalid custom inheritance') + + self.results_model.families.set(Family.objects.filter(family_id='no_individuals')) + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], inheritance_filter={}) + self.assertEqual(str(cm.exception), 'No search data found for families no_individuals') + self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}) + self._assert_expected_search([]) + self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') + + Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}) self.assertEqual( str(cm.exception), 'The following samples are incorrectly configured and have different affected statuses in different projects: ' 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', ) + self._set_grch37_search() + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': build_specific_genes}) + self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') + + self._set_multi_project_search() + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus=None) + self.assertEqual(str(cm.exception),'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project') + + self._set_single_family_search() + self.set_cache({'total_results': 20000}) + with self.assertRaises(InvalidSearchException) as cm: + query_variants(self.results_model, page=1, num_results=2, load_all=True) + self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') + @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 483bc9b8f7..2ea8be9c00 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -68,75 +68,6 @@ def setUp(self): ]) self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) - @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) - @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) - def test_invalid_search_params(self): - - - self.search_model.search['inheritance']['filter'] = {'affected': {'I000007_na20870': 'N'}} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual( - str(cm.exception), - 'Inheritance based search is disabled in families with no data loaded for affected individuals', - ) - - self.search_model.search['inheritance']['mode'] = None - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Inheritance must be specified if custom affected status is set') - - self.search_model.search['inheritance']['filter'] = {'genotype': {'I000004_hg00731': 'ref_ref'}} - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Invalid custom inheritance') - - self.results_model.families.set(Family.objects.filter(family_id='no_individuals')) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'No search data found for families no_individuals') - - self.results_model.families.set(Family.objects.all()) - self.search_model.search['annotations'] = None - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual( - str(cm.exception), - 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde, Test Reprocessed Project; 38 - Non-Analyst Project', - ) - - self._test_invalid_no_location_search_params() - - self.results_model.families.set(Family.objects.filter(guid='F000014_14')) - self.search_model.search['locus']['rawItems'] = build_specific_genes - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') - - def _test_invalid_no_location_search_params(self): - self.search_model.search['inheritance'] = {} - self.results_model.families.set(Family.objects.filter(id__in=[2, 11])) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') - - self.search_model.search['freqs'] = {} - self.results_model.families.set(self.families) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model) - self.assertEqual(str(cm.exception), 'seqr AC frequency of at least 5000 must be specified to search across multiple families') - - - def test_invalid_search_query_variants(self): - with self.assertRaises(InvalidSearchException) as se: - query_variants(self.results_model, sort='prioritized_gene', num_results=2) - self.assertEqual(str(se.exception), 'Phenotype sort is only supported for single-family search.') - - self.set_cache({'total_results': 20000}) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, page=1, num_results=2, load_all=True) - self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') - def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', From 40237c230958c0a50e2b477e4de2a76bbe2d262f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 16:26:27 -0500 Subject: [PATCH 084/252] deprecate skip_genotype_filter --- seqr/utils/search/search_utils_tests.py | 11 ++--------- seqr/utils/search/utils.py | 4 ++-- seqr/views/apis/variant_search_api.py | 4 +--- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 2ea8be9c00..7086553c4e 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -145,20 +145,13 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v return parsed_variants mock_get_variants.side_effect = _mock_get_variants - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, parsed_variants) - self.assertEqual(total, 5) results_cache = {'all_results': parsed_variants, 'total_results': 5} - self.assert_cached_results(results_cache) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - ) query_variants( - self.results_model, user=self.user, sort='cadd', skip_genotype_filter=True, page=3, num_results=10, + self.results_model, user=self.user, page=3, num_results=10, ) self._test_expected_search_call( - mock_get_variants, results_cache, sort='cadd', page=3, num_results=10, skip_genotype_filter=True, + mock_get_variants, results_cache, page=3, num_results=10, ) query_variants(self.results_model, user=self.user, load_all=True) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 66ceb9f7b6..e031bfca8a 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -169,7 +169,7 @@ def _get_previous_search_results(search_model, sort, page, num_results, load_all return previous_search_results, cached_page, num_results -def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): +def query_variants(search_model, sort=XPOS_SORT_KEY, load_all=False, user=None, page=1, num_results=100): genome_version = _get_search_genome_version(search_model) previous_search_results, cached_page, num_results = _get_previous_search_results( search_model, sort, page, num_results, load_all, genome_version, @@ -179,7 +179,7 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, variants, total_results = _query_variants( search_model, user, previous_search_results, genome_version, sort=sort, page=page, num_results=num_results, - skip_genotype_filter=skip_genotype_filter) + ) if load_all: _validate_export_variant_count(total_results) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 23fcd16bf7..fd1f9bdb38 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -62,10 +62,8 @@ def query_variants_handler(request, search_hash): return create_json_response({'error': str(e)}, status=400, reason=str(e)) _check_results_permission(results_model, request.user) - skip_genotype_filter = bool(_all_project_family_search_genome(search_context)) - variants, total_results = query_variants(results_model, sort=sort, page=page, num_results=per_page, - skip_genotype_filter=skip_genotype_filter, user=request.user) + variants, total_results = query_variants(results_model, sort=sort, page=page, num_results=per_page, user=request.user) response = _process_variants(variants or [], results_model.families.all(), request, genome_version=results_model.variant_search.search.get('no_access_project_genome_version')) From 201938f070d9f3064353aa146a7586c461cbee1d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 16:51:52 -0500 Subject: [PATCH 085/252] test pagination --- clickhouse_search/search_tests.py | 25 +++++++++++++++--------- seqr/utils/search/search_utils_tests.py | 26 ------------------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index c0841bd6db..00c36fac62 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -105,7 +105,7 @@ def setUp(self): super().setUp() self.mock_redis.get.return_value = None - def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', results_model=None, **search_kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): results_model = results_model or self.results_model self.search_model.search.update(search_kwargs or {}) self.search_model.search['qualityFilter'] = quality_filter @@ -113,22 +113,23 @@ def _assert_expected_search(self, expected_results, gene_counts=None, inheritanc if inheritance_filter is not None: self.search_model.search['inheritance']['filter'] = inheritance_filter - variants, total = query_variants(results_model, user=self.user, sort=sort) + variants, total = query_variants(results_model, user=self.user, sort=sort, load_all=load_all, page=page, num_results=num_results) self.assertEqual(total, len(expected_results)) self._assert_expected_variants( variants, expected_results, cache_key=f'search_results__{results_model.guid}__{sort}', sort=sort, format_cached_variants=self._format_cached_variants, cached_variant_fields=cached_variant_fields, + results_page=results_page, ) if gene_counts: gene_counts_json = get_variant_query_gene_counts(results_model, self.user) self.assertDictEqual(gene_counts_json, gene_counts) - def _assert_expected_variants(self, variants, expected_results, cache_key=None, format_cached_variants=None, sort='xpos', **kwargs): + def _assert_expected_variants(self, variants, expected_results, results_page=None, cache_key=None, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) - self.assertListEqual(encoded_variants, expected_results) + self.assertListEqual(encoded_variants, results_page or expected_results) if cache_key: - cached_variants = format_cached_variants(encoded_variants, **kwargs) if format_cached_variants else encoded_variants + cached_variants = format_cached_variants(expected_results, **kwargs) if format_cached_variants else expected_results self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) else: self.mock_redis.get.assert_not_called() @@ -802,7 +803,7 @@ def test_invalid_search(self): self.assertEqual(str(cm.exception),'Location must be specified to search for compound heterozygous variants across many families') with self.assertRaises(InvalidSearchException) as se: - query_variants(self.results_model, sort='prioritized_gene', num_results=2) + self._assert_expected_search([], sort='prioritized_gene', num_results=2) self.assertEqual(str(se.exception), 'Phenotype sort is only supported for single-family search.') with self.assertRaises(InvalidSearchException) as cm: @@ -868,7 +869,7 @@ def test_invalid_search(self): self._set_single_family_search() self.set_cache({'total_results': 20000}) with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, page=1, num_results=2, load_all=True) + self._assert_expected_search([], page=1, num_results=2, load_all=True) self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') @mock.patch('seqr.utils.search.utils.LiftOver') @@ -1541,10 +1542,16 @@ def test_sort(self): sort='pathogenicity_hgmd', ) + sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] + self._assert_expected_search(sorted_variants, sort='gnomad') + + self._assert_expected_search( + sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, + ) self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - sort='gnomad', + sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, ) + self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4) self._reset_search_families() self._assert_expected_search( diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 7086553c4e..4308ed8540 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -147,32 +147,6 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v results_cache = {'all_results': parsed_variants, 'total_results': 5} - query_variants( - self.results_model, user=self.user, page=3, num_results=10, - ) - self._test_expected_search_call( - mock_get_variants, results_cache, page=3, num_results=10, - ) - - query_variants(self.results_model, user=self.user, load_all=True) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=1000, skip_genotype_filter=False, - ) - - with mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 4): - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user, load_all=True) - self.assertEqual(str(cm.exception), 'Unable to export more than 4 variants (5 requested)') - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=4, skip_genotype_filter=False, - ) - - self.set_cache({'total_results': 22}) - query_variants(self.results_model, user=self.user, load_all=True) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=22, skip_genotype_filter=False, - ) - self._test_locus_query_variants(mock_get_variants, results_cache) locus_items = self.search_model.search['locus']['rawItems'] From 203f7731daab678dd6fe483f957d0fcfe4e1d097 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 17:08:12 -0500 Subject: [PATCH 086/252] annotation test move; --- clickhouse_search/search_tests.py | 7 ++ seqr/utils/search/search_utils_tests.py | 117 ------------------------ 2 files changed, 7 insertions(+), 117 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 00c36fac62..9cc1f501d4 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1153,6 +1153,13 @@ def test_annotations_filter(self): ] ) + annotations['missense'] = [] + annotations['in_frame'] = [] + del annotations['splice_ai'] + self._assert_expected_search( + [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], annotations=annotations, + ) + self._set_sv_family_search() self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 4308ed8540..3a4896ea45 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -147,99 +147,6 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v results_cache = {'all_results': parsed_variants, 'total_results': 5} - self._test_locus_query_variants(mock_get_variants, results_cache) - - locus_items = self.search_model.search['locus']['rawItems'] - del self.search_model.search['locus'] - self.search_model.search['exclude'] = {'clinvar': ['benign'], 'rawItems': locus_items} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, exclude_locations=True, exclude={'clinvar': ['benign']}, - ) - - del self.search_model.search['exclude']['rawItems'] - self.search_model.search.update({'pathogenicity': {'clinvar': ['pathogenic', 'likely_pathogenic']}, - 'annotations': {'frameshift': [], 'structural': []}}) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - search_fields=['exclude', 'pathogenicity'], annotations={}, dataset_type='SNV_INDEL', omitted_sample_guids=SV_SAMPLES, - ) - - self.search_model.search = { - 'inheritance': {'mode': 'recessive'}, 'annotations': {'frameshift': ['frameshift_variant']}, - 'freqs': self.search_model.search['freqs'], - } - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type=None, - search_fields=['annotations'], omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'], - ) - - self.search_model.search['annotations_secondary'] = {'structural_consequence': ['LOF']} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SV', - search_fields=['annotations', 'annotations_secondary'] - ) - - screen_annotations = {'SCREEN': ['dELS', 'DNase-only']} - self.search_model.search['annotations_secondary'].update({'SCREEN': ['dELS', 'DNase-only']}) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='ALL', - search_fields=['annotations', 'annotations_secondary'] - ) - - self.search_model.search['annotations_secondary']['structural_consequence'] = [] - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, - omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'], - ) - - self.set_cache(None) - mock_get_variants.reset_mock() - self.search_model.search = { - 'inheritance': {'mode': 'any_affected'}, - 'exclude': {'previousSearch': True, 'previousSearchHash': 'abc1234', 'clinvar': ['benign']}, - 'freqs': self.search_model.search['freqs'], - } - previous_search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': self.search_model.search['freqs']}) - previous_results_model = VariantSearchResults.objects.create(variant_search=previous_search_model, search_hash='abc1234') - previous_results_model.families.set(self.families) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], - exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, - ) - self.assertEqual(mock_get_variants.call_count, 2) - call_args, call_kwargs, _ = self._get_expected_search_call(results_cache, inheritance_mode='de_novo', sort=None, num_results=100) - mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)]) - - # Test when previous results are cached - mock_get_variants.reset_mock() - self.mock_redis.get.side_effect = [ - None, - json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), - ] - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], - exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]}, - exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]}, - ) - self.assertEqual(mock_get_variants.call_count, 1) - del self.search_model.search['exclude'] self.search_model.search['exclude_svs'] = True query_variants(self.results_model, user=self.user) @@ -256,30 +163,6 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v omitted_sample_guids=NON_SNP_INDEL_SAMPLES, dataset_type='SNV_INDEL_only', ) - def _test_locus_query_variants(self, mock_get_variants, results_cache): - self.search_model.search['locus'] = {'rawVariantItems': '1-248367227-TC-T,2-103343353-GAGA-G'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=2, skip_genotype_filter=False, - variant_ids=['1-248367227-TC-T', '2-103343353-GAGA-G'], - parsed_variant_ids=[('1', 248367227, 'TC', 'T'), ('2', 103343353, 'GAGA', 'G')], dataset_type='SNV_INDEL', - omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733'], - ) - - self.search_model.search['locus']['rawItems'] = 'WASH7P' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, single_gene_search=True, omitted_sample_guids=MITO_SAMPLES, - ) - - self.search_model.search['locus']['rawItems'] = 'WASH7P, chr2:1234-5678, chr7:100-10100%10, ENSG00000186092' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, omitted_sample_guids=MITO_SAMPLES, - ) - def test_cached_query_variants(self): Project.objects.filter(id=1).update(genome_version='38') From d0ff2ce5fbd09448bcf0e471e69a160631d5420e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 5 Mar 2026 17:11:00 -0500 Subject: [PATCH 087/252] test exclude sv behavior --- clickhouse_search/search_tests.py | 4 ++-- seqr/utils/search/search_utils_tests.py | 27 ------------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9cc1f501d4..5aa64a5aba 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -181,12 +181,12 @@ def test_single_family_search(self): } self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4], gene_counts=variant_gene_counts, locus={'rawItems': '1:1-100000000'}, - **ALL_SNV_INDEL_PASS_FILTERS, + exclude_svs=True, ) self._assert_expected_search( [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=MITO_GENE_COUNTS, locus={'rawItems': 'M:1-100000000'}, - annotations=None, pathogenicity=None, + exclude_svs=False, ) self._assert_expected_search( diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 3a4896ea45..cccfaffeae 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -136,33 +136,6 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid self.assertSetEqual(set(mock_get_variants.call_args.args[0]), set(searched_samples)) self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_query_variants(self, mock_get_variants): - parsed_variants = [{**v, 'key': (i+1) * 1000 } for i, v in enumerate(PARSED_VARIANTS)] - def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] =parsed_variants - previous_search_results['total_results'] = 5 - return parsed_variants - mock_get_variants.side_effect = _mock_get_variants - - results_cache = {'all_results': parsed_variants, 'total_results': 5} - - del self.search_model.search['exclude'] - self.search_model.search['exclude_svs'] = True - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', omitted_sample_guids=SV_SAMPLES, dataset_type='SNV_INDEL', - ) - - self.search_model.search['locus'] = {'rawItems': 'WASH7P'} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - inheritance_mode='any_affected', has_gene_search=True, single_gene_search=True, - omitted_sample_guids=NON_SNP_INDEL_SAMPLES, dataset_type='SNV_INDEL_only', - ) - def test_cached_query_variants(self): Project.objects.filter(id=1).update(genome_version='38') From 2f269bd24281a32da94bce5cc399dbbb2c35d046 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 6 Mar 2026 14:38:31 -0500 Subject: [PATCH 088/252] move cache tests --- clickhouse_search/search_tests.py | 46 ++++++++++++++++++++++++- seqr/utils/search/search_utils_tests.py | 41 ---------------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 5aa64a5aba..e7834bc14a 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -126,7 +126,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.assertDictEqual(gene_counts_json, gene_counts) def _assert_expected_variants(self, variants, expected_results, results_page=None, cache_key=None, format_cached_variants=None, sort='xpos', **kwargs): - encoded_variants = json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) + encoded_variants = self._encode_variants(variants) self.assertListEqual(encoded_variants, results_page or expected_results) if cache_key: cached_variants = format_cached_variants(expected_results, **kwargs) if format_cached_variants else expected_results @@ -135,6 +135,10 @@ def _assert_expected_variants(self, variants, expected_results, results_page=Non self.mock_redis.get.assert_not_called() self.mock_redis.set.assert_not_called() + @staticmethod + def _encode_variants(variants): + return json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) + def _format_cached_variants(self, variants, cached_variant_fields=None): cached_variants = [ self._get_cached_variant(variant, (cached_variant_fields[i] if cached_variant_fields else None)) @@ -1865,6 +1869,46 @@ def test_gene_variant_lookup(self): 'searchedVariants': [], }) + def test_cached_query_variants(self): + cache_key_prefix = f'search_results__{self.results_model.guid}' + cached_variants = [VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1, VARIANT2] + cache_result = self._format_cached_variants(cached_variants) + self.set_cache(cache_result) + + variants, total = query_variants(self.results_model, user=self.user) + self.assertEqual(total, 5) + self.assertListEqual(self._encode_variants(variants), cached_variants) + self.mock_redis.get.assert_called_with(f'{cache_key_prefix}__xpos') + self.mock_redis.set.assert_not_called() + + variants, _ = query_variants(self.results_model, user=self.user, num_results=2, page=2) + self.assertListEqual(variants, [GCNV_VARIANT1, MITO_VARIANT1]) + + gene_counts = get_variant_query_gene_counts(self.results_model, self.user) + self.assertDictEqual(gene_counts, { + 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, + }) + + self.mock_redis.get.side_effect = [None, json.dumps(cache_result)] + self.mock_redis.keys.return_value = [f'{cache_key_prefix}__xpos', f'{cache_key_prefix}__gnomad'] + + variants, total = query_variants(self.results_model, user=self.user, sort='cadd') + self.assertEqual(total, 5) + sorted_variants = [VARIANT2, VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1] + self.assertListEqual(self._encode_variants(variants), sorted_variants) + self.mock_redis.get.assert_has_calls([ + mock.call(f'{cache_key_prefix}__cadd'), + mock.call(f'{cache_key_prefix}__xpos'), + ]) + self.mock_redis.keys.assert_called_with(pattern=f'{cache_key_prefix}__*') + self.assert_cached_results( + {'all_results': [format_cached_variant(v) for v in sorted_variants], 'total_results': 5}, + sort='cadd', + ) + class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): databases = '__all__' fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index cccfaffeae..7b5642e0e9 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -136,38 +136,7 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid self.assertSetEqual(set(mock_get_variants.call_args.args[0]), set(searched_samples)) self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - def test_cached_query_variants(self): - Project.objects.filter(id=1).update(genome_version='38') - self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS}) - variants, total = query_variants(self.results_model, user=self.user) - self._assert_expected_cached_variants(variants, 4) - self.assertEqual(total, 4) - - variants, total = query_variants(self.results_model, user=self.user, num_results=2) - self._assert_expected_cached_variants(variants, 2) - self.assertEqual(total, 4) - - cache_key_prefix = f'search_results__{self.results_model.guid}' - self.mock_redis.get.side_effect = [None, json.dumps({'total_results': 4, 'all_results': self.CACHED_VARIANTS})] - self.mock_redis.keys.return_value = [f'{cache_key_prefix}__xpos', f'{cache_key_prefix}__gnomad'] - - variants, total = query_variants(self.results_model, user=self.user, sort='cadd') - self.assertEqual(total, 4) - self.assertListEqual( - json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), - [VARIANT4, VARIANT3, VARIANT2, VARIANT1] - ) - self.mock_redis.get.assert_has_calls([ - mock.call(f'{cache_key_prefix}__cadd'), - mock.call(f'{cache_key_prefix}__xpos'), - ]) - self.mock_redis.keys.assert_called_with(pattern=f'{cache_key_prefix}__*') - self.assert_cached_results( - {'all_results': [format_cached_variant(v) for v in [VARIANT4, VARIANT3, VARIANT2, VARIANT1]], - 'total_results': 4}, - sort='cadd', - ) def _assert_expected_cached_variants(self, variants, num_results): self.assertListEqual( @@ -192,13 +161,3 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v self._test_expected_search_call( mock_get_variants, results_cache, **kwargs, ) - - def test_cached_get_variant_query_gene_counts(self): - self.set_cache({'all_results': self.CACHED_VARIANTS + [SV_VARIANT1], 'total_results': 5}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, - }) From 45563200923f18e4f8ff7bb2cb7da91ce8bde966 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 6 Mar 2026 14:45:15 -0500 Subject: [PATCH 089/252] remove old utils tests --- clickhouse_search/search_tests.py | 28 +++- seqr/utils/search/search_utils_tests.py | 163 ------------------------ 2 files changed, 24 insertions(+), 167 deletions(-) delete mode 100644 seqr/utils/search/search_utils_tests.py diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e7834bc14a..8b859bebc9 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,3 +1,5 @@ +from datetime import timedelta +from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist from django.core.management import call_command from django.db import connections @@ -35,7 +37,6 @@ format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.utils.search.search_utils_tests import SearchTestHelper from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets @@ -96,14 +97,33 @@ def setUpTestData(cls): super().setUpTestData() -class ClickhouseSearchTests(SearchTestHelper, ClickhouseSearchTestCase): +class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): - super().set_up() - super().setUp() + patcher = mock.patch('seqr.utils.redis_utils.redis.StrictRedis') + self.mock_redis = patcher.start().return_value self.mock_redis.get.return_value = None + self.addCleanup(patcher.stop) + + self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) + self.user = User.objects.get(username='test_user') + + self.search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': {'callset': {'ac': 1000}}}) + self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) + self.results_model.families.set(self.families) + + super().setUp() + + def set_cache(self, cached): + self.mock_redis.get.return_value = json.dumps(cached) + + def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): + cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' + self.mock_redis.set.assert_called_with(cache_key, mock.ANY) + self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) + self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): results_model = results_model or self.results_model diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py deleted file mode 100644 index 7b5642e0e9..0000000000 --- a/seqr/utils/search/search_utils_tests.py +++ /dev/null @@ -1,163 +0,0 @@ -from datetime import timedelta -from django.contrib.auth.models import User -from django.test import TestCase -import json -import mock - -from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant, \ - GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, SV_VARIANT1 -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults -from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.utils.search.utils import get_single_variant, get_variant_query_gene_counts, \ - query_variants, variant_lookup, InvalidSearchException -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, \ - PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS - - -SV_SAMPLES = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] -MITO_SAMPLES = ['S000149_hg00733'] -NON_SNP_INDEL_SAMPLES = SV_SAMPLES + MITO_SAMPLES -FAMILY_3_SAMPLE = 'S000135_na20870' - -GENE_COUNTS = { - **GENE_COUNTS, - 'ENSG00000177000': {**GENE_COUNTS['ENSG00000177000'], 'total': 3}, - 'ENSG00000277258': {**GENE_COUNTS['ENSG00000277258'], 'total': 2}, -} - -class SearchTestHelper(object): - - def set_up(self): - patcher = mock.patch('seqr.utils.redis_utils.redis.StrictRedis') - self.mock_redis = patcher.start().return_value - self.addCleanup(patcher.stop) - - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - self.user = User.objects.get(username='test_user') - - self.search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': {'callset': {'ac': 1000}}}) - self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) - self.results_model.families.set(self.families) - - def set_cache(self, cached): - self.mock_redis.get.return_value = json.dumps(cached) - - def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): - cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' - self.mock_redis.set.assert_called_with(cache_key, mock.ANY) - self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) - self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - - -class SearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchTestHelper): - databases = '__all__' - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] - - PARSED_CACHED_VARIANTS = [VARIANT1, VARIANT2, VARIANT3, VARIANT4] - CACHED_VARIANTS = [format_cached_variant(v) for v in PARSED_CACHED_VARIANTS] - - def setUp(self): - self.set_up() - - self.non_affected_search_samples = Sample.objects.filter(guid__in=[ - 'S000137_na20874', - ]) - self.affected_search_samples = Sample.objects.filter(guid__in=[ - 'S000132_hg00731', 'S000133_hg00732', 'S000134_hg00733', FAMILY_3_SAMPLE, - 'S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733', 'S000149_hg00733', - ]) - self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) - - - def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, - variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', - dataset_type=None, secondary_dataset_type=None, exclude_keys=None, exclude_key_pairs=None, - exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): - genes = intervals = None - if has_gene_search: - genes = {'ENSG00000186092': mock.ANY, 'ENSG00000227232': mock.ANY} - intervals = [{'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}] - if single_gene_search: - genes = {'ENSG00000227232': mock.ANY} - intervals = intervals[2:3] - - expected_search = { - 'inheritance_mode': inheritance_mode, - 'inheritance_filter': {}, - 'skipped_samples': mock.ANY, - 'dataset_type': dataset_type, - 'secondary_dataset_type': secondary_dataset_type, - 'exclude_locations': exclude_locations, - 'genes': genes, - 'intervals': intervals, - 'freqs': {'callset': {'ac': 1000}}, - } - if not genes: - expected_search.update({ - 'parsed_variant_ids': parsed_variant_ids, - 'variant_ids': variant_ids, - }) - expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) - if exclude: - expected_search['exclude'] = exclude - if annotations is not None: - expected_search['annotations'] = annotations - if annotations_secondary is not None: - expected_search['annotations_secondary'] = annotations_secondary - if exclude_keys is not None: - expected_search['exclude_keys'] = exclude_keys - if exclude_key_pairs is not None: - expected_search['exclude_key_pairs'] = exclude_key_pairs - - return (mock.ANY, expected_search, self.user, results_cache, '37'), kwargs, genes - - def _test_expected_search_call(self, mock_get_variants, *args, has_gene_search=False, exclude_locations=False, omitted_sample_guids=None, **kwargs): - call_args, call_kwargs, genes = self._get_expected_search_call(*args, has_gene_search=has_gene_search, exclude_locations=exclude_locations, **kwargs) - mock_get_variants.assert_called_with(*call_args, **call_kwargs) - self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - - if genes: - parsed_genes = mock_get_variants.call_args.args[1]['genes'] - for gene in parsed_genes.values(): - self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) - self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') - if len(genes) > 1: - self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') - - - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): - searched_samples = self.affected_search_samples - non_affected_search_samples = self.non_affected_search_samples - if omitted_sample_guids: - searched_samples = searched_samples.exclude(guid__in=omitted_sample_guids) - non_affected_search_samples = non_affected_search_samples.exclude(guid__in=omitted_sample_guids) - if has_gene_search: - searched_samples = searched_samples.exclude(guid__in=MITO_SAMPLES) - self.assertSetEqual(set(mock_get_variants.call_args.args[0]), set(searched_samples)) - self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - - - - def _assert_expected_cached_variants(self, variants, num_results): - self.assertListEqual( - json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)), - self.PARSED_CACHED_VARIANTS[:num_results], - ) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_get_variant_query_gene_counts(self, mock_get_variants): - gene_agg_all_results = self.CACHED_VARIANTS + [format_cached_variant(PROJECT_2_VARIANT2)] - def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = gene_agg_all_results - return None - - mock_get_variants.side_effect = _mock_get_variants - - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, GENE_COUNTS) - results_cache = {'all_results': gene_agg_all_results} - self.assert_cached_results(results_cache) - kwargs = dict(sort=None, num_results=100) - self._test_expected_search_call( - mock_get_variants, results_cache, **kwargs, - ) From 923be9c4e5c7b2b20ef3b29c982886b24640273f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 6 Mar 2026 14:55:04 -0500 Subject: [PATCH 090/252] pr feedback --- seqr/views/apis/data_manager_api.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index b5074f3cbb..3b13b3e748 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -86,14 +86,20 @@ def update_rna_seq(request): if sample.get(sample_id_field) ] error_messages = [] - if len(sample.get(TISSUE_FIELD, [])) != 1: - error_messages.append('no tissue specified' if not sample.get(TISSUE_FIELD) else 'multiple tissues specified') - elif sample[TISSUE_FIELD][0] not in TISSUE_TYPE_MAP: + tissues = sample.get(TISSUE_FIELD, []) + if not tissues: + error_messages.append('no tissue specified') + elif len(tissues) > 1: + error_messages.append('multiple tissues specified') + elif tissues[0] not in TISSUE_TYPE_MAP: error_messages.append('invalid tissue specified') - if len(sample['pdos']) != 1: + + pdos = sample.get('pdos', []) + if len(pdos) > 1: error_messages.append('multiple conflicting PDOs') - elif sample['pdos'][0]['project_guid'] is None: + elif not pdos or pdos[0]['project_guid'] is None: error_messages.append('no project specified') + if error_messages: for sample_id in sample_ids: misconfigured_samples[sample_id].extend(error_messages) From b09f3b76cb81b58c423a4da56e301b2b37517f69 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 6 Mar 2026 15:07:58 -0500 Subject: [PATCH 091/252] fix tests --- clickhouse_search/search_tests.py | 2 +- seqr/views/apis/variant_search_api_tests.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8b859bebc9..ab4bd6df3c 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -145,7 +145,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun gene_counts_json = get_variant_query_gene_counts(results_model, self.user) self.assertDictEqual(gene_counts_json, gene_counts) - def _assert_expected_variants(self, variants, expected_results, results_page=None, cache_key=None, format_cached_variants=None, sort='xpos', **kwargs): + def _assert_expected_variants(self, variants, expected_results, cache_key=None, results_page=None, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = self._encode_variants(variants) self.assertListEqual(encoded_variants, results_page or expected_results) if cache_key: diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index a1d75d74b1..f5beb4e8e7 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -433,7 +433,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self._assert_expected_results_context(response_json) results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) + mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, user=self.collaborator_user) mock_error_logger.assert_not_called() # include project context info @@ -462,13 +462,13 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro # Test pagination response = self.client.get('{}?page=3'.format(url)) self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=3, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) + mock_get_variants.assert_called_with(results_model, sort='xpos', page=3, num_results=100, user=self.collaborator_user) mock_error_logger.assert_not_called() # Test sort response = self.client.get('{}?sort=pathogenicity'.format(url)) self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='pathogenicity', page=1, num_results=100, skip_genotype_filter=False, user=self.collaborator_user) + mock_get_variants.assert_called_with(results_model, sort='pathogenicity', page=1, num_results=100, user=self.collaborator_user) mock_error_logger.assert_not_called() # Test export @@ -599,7 +599,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertDictEqual(response_json, expected_search_results) self._assert_expected_results_context(response_json) - mock_get_variants.assert_called_with(results_model, sort='pathogenicity_hgmd', page=1, num_results=100, skip_genotype_filter=False, user=self.analyst_user) + mock_get_variants.assert_called_with(results_model, sort='pathogenicity_hgmd', page=1, num_results=100, user=self.analyst_user) mock_error_logger.assert_not_called() # Test no results @@ -652,7 +652,7 @@ def _get_variants(results_model, **kwargs): }} self.assertDictEqual(response_json, empty_search_response) results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=True, user=self.no_access_user) + mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, user=self.no_access_user) VariantSearchResults.objects.filter(search_hash=SEARCH_HASH).delete() self.login_data_manager_user() @@ -677,7 +677,7 @@ def _get_variants(results_model, **kwargs): result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) mock_get_variants.assert_called_with(result_model, sort='xpos', page=1, num_results=100, - skip_genotype_filter=True, user=self.collaborator_user) + user=self.collaborator_user) result_model.delete() body['includeNoAccessProjects'] = True @@ -692,7 +692,7 @@ def _get_variants(results_model, **kwargs): result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) mock_get_variants.assert_called_with( - result_model, sort='xpos', page=1, num_results=100, skip_genotype_filter=True, user=self.collaborator_user, + result_model, sort='xpos', page=1, num_results=100, user=self.collaborator_user, ) result_model.delete() @@ -759,7 +759,7 @@ def test_query_all_project_families_variants(self, mock_get_variants): mock_get_variants.assert_called_with( VariantSearchResults.objects.get(search_hash=SEARCH_HASH), sort='xpos', page=1, num_results=100, - skip_genotype_filter=False, user=self.collaborator_user) + user=self.collaborator_user) # Test export disabled in demo projects export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) From b56de27a15e94080d7dc97bbcc3b6223fea463f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 6 Mar 2026 16:17:42 -0500 Subject: [PATCH 092/252] first pass --- clickhouse_search/search_tests.py | 122 ++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 38 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ab4bd6df3c..6d65cc1402 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -6,6 +6,7 @@ from django.urls.base import reverse import json import mock +import random import responses from clickhouse_search.models.gt_stats_models import ProjectGtStatsSnvIndel, \ @@ -114,6 +115,9 @@ def setUp(self): self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) self.results_model.families.set(self.families) + url = reverse(query_variants_handler, args=['abc123']) + self.check_require_login(url) + super().setUp() def set_cache(self, cached): @@ -125,7 +129,57 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, request_body=None, format_response_body=None, format_cache_key=None, **search_kwargs): + search_hash = str(random.randint(1000, 10000)) + url = reverse(query_variants_handler, args=[search_hash]) + + search_body = { + 'inheritance': {'mode': inheritance_mode}, + 'freqs': {'callset': {'ac': 1000}}, + 'qualityFilter': quality_filter, + **(search_kwargs or {}), + } + if inheritance_filter is not None: + search_body['inheritance']['filter'] = inheritance_filter + + response = self.client.post(url, content_type='application/json', data=json.dumps({ + **request_body, 'search': search_body, + })) + self.assertEqual(response.status_code, 200) + expected_response = { + 'searchedVariants': results_page or expected_results, + 'genesById': mock.ANY, + 'search': { + 'search': format_response_body(search_body) if format_response_body else search_body, + 'projectFamilies': [], + 'totalResults': len(expected_results), + }, + 'totalSampleCounts': { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + }, + 'locusListsByGuid': {}, + 'mmeSubmissionsByGuid': {}, + 'omimIntervals': {}, + 'phenotypeGeneScores': {}, + 'rnaSeqData': {}, + 'savedVariantsByGuid': {}, + 'variantFunctionalDataByGuid': {}, + 'variantNotesByGuid': {}, + 'variantTagsByGuid': {}, + } + self.assertDictEqual(response.json(), expected_response) + + cache_key = format_cache_key() if format_cache_key else f'search_results__{search_hash}__{sort}' + if cache_key: + cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) + self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) + else: + self.mock_redis.get.assert_not_called() + self.mock_redis.set.assert_not_called() + + def _assert_expected_search_old(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): results_model = results_model or self.results_model self.search_model.search.update(search_kwargs or {}) self.search_model.search['qualityFilter'] = quality_filter @@ -1753,8 +1807,36 @@ def test_multi_data_type_comp_het_sort(self): def test_gene_variant_lookup(self): url = reverse(query_variants_handler, args=['abc123']) - self.check_require_login(url) + request_body = { + 'allGenomeProjectFamilies': '38', + 'includeNoAccessProjects': True, + } + response = self.client.post(url, content_type='application/json', data=json.dumps(request_body)) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), { + 'error': 'Including external projects is only available when searching for a single gene', + }) + + format_response_body = lambda search_body: {**search_body, 'no_access_project_genome_version': '38'} + variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} + del variant4['familyGuids'] + del variant4['genotypes'] + self._assert_expected_search( + [variant4], request_body=request_body, format_response_body=format_response_body, + annotations={ + 'missense': ['missense_variant'], + 'other': ['non_coding_transcript_exon_variant'], + }, + freqs={ + 'callset': {'ac': 3000}, + 'gnomad_genomes': {'af': 0.003}, + 'gnomad_exomes': {'af': 0.003}, + }, + locus={'rawItems': 'ENSG00000097046'}, + ) + + # TODO body = { 'allGenomeProjectFamilies': '38', 'includeNoAccessProjects': True, @@ -1770,43 +1852,7 @@ def test_gene_variant_lookup(self): }, } } - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'error': 'Including external projects is only available when searching for a single gene', - }) - body['search']['locus'] = {'rawItems': 'ENSG00000097046'} - response = self.client.post(url+'1', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} - del variant4['familyGuids'] - del variant4['genotypes'] - expected_response = { - 'searchedVariants': [variant4], - 'genesById': {'ENSG00000097046': mock.ANY}, - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': [], - 'totalResults': 1, - }, - 'totalSampleCounts': { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - }, - 'locusListsByGuid': {}, - 'mmeSubmissionsByGuid': {}, - 'omimIntervals': {}, - 'phenotypeGeneScores': {}, - 'rnaSeqData': {}, - 'savedVariantsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'variantNotesByGuid': {}, - 'variantTagsByGuid': {}, - } - self.assertDictEqual(response.json(), expected_response) - body['search']['freqs'] = {'callset': body['search']['freqs']['callset']} response = self.client.post(url+'2', content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) From c71d218a2ab1a2fe5cd34b0b8005882d3a55b79b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 12:31:01 -0400 Subject: [PATCH 093/252] fix helpers --- clickhouse_search/search_tests.py | 70 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6d65cc1402..9040cabe2b 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -108,6 +108,11 @@ def setUp(self): self.mock_redis.get.return_value = None self.addCleanup(patcher.stop) + patcher = mock.patch('seqr.models.VariantSearchResults._compute_guid') + self.mock_results_guid = patcher.start() + self.mock_results_guid.return_value = random.randint(1000, 10000) + self.addCleanup(patcher.stop) + self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) self.user = User.objects.get(username='test_user') @@ -129,8 +134,10 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, request_body=None, format_response_body=None, format_cache_key=None, **search_kwargs): - search_hash = str(random.randint(1000, 10000)) + def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, request_body=None, **search_kwargs): + search_hash = random.randint(1000, 10000) + result_guid = f'VRS{search_hash:07d}' + self.mock_results_guid.return_value = result_guid url = reverse(query_variants_handler, args=[search_hash]) search_body = { @@ -142,9 +149,12 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun if inheritance_filter is not None: search_body['inheritance']['filter'] = inheritance_filter - response = self.client.post(url, content_type='application/json', data=json.dumps({ + return self.client.post(url, content_type='application/json', data=json.dumps({ **request_body, 'search': search_body, - })) + })), result_guid, search_body + + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, format_response_body=None, format_cache_key=None, **kwargs): + response, result_guid, search_body = self._execute_search(**kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, @@ -171,7 +181,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun } self.assertDictEqual(response.json(), expected_response) - cache_key = format_cache_key() if format_cache_key else f'search_results__{search_hash}__{sort}' + cache_key = format_cache_key() if format_cache_key else f'search_results__{result_guid}__{sort}' if cache_key: cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) @@ -179,6 +189,11 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.mock_redis.get.assert_not_called() self.mock_redis.set.assert_not_called() + def _assert_expected_search_error(self, error, **kwargs): + response, _, _ = self._execute_search(**kwargs) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': error}) + def _assert_expected_search_old(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): results_model = results_model or self.results_model self.search_model.search.update(search_kwargs or {}) @@ -1812,46 +1827,29 @@ def test_gene_variant_lookup(self): 'allGenomeProjectFamilies': '38', 'includeNoAccessProjects': True, } - response = self.client.post(url, content_type='application/json', data=json.dumps(request_body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'error': 'Including external projects is only available when searching for a single gene', - }) + self._assert_expected_search_error( + 'Including external projects is only available when searching for a single gene', request_body=request_body, + ) + annotations = { + 'missense': ['missense_variant'], + 'other': ['non_coding_transcript_exon_variant'], + } + freqs = { + 'callset': {'ac': 3000}, + 'gnomad_genomes': {'af': 0.003}, + 'gnomad_exomes': {'af': 0.003}, + } format_response_body = lambda search_body: {**search_body, 'no_access_project_genome_version': '38'} variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} del variant4['familyGuids'] del variant4['genotypes'] self._assert_expected_search( [variant4], request_body=request_body, format_response_body=format_response_body, - annotations={ - 'missense': ['missense_variant'], - 'other': ['non_coding_transcript_exon_variant'], - }, - freqs={ - 'callset': {'ac': 3000}, - 'gnomad_genomes': {'af': 0.003}, - 'gnomad_exomes': {'af': 0.003}, - }, - locus={'rawItems': 'ENSG00000097046'}, + cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], + annotations=annotations, freqs=freqs, locus={'rawItems': 'ENSG00000097046'}, ) - # TODO - body = { - 'allGenomeProjectFamilies': '38', - 'includeNoAccessProjects': True, - 'search': { - 'annotations': { - 'missense': ['missense_variant'], - 'other': ['non_coding_transcript_exon_variant'], - }, - 'freqs': { - 'callset': {'ac': 3000}, - 'gnomad_genomes': {'af': 0.003}, - 'gnomad_exomes': {'af': 0.003}, - }, - } - } body['search']['locus'] = {'rawItems': 'ENSG00000097046'} body['search']['freqs'] = {'callset': body['search']['freqs']['callset']} response = self.client.post(url+'2', content_type='application/json', data=json.dumps(body)) From f847aed5e83fef8d2e11c0dbb6525cf18e9e8a9a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 12:41:48 -0400 Subject: [PATCH 094/252] test empty --- clickhouse_search/search_tests.py | 80 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9040cabe2b..a9df339ecc 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -158,27 +158,31 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, - 'genesById': mock.ANY, 'search': { 'search': format_response_body(search_body) if format_response_body else search_body, 'projectFamilies': [], 'totalResults': len(expected_results), }, - 'totalSampleCounts': { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - }, - 'locusListsByGuid': {}, - 'mmeSubmissionsByGuid': {}, - 'omimIntervals': {}, - 'phenotypeGeneScores': {}, - 'rnaSeqData': {}, - 'savedVariantsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'variantNotesByGuid': {}, - 'variantTagsByGuid': {}, + } + if expected_results: + expected_response.update({ + 'genesById': mock.ANY, + 'locusListsByGuid': {}, + 'mmeSubmissionsByGuid': {}, + 'omimIntervals': {}, + 'phenotypeGeneScores': {}, + 'rnaSeqData': {}, + 'savedVariantsByGuid': {}, + 'variantFunctionalDataByGuid': {}, + 'variantNotesByGuid': {}, + 'variantTagsByGuid': {}, + 'totalSampleCounts': { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + }, + }) self.assertDictEqual(response.json(), expected_response) cache_key = format_cache_key() if format_cache_key else f'search_results__{result_guid}__{sort}' @@ -1840,6 +1844,7 @@ def test_gene_variant_lookup(self): 'gnomad_genomes': {'af': 0.003}, 'gnomad_exomes': {'af': 0.003}, } + locus = {'rawItems': 'ENSG00000097046'} format_response_body = lambda search_body: {**search_body, 'no_access_project_genome_version': '38'} variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} del variant4['familyGuids'] @@ -1847,40 +1852,31 @@ def test_gene_variant_lookup(self): self._assert_expected_search( [variant4], request_body=request_body, format_response_body=format_response_body, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], - annotations=annotations, freqs=freqs, locus={'rawItems': 'ENSG00000097046'}, + annotations=annotations, freqs=freqs, locus=locus, ) - body['search']['locus'] = {'rawItems': 'ENSG00000097046'} - body['search']['freqs'] = {'callset': body['search']['freqs']['callset']} - response = self.client.post(url+'2', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) + freqs = {'callset': freqs['callset']} variant3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611', 'numFamilies': 4} del variant3['familyGuids'] del variant3['genotypes'] - expected_response['searchedVariants'].insert(0, variant3) - expected_response['genesById']['ENSG00000177000'] = mock.ANY - expected_response['search']['search'].update(body['search']) - expected_response['search']['totalResults'] = 2 - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [variant3, variant4], request_body=request_body, format_response_body=format_response_body, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, + ], + annotations=annotations, freqs=freqs, locus=locus, + ) - body['search']['inheritance'] = {'mode': 'recessive'} - response = self.client.post(url + '3', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), { - 'error': 'Compound heterozygous search is not supported when including external projects', - }) + self._assert_expected_search_error( + 'Compound heterozygous search is not supported when including external projects', + request_body=request_body, locus=locus, annotations=annotations, inheritance_mode='recessive', + ) - body['search']['inheritance']['mode'] = 'homozygous_recessive' - response = self.client.post(url + '4', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': [], - 'totalResults': 0, - }, - 'searchedVariants': [], - }) + self._assert_expected_search( + [], request_body=request_body, format_response_body=format_response_body, + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='homozygous_recessive', + ) body['search']['inheritance']['mode'] = 'de_novo' response = self.client.post(url + '5', content_type='application/json', data=json.dumps(body)) From a21e66aaf401de86ed6b4c158674e487e7c15527 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 13:59:02 -0400 Subject: [PATCH 095/252] gene lookup test update --- clickhouse_search/search_tests.py | 96 +++++++++++++++---------------- 1 file changed, 46 insertions(+), 50 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a9df339ecc..3f299d41a0 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -153,14 +153,14 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit **request_body, 'search': search_body, })), result_guid, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, format_response_body=None, format_cache_key=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, response_search=None, format_cache_key=None, project_families=None, **kwargs): response, result_guid, search_body = self._execute_search(**kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, 'search': { - 'search': format_response_body(search_body) if format_response_body else search_body, - 'projectFamilies': [], + 'search': {**search_body, **(response_search or {})}, + 'projectFamilies': project_families or [], 'totalResults': len(expected_results), }, @@ -168,10 +168,10 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun if expected_results: expected_response.update({ 'genesById': mock.ANY, - 'locusListsByGuid': {}, + 'locusListsByGuid': mock.ANY, 'mmeSubmissionsByGuid': {}, 'omimIntervals': {}, - 'phenotypeGeneScores': {}, + 'phenotypeGeneScores': mock.ANY, 'rnaSeqData': {}, 'savedVariantsByGuid': {}, 'variantFunctionalDataByGuid': {}, @@ -193,6 +193,11 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.mock_redis.get.assert_not_called() self.mock_redis.set.assert_not_called() + if gene_counts: + raise NotImplementedError + + return response.json() + def _assert_expected_search_error(self, error, **kwargs): response, _, _ = self._execute_search(**kwargs) self.assertEqual(response.status_code, 400) @@ -1825,8 +1830,6 @@ def test_multi_data_type_comp_het_sort(self): ) def test_gene_variant_lookup(self): - url = reverse(query_variants_handler, args=['abc123']) - request_body = { 'allGenomeProjectFamilies': '38', 'includeNoAccessProjects': True, @@ -1845,28 +1848,30 @@ def test_gene_variant_lookup(self): 'gnomad_exomes': {'af': 0.003}, } locus = {'rawItems': 'ENSG00000097046'} - format_response_body = lambda search_body: {**search_body, 'no_access_project_genome_version': '38'} + response_search = {'no_access_project_genome_version': '38'} variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} del variant4['familyGuids'] del variant4['genotypes'] - self._assert_expected_search( - [variant4], request_body=request_body, format_response_body=format_response_body, + response_json = self._assert_expected_search( + [variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], annotations=annotations, freqs=freqs, locus=locus, ) + self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY}) freqs = {'callset': freqs['callset']} variant3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611', 'numFamilies': 4} del variant3['familyGuids'] del variant3['genotypes'] - self._assert_expected_search( - [variant3, variant4], request_body=request_body, format_response_body=format_response_body, + response_json = self._assert_expected_search( + [variant3, variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], annotations=annotations, freqs=freqs, locus=locus, ) + self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY, 'ENSG00000177000': mock.ANY}) self._assert_expected_search_error( 'Compound heterozygous search is not supported when including external projects', @@ -1874,60 +1879,51 @@ def test_gene_variant_lookup(self): ) self._assert_expected_search( - [], request_body=request_body, format_response_body=format_response_body, + [], request_body=request_body, response_search=response_search, annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='homozygous_recessive', ) - body['search']['inheritance']['mode'] = 'de_novo' - response = self.client.post(url + '5', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - expected_response['search']['search'].update(body['search']) variant3['numFamilies'] = 1 variant4['numFamilies'] = 1 - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [variant3, variant4], request_body=request_body, response_search=response_search, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, + ], + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) self.login_collaborator() - response = self.client.post(url + '6', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': mock.ANY}] - expected_response['search']['projectFamilies'] = project_families - expected_response.update({ - 'searchedVariants': [ + response_json = self._assert_expected_search([ {**FAMILY_3_VARIANT, 'selectedMainTranscriptId': 'ENST00000497611'}, {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997'}, + ], request_body=request_body, response_search=response_search, project_families=project_families, + cached_variant_fields=[ + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, + {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY}, - 'phenotypeGeneScores': {'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}, - }) - self.assertDictEqual(response.json(), expected_response) + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) + self.assertEqual(response_json['locusListsByGuid'], {'LL00049_pid_genes_autosomal_do': mock.ANY}) + self.assertEqual(response_json['phenotypeGeneScores'], {'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}) - body['search']['locus']['rawItems'] = 'ENSG00000171621' - response = self.client.post(url + '7', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - expected_response['search']['search'].update(body['search']) - expected_response['search']['totalResults'] = 1 + locus['rawItems'] = 'ENSG00000171621' other_project_variant = {**PROJECT_4_COMP_HET_VARIANT, 'numFamilies': 1} del other_project_variant['familyGuids'] del other_project_variant['genotypes'] - expected_response.update({ - 'searchedVariants': [other_project_variant], - 'genesById': {'ENSG00000171621': mock.ANY}, - 'locusListsByGuid': {}, - 'phenotypeGeneScores': {}, - }) - self.assertDictEqual(response.json(), expected_response) + self._assert_expected_search( + [other_project_variant], request_body=request_body, response_search=response_search, + project_families=project_families, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[22][0]}], + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) - body['search']['locus']['rawItems'] = 'ENSG00000229905' - response = self.client.post(url+'8', content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { - 'search': { - 'search': {**body['search'], 'no_access_project_genome_version': '38'}, - 'projectFamilies': project_families, - 'totalResults': 0, - }, - 'searchedVariants': [], - }) + locus['rawItems'] = 'ENSG00000229905' + self._assert_expected_search( + [], request_body=request_body, response_search=response_search, + project_families=project_families, annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + ) def test_cached_query_variants(self): cache_key_prefix = f'search_results__{self.results_model.guid}' From 5662f5caddfc6a6d644f8bc84453876d8f91e447 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 14:23:17 -0400 Subject: [PATCH 096/252] adjust searches for missed discoveries --- .../commands/tag_seqr_prioritized_variants.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index b44add1e21..e06d4eb33c 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -40,7 +40,6 @@ RECESSIVE_MOI = 'R' MITO_MOI = 'M' -MAX_AFFECTED_FAMILY_FILTER = 'max_affected' CONFIRMED_FAMILY_FILTER = 'confirmed_inheritance' AFFECTED_MALE_FAMILY_FILTER = 'affected_males' @@ -217,7 +216,6 @@ }, 'Compound Heterozygous - Confirmed': { 'family_filter': { - MAX_AFFECTED_FAMILY_FILTER: 1, CONFIRMED_FAMILY_FILTER: True }, 'inheritance_mode': COMPOUND_HET, @@ -392,7 +390,7 @@ }, 'qualityFilter': { 'min_hl': 5, - 'min_mitoCn': 250, + 'min_mitoCn': 150, }, }, }, @@ -408,6 +406,20 @@ 'freqs': FREQ_FILTER, 'qualityFilter': PASS_QUALITY_FILTER, }, + 'Compound Heterozygous - Clinvar Pathogenic/ SV': { + 'annotations': SV_ANNOTATIONS, + 'pathogenicity': CLINVAR_FILTER, + 'freqs': FREQ_FILTER, + 'qualityFilter': PASS_QUALITY_FILTER, + }, + 'Compound Heterozygous - High Splice AI/ SV': { + 'annotations': { + **SV_ANNOTATIONS, + 'splice_ai': 0.8, + }, + 'freqs': FREQ_FILTER, + 'qualityFilter': PASS_QUALITY_FILTER, + }, 'Compound Heterozygous - One SV - Confirmed': { 'family_filter': { CONFIRMED_FAMILY_FILTER: True, @@ -534,8 +546,6 @@ def _get_valid_family_sample_data(cls, project, sample_type, samples_by_family, @staticmethod def _family_passes_filter(sample_data, family_filter): - if family_filter.get(MAX_AFFECTED_FAMILY_FILTER) and len(sample_data['affecteds']) > family_filter[MAX_AFFECTED_FAMILY_FILTER]: - return False if family_filter.get(AFFECTED_MALE_FAMILY_FILTER) and all(s['sex'] not in Individual.MALE_SEXES for s in sample_data['affecteds']): return False if CONFIRMED_FAMILY_FILTER in family_filter: From 3c964cf6cc3440434b42012cbdbf08c050ab1313 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 14:51:50 -0400 Subject: [PATCH 097/252] adjust multi dt quality filter --- .../commands/tag_seqr_prioritized_variants.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index e06d4eb33c..5ddf7856df 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -1,5 +1,6 @@ from collections import defaultdict from datetime import datetime + from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand, CommandError from django.db.models import Q @@ -113,6 +114,10 @@ **SV_QUALITY_FILTER, 'vcf_filter': 'PASS', } +PERMISSIVE_PASS_QUALITY_FILTER = { + **PASS_QUALITY_FILTER, + 'min_gq_sv': 50, +} CONFIRMED_HIGH_SPLICE_AI_SEARCH = { 'family_filter': { @@ -404,13 +409,13 @@ }, 'in_silico': IN_SILICO_FILTER, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, 'Compound Heterozygous - Clinvar Pathogenic/ SV': { 'annotations': SV_ANNOTATIONS, 'pathogenicity': CLINVAR_FILTER, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, 'Compound Heterozygous - High Splice AI/ SV': { 'annotations': { @@ -418,7 +423,7 @@ 'splice_ai': 0.8, }, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, 'Compound Heterozygous - One SV - Confirmed': { 'family_filter': { @@ -430,7 +435,7 @@ }, 'in_silico': IN_SILICO_FILTER, 'freqs': FREQ_FILTER, - 'qualityFilter': PASS_QUALITY_FILTER, + 'qualityFilter': PERMISSIVE_PASS_QUALITY_FILTER, }, } From 9097a7298dd83e45633e48956fd5fe2aecf89295 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 15:55:27 -0400 Subject: [PATCH 098/252] fix per gene behavior for mane vs canonical --- clickhouse_search/managers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 93c0f38df5..8eae82707b 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -692,14 +692,15 @@ def _consequence_term_filter(consequences, **kwargs): def _annotate_filtered_transcripts(self, results, consequence_field, transcript_filters, *args, require_mane_canonical=False, **kwargs): if require_mane_canonical: - filtered_expr = ArrayFilter(consequence_field, conditions=[{'canonical': (0, '{field} > {value}')}]) if 'isManeSelect' in self.sorted_transcript_consequence_fields: - filtered_expr = If( - ArrayFilter(consequence_field, conditions=[{'isManeSelect': (True, '{field}')}]), - filtered_expr, - condition='arrayExists(x -> x.isManeSelect, sortedTranscriptConsequences), ', - ) - results = results.annotate(**{self.FILTERED_CONSEQUENCE_FIELD: filtered_expr}) + mane_genes_expression = 'arrayMap(a -> a.geneId, arrayFilter(b -> b.isManeSelect, sortedTranscriptConsequences))' + conditions=[ + {'isManeSelect': (True, '{field}')}, + {'canonical': (mane_genes_expression, 'and({field} > 0, not has({value}, x.geneId))')}, + ] + else: + conditions = [{'canonical': (0, '{field} > {value}')}] + results = results.annotate(**{self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=conditions)}) consequence_field = self.FILTERED_CONSEQUENCE_FIELD return super()._annotate_filtered_transcripts(results, consequence_field, transcript_filters, **kwargs) From f5194056815b4080a800567e5eda5fb7b8313a6c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 15:57:46 -0400 Subject: [PATCH 099/252] fix tests --- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 9b7c80a949..284ed0e963 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -44,6 +44,8 @@ } MULTI_TYPE_MATCHES = { 'Compound Heterozygous - One SV': 0, + 'Compound Heterozygous - Clinvar Pathogenic/ SV': 1, + 'Compound Heterozygous - High Splice AI/ SV': 0, 'Compound Heterozygous - One SV - Confirmed': 1, } @@ -90,7 +92,7 @@ def test_command(self, mock_datetime, mock_slack, mock_email): expected_tags = { (2,): {"Clinvar Pathogenic - Recessive": "2025-11-15", "Recessive": "2025-11-15"}, - (2, 19): {"Compound Heterozygous - One SV - Confirmed": "2025-11-15"}, + (2, 19): {"Compound Heterozygous - One SV - Confirmed": "2025-11-15", "Compound Heterozygous - Clinvar Pathogenic/ SV": "2025-11-15"}, (3, 4): {"Compound Heterozygous": "2025-11-15"}, (6,): {'Mitochondrial - De Novo/ Dominant': '2025-11-15'}, (8,): {"Mitochondrial - Pathogenic": "2025-11-15"}, From f4ed4daa29a872f75e9c47880cbcc3b8276a5f3f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 15:58:42 -0400 Subject: [PATCH 100/252] clean up --- seqr/management/commands/tag_seqr_prioritized_variants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 5ddf7856df..9ef1fca0d6 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -1,6 +1,5 @@ from collections import defaultdict from datetime import datetime - from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand, CommandError from django.db.models import Q From 6f9dc82dfa7cdfff5863180055da68456dd86dfa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 17:10:37 -0400 Subject: [PATCH 101/252] fix check auth behavior --- clickhouse_search/search_tests.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3f299d41a0..5b23069afa 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -44,6 +44,8 @@ from seqr.views.utils.test_utils import AnvilAuthenticationTestCase from seqr.views.apis.variant_search_api import query_variants_handler +SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] + class ClickhouseSearchTestCase(AnvilAuthenticationTestCase): @@ -120,9 +122,6 @@ def setUp(self): self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) self.results_model.families.set(self.families) - url = reverse(query_variants_handler, args=['abc123']) - self.check_require_login(url) - super().setUp() def set_cache(self, cached): @@ -134,7 +133,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, request_body=None, **search_kwargs): + def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, **search_kwargs): search_hash = random.randint(1000, 10000) result_guid = f'VRS{search_hash:07d}' self.mock_results_guid.return_value = result_guid @@ -149,12 +148,19 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit if inheritance_filter is not None: search_body['inheritance']['filter'] = inheritance_filter - return self.client.post(url, content_type='application/json', data=json.dumps({ - **request_body, 'search': search_body, - })), result_guid, search_body + request_data = { + **(request_body or {'projectFamilies': project_families}), 'search': search_body, + } + + if check_login: + check_login(url, request_data=request_data) + + response = self.client.post(url, content_type='application/json', data=json.dumps(request_data)) + + return response, result_guid, search_body def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, response_search=None, format_cache_key=None, project_families=None, **kwargs): - response, result_guid, search_body = self._execute_search(**kwargs) + response, result_guid, search_body = self._execute_search(project_families=project_families, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, @@ -204,6 +210,7 @@ def _assert_expected_search_error(self, error, **kwargs): self.assertDictEqual(response.json(), {'error': error}) def _assert_expected_search_old(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): + # TODO remove results_model = results_model or self.results_model self.search_model.search.update(search_kwargs or {}) self.search_model.search['qualityFilter'] = quality_filter @@ -260,6 +267,7 @@ def _set_multi_project_search(self): self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11'])) def _set_single_family_search(self): + # TODO remove self.results_model.families.set(self.families.filter(guid='F000002_2')) def _set_sv_family_search(self): @@ -275,7 +283,6 @@ def _set_grch37_search(self): self._set_single_family_search() def test_single_family_search(self): - self._set_single_family_search() variant_gene_counts = { 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, @@ -283,7 +290,7 @@ def test_single_family_search(self): } self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4], gene_counts=variant_gene_counts, locus={'rawItems': '1:1-100000000'}, - exclude_svs=True, + exclude_svs=True, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1836,6 +1843,7 @@ def test_gene_variant_lookup(self): } self._assert_expected_search_error( 'Including external projects is only available when searching for a single gene', request_body=request_body, + check_login=self.check_require_login, ) annotations = { From f91cee37e0c9350d7f8d685c17f2c99c9d0c0757 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 23:02:47 -0400 Subject: [PATCH 102/252] gene counts test --- clickhouse_search/search_tests.py | 52 ++++++++++--------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 5b23069afa..b722374bb6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -42,7 +42,7 @@ from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.views.apis.variant_search_api import query_variants_handler +from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] @@ -115,6 +115,7 @@ def setUp(self): self.mock_results_guid.return_value = random.randint(1000, 10000) self.addCleanup(patcher.stop) + # TODO remove self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) self.user = User.objects.get(username='test_user') @@ -135,8 +136,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, **search_kwargs): search_hash = random.randint(1000, 10000) - result_guid = f'VRS{search_hash:07d}' - self.mock_results_guid.return_value = result_guid + self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) search_body = { @@ -157,10 +157,10 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit response = self.client.post(url, content_type='application/json', data=json.dumps(request_data)) - return response, result_guid, search_body + return response, search_hash, search_body def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, response_search=None, format_cache_key=None, project_families=None, **kwargs): - response, result_guid, search_body = self._execute_search(project_families=project_families, **kwargs) + response, search_hash, search_body = self._execute_search(project_families=project_families, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, @@ -191,7 +191,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun }) self.assertDictEqual(response.json(), expected_response) - cache_key = format_cache_key() if format_cache_key else f'search_results__{result_guid}__{sort}' + cache_key = format_cache_key() if format_cache_key else f'search_results__VRS{search_hash:07d}__{sort}' if cache_key: cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) @@ -200,7 +200,13 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.mock_redis.set.assert_not_called() if gene_counts: - raise NotImplementedError + gene_breakdown_url = reverse(get_variant_gene_breakdown, args=[search_hash]) + gene_breakdown_response = self.client.get(gene_breakdown_url) + self.assertEqual(gene_breakdown_response.status_code, 200) + self.assertDictEqual(gene_breakdown_response.json(), { + 'searchGeneBreakdown': {str(search_hash): gene_counts}, + 'genesById': mock.ANY, + }) return response.json() @@ -209,27 +215,6 @@ def _assert_expected_search_error(self, error, **kwargs): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': error}) - def _assert_expected_search_old(self, expected_results, results_page=None, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, **search_kwargs): - # TODO remove - results_model = results_model or self.results_model - self.search_model.search.update(search_kwargs or {}) - self.search_model.search['qualityFilter'] = quality_filter - self.search_model.search['inheritance']['mode'] = inheritance_mode - if inheritance_filter is not None: - self.search_model.search['inheritance']['filter'] = inheritance_filter - - variants, total = query_variants(results_model, user=self.user, sort=sort, load_all=load_all, page=page, num_results=num_results) - self.assertEqual(total, len(expected_results)) - self._assert_expected_variants( - variants, expected_results, cache_key=f'search_results__{results_model.guid}__{sort}', sort=sort, - format_cached_variants=self._format_cached_variants, cached_variant_fields=cached_variant_fields, - results_page=results_page, - ) - - if gene_counts: - gene_counts_json = get_variant_query_gene_counts(results_model, self.user) - self.assertDictEqual(gene_counts_json, gene_counts) - def _assert_expected_variants(self, variants, expected_results, cache_key=None, results_page=None, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = self._encode_variants(variants) self.assertListEqual(encoded_variants, results_page or expected_results) @@ -266,10 +251,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields): def _set_multi_project_search(self): self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11'])) - def _set_single_family_search(self): - # TODO remove - self.results_model.families.set(self.families.filter(guid='F000002_2')) - def _set_sv_family_search(self): self.results_model.families.set(Family.objects.filter(guid__in=['F000014_14'])) @@ -280,7 +261,6 @@ def _set_grch37_search(self): Project.objects.filter(id=1).update(genome_version='37') Sample.objects.filter(sample_id='HG00732').update(is_active=False) Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False) - self._set_single_family_search() def test_single_family_search(self): variant_gene_counts = { @@ -295,12 +275,13 @@ def test_single_family_search(self): self._assert_expected_search( [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=MITO_GENE_COUNTS, locus={'rawItems': 'M:1-100000000'}, - exclude_svs=False, + exclude_svs=False, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts=GCNV_GENE_COUNTS, locus=None, annotations={'structural': COMP_HET_ALL_PASS_FILTERS['annotations']['structural']}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_sv_family_search() @@ -313,10 +294,11 @@ def test_single_family_search(self): [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT]) + self._assert_expected_search([GRCH37_VARIANT], project_families=SINGLE_FAMILY_PROJECT_FAMILIES) def test_standard_searches(self): results_model = self._saved_search_results_model('De Novo/Dominant Restrictive') From d146cfeaa4d144af5bcb50672af5e83cd951a0fc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 9 Mar 2026 23:27:17 -0400 Subject: [PATCH 103/252] 37 tests --- clickhouse_search/search_tests.py | 52 +++++++++++++++++-------------- clickhouse_search/test_utils.py | 3 ++ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index b722374bb6..6aca399166 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -35,7 +35,7 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - format_cached_variant + SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException @@ -44,8 +44,6 @@ from seqr.views.utils.test_utils import AnvilAuthenticationTestCase from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown -SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] - class ClickhouseSearchTestCase(AnvilAuthenticationTestCase): @@ -159,7 +157,7 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', load_all=False, page=1, num_results=100, results_model=None, response_search=None, format_cache_key=None, project_families=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=None, is_37=False, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -175,20 +173,29 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun expected_response.update({ 'genesById': mock.ANY, 'locusListsByGuid': mock.ANY, - 'mmeSubmissionsByGuid': {}, - 'omimIntervals': {}, + 'mmeSubmissionsByGuid': mock.ANY, 'phenotypeGeneScores': mock.ANY, - 'rnaSeqData': {}, - 'savedVariantsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'variantNotesByGuid': {}, - 'variantTagsByGuid': {}, - 'totalSampleCounts': { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - }, + 'rnaSeqData': mock.ANY, + 'savedVariantsByGuid': mock.ANY, + 'variantFunctionalDataByGuid': mock.ANY, + 'variantNotesByGuid': mock.ANY, + 'variantTagsByGuid': mock.ANY, }) + if is_37: + expected_response.update({ + 'totalSampleCounts': { + 'SNV_INDEL': {'WES': 6}, + } + }) + else: + expected_response.update({ + 'omimIntervals': mock.ANY, + 'totalSampleCounts': { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + } + }) self.assertDictEqual(response.json(), expected_response) cache_key = format_cache_key() if format_cache_key else f'search_results__VRS{search_hash:07d}__{sort}' @@ -251,9 +258,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields): def _set_multi_project_search(self): self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11'])) - def _set_sv_family_search(self): - self.results_model.families.set(Family.objects.filter(guid__in=['F000014_14'])) - def _reset_search_families(self): self.results_model.families.set(self.families) @@ -284,21 +288,21 @@ def test_single_family_search(self): project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, annotations=None, + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, + project_families=SV_PROJECT_FAMILIES, ) - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000014_14'])) self._assert_expected_search( [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, - project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT], project_families=SINGLE_FAMILY_PROJECT_FAMILIES) + self._assert_expected_search([GRCH37_VARIANT], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, is_37=True) def test_standard_searches(self): results_model = self._saved_search_results_model('De Novo/Dominant Restrictive') diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 4364279d52..a679ba1fa2 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1393,3 +1393,6 @@ def format_cached_variant(variant): 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, } + +SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] +SV_PROJECT_FAMILIES = [{'projectGuid': 'R0004_non_analyst_project', 'familyGuids': ['F000014_14']}] From a9908a76e7ec8ef9c7ee420c4b309df5cc609f7d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 12:47:23 -0400 Subject: [PATCH 104/252] handle individuals with no loaded data --- ui/pages/Project/components/FamilyTable/IndividualRow.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 16edcef0bf..2d4b134359 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -547,7 +547,7 @@ class IndividualRow extends React.PureComponent { project: PropTypes.object.isRequired, individual: PropTypes.object.isRequired, mmeSubmission: PropTypes.object, - datasets: PropTypes.arrayOf(PropTypes.object).isRequired, + datasets: PropTypes.arrayOf(PropTypes.object), alignmentSample: PropTypes.object, dispatchUpdateIndividual: PropTypes.func, dispatchUpdateIndividualIGV: PropTypes.func, @@ -580,7 +580,7 @@ class IndividualRow extends React.PureComponent { const { displayName, sex, affected, createdDate } = individual // only show active or first/ last inactive samples - const loadedDatasets = datasets.reverse().filter( + const loadedDatasets = (datasets || []).reverse().filter( ({ isActive }, i) => isActive || i === 0 || i === datasets.length - 1, ) From 0dbe5d58e16d03b1ffd70663943ea999b319c544 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 14:43:43 -0400 Subject: [PATCH 105/252] standard search test --- clickhouse_search/search_tests.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6aca399166..7888b84b58 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -305,34 +305,26 @@ def test_single_family_search(self): self._assert_expected_search([GRCH37_VARIANT], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, is_37=True) def test_standard_searches(self): - results_model = self._saved_search_results_model('De Novo/Dominant Restrictive') self._assert_expected_search( - [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': None}, {}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='De Novo/Dominant Restrictive').search, + check_login=self.check_collaborator_login, ) - results_model = self._saved_search_results_model('De Novo/Dominant Permissive') self._assert_expected_search( - [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + [VARIANT1, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': None}, {}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='De Novo/Dominant Permissive').search, ) - results_model = self._saved_search_results_model('Recessive Restrictive') - self._assert_expected_search([VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + self._assert_expected_search([VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][0]}, {}, - ]) + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='Recessive Restrictive').search) - results_model = self._saved_search_results_model('Recessive Permissive') - self._assert_expected_search([VARIANT2, MITO_VARIANT3], results_model=results_model, cached_variant_fields=[ + self._assert_expected_search([VARIANT2, MITO_VARIANT3], cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][0]}, {}, - ]) - - def _saved_search_results_model(self, name): - results_model = VariantSearchResults.objects.create(variant_search=VariantSearch.objects.get(name=name), search_hash=name) - results_model.families.set(self.families.filter(guid='F000002_2')) - return results_model + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **VariantSearch.objects.get(name='Recessive Permissive').search) def test_single_project_search(self): variant_gene_counts = { From 8b762028a0d5d4add13acf5c9b321bf774a88637 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 14:56:23 -0400 Subject: [PATCH 106/252] add efaults --- clickhouse_search/search_tests.py | 58 +++++++++++++++---------------- clickhouse_search/test_utils.py | 1 + 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 7888b84b58..ca8925583f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -35,7 +35,7 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, format_cached_variant + DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException @@ -157,7 +157,7 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=None, is_37=False, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=DEFAULT_PROJECT_FAMILIES, is_37=False, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -180,22 +180,10 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun 'variantFunctionalDataByGuid': mock.ANY, 'variantNotesByGuid': mock.ANY, 'variantTagsByGuid': mock.ANY, + 'totalSampleCounts': mock.ANY, }) - if is_37: - expected_response.update({ - 'totalSampleCounts': { - 'SNV_INDEL': {'WES': 6}, - } - }) - else: - expected_response.update({ - 'omimIntervals': mock.ANY, - 'totalSampleCounts': { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - } - }) + if not is_37: + expected_response['omimIntervals'] = mock.ANY self.assertDictEqual(response.json(), expected_response) cache_key = format_cache_key() if format_cache_key else f'search_results__VRS{search_hash:07d}__{sort}' @@ -335,7 +323,7 @@ def test_single_project_search(self): } self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, - GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], check_login=self.check_collaborator_login, gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} ) @@ -368,6 +356,7 @@ def test_multi_project_search(self): [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], gene_counts=gene_counts, locus={'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + check_login=self.check_collaborator_login, ) self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) @@ -455,7 +444,7 @@ def test_inheritance_filter(self): self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - inheritance_mode=inheritance_mode, + inheritance_mode=inheritance_mode, check_login=self.check_collaborator_login, ) self._set_sv_family_search() @@ -660,7 +649,7 @@ def test_exclude_previous_search_results(self): [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], check_login=self.check_collaborator_login, ) self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ @@ -678,7 +667,7 @@ def test_quality_filter(self): quality_filter = {'vcf_filter': 'pass'} self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], - quality_filter=quality_filter + quality_filter=quality_filter, check_login=self.check_collaborator_login, ) self._set_sv_family_search() @@ -783,7 +772,7 @@ def test_location_search(self): self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4], **LOCATION_SEARCH, cached_variant_fields=[ {'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'} - ], + ], check_login=self.check_collaborator_login, ) sv_locus = {'rawItems': 'ENSG00000275023, ENSG00000171621'} @@ -830,7 +819,7 @@ def test_location_search(self): self._assert_expected_search([GRCH37_VARIANT], locus={'rawItems': '7:143268894-143271480'}) def test_variant_id_search(self): - self._assert_expected_search([VARIANT1], **VARIANT_ID_SEARCH) + self._assert_expected_search([VARIANT1], **VARIANT_ID_SEARCH, check_login=self.check_collaborator_login) self._assert_expected_search( [VARIANT1], locus={'rawVariantItems': VARIANT_IDS[0]}, @@ -1094,7 +1083,7 @@ def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {'ac': 7}, **sv_callset_filter}, + freqs={'callset': {'ac': 7}, **sv_callset_filter}, check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1169,6 +1158,7 @@ def test_annotations_filter(self): pathogenicity = {'clinvar': clinvar_paths, 'hgmd': []} self._assert_expected_search( [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, + check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1341,6 +1331,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='compound_het', annotations=annotations_1, annotations_secondary=annotations_2, cached_variant_fields=[comp_het_cached_fields], + check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1569,6 +1560,7 @@ def test_in_silico_filter(self): main_in_silico = {'eigen': '3.5', 'mut_taster': 'N', 'vest': 0.5} self._assert_expected_search( [VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], in_silico=main_in_silico, + check_login=self.check_collaborator_login, ) in_silico = {**main_in_silico, 'requireScore': True} @@ -1604,7 +1596,7 @@ def test_sort(self): self._set_single_family_search() self._assert_expected_search( [VARIANT4, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1, VARIANT2, MITO_VARIANT2, MITO_VARIANT3, MITO_VARIANT1, VARIANT3, VARIANT1], - sort='protein_consequence', + sort='protein_consequence', check_login=self.check_collaborator_login, ) self._reset_search_families() @@ -1740,7 +1732,8 @@ def test_multi_data_type_comp_het_sort(self): sort='protein_consequence', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], - [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], {}, {}] + [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], {}, {}], + check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1841,9 +1834,14 @@ def test_gene_variant_lookup(self): response_json = self._assert_expected_search( [variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], - annotations=annotations, freqs=freqs, locus=locus, + annotations=annotations, freqs=freqs, locus=locus, project_families=None, ) self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY}) + self.assertEqual(response_json['totalSampleCounts'], { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + }) freqs = {'callset': freqs['callset']} variant3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611', 'numFamilies': 4} @@ -1855,7 +1853,7 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - annotations=annotations, freqs=freqs, locus=locus, + annotations=annotations, freqs=freqs, locus=locus, project_families=None, ) self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY, 'ENSG00000177000': mock.ANY}) @@ -1865,7 +1863,7 @@ def test_gene_variant_lookup(self): ) self._assert_expected_search( - [], request_body=request_body, response_search=response_search, + [], request_body=request_body, response_search=response_search, project_families=None, annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='homozygous_recessive', ) @@ -1877,7 +1875,7 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=None, ) self.login_collaborator() diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index a679ba1fa2..300bed5f07 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1394,5 +1394,6 @@ def format_cached_variant(variant): 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, } +DEFAULT_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2', 'F000003_3', 'F000005_5']}] SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] SV_PROJECT_FAMILIES = [{'projectGuid': 'R0004_non_analyst_project', 'familyGuids': ['F000014_14']}] From 9278aabf1f4a99109190679814d8bffba80447bd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 16:43:19 -0400 Subject: [PATCH 107/252] mmore test updates --- clickhouse_search/search_tests.py | 206 ++++++++++++------------------ clickhouse_search/test_utils.py | 3 + 2 files changed, 85 insertions(+), 124 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ca8925583f..052cd453eb 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -35,7 +35,8 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, format_cached_variant + DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ + format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException @@ -243,12 +244,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields): **(cached_variant_fields or {}), } - def _set_multi_project_search(self): - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11'])) - - def _reset_search_families(self): - self.results_model.families.set(self.families) - def _set_grch37_search(self): Project.objects.filter(id=1).update(genome_version='37') Sample.objects.filter(sample_id='HG00732').update(is_active=False) @@ -345,39 +340,39 @@ def test_single_project_search(self): ) def test_multi_project_search(self): - self._set_multi_project_search() gene_counts = { **GENE_COUNTS, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2, 'F000011_11': 1}}, } + locus = {'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'} self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts=gene_counts, locus={'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, - check_login=self.check_collaborator_login, + gene_counts=gene_counts, locus=locus, check_login=self.check_collaborator_login, project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) + self.login_manager() self._assert_expected_search( [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, - MITO_VARIANT2, MITO_VARIANT3], gene_counts={**gene_counts, **SV_GENE_COUNTS}, + MITO_VARIANT2, MITO_VARIANT3], gene_counts={**gene_counts, **SV_GENE_COUNTS}, locus=locus, project_families=[ + *MULTI_PROJECT_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES, + ] ) def test_both_sample_types_search(self): Sample.objects.filter(dataset_type='MITO').update(is_active=False) # One family (F000011_11) in a multi-project search has identical exome and genome data. - self._set_multi_project_search() self._add_sample_type_samples('WES', individual__family__guid='F000011_11') self._assert_expected_search( MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) - self._set_single_family_search() self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731']) # Variant 1 is de novo in exome but inherited and homozygous in genome. @@ -387,34 +382,34 @@ def test_both_sample_types_search(self): self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode='any_affected', locus=None, + inheritance_mode='any_affected', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4], - inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, + inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', quality_filter=None, + inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode='any_affected', + inheritance_mode='any_affected', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', + inheritance_mode='de_novo', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], - inheritance_mode='de_novo', quality_filter={'min_gq': 40} + inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self.maxDiff = None @@ -422,7 +417,7 @@ def test_both_sample_types_search(self): [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4], [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], + GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, @@ -447,16 +442,16 @@ def test_inheritance_filter(self): inheritance_mode=inheritance_mode, check_login=self.check_collaborator_login, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], inheritance_mode=inheritance_mode, + project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, + [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT3], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, ) @@ -467,14 +462,12 @@ def test_inheritance_filter(self): inheritance_mode=inheritance_mode, annotations=None, ) - self._set_sv_family_search() sv_affected = {'affected': {'I000019_na21987': 'N'}} self._assert_expected_search( - [SV_VARIANT1], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, + [SV_VARIANT1], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, project_families=SV_PROJECT_FAMILIES, ) inheritance_mode = 'x_linked_recessive' - self._reset_search_families() self._assert_expected_search([], inheritance_mode=inheritance_mode) # self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) @@ -484,29 +477,25 @@ def test_inheritance_filter(self): inheritance_mode=inheritance_mode, ) - self._set_multi_project_search() self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2, GCNV_VARIANT3, MITO_VARIANT3], + [PROJECT_2_VARIANT1, VARIANT2, GCNV_VARIANT3, MITO_VARIANT3], project_families=MULTI_PROJECT_PROJECT_FAMILIES, inheritance_mode=inheritance_mode, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4], inheritance_mode=inheritance_mode, locus=None, + [SV_VARIANT4], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, project_families=SV_PROJECT_FAMILIES, ) gt_inheritance_filter = {'genotype': {'I000006_hg00733': 'ref_ref', 'I000005_hg00732': 'has_alt'}} - self._set_single_family_search() - self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_filter=gt_inheritance_filter) + self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_filter=gt_inheritance_filter, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( [VARIANT2, GCNV_VARIANT2, GCNV_VARIANT3], inheritance_mode='any_affected', inheritance_filter={'affected': { 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'U', - }}, + }}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) inheritance_mode = 'compound_het' - self._reset_search_families() self._assert_expected_search( [[VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, inheritance_filter={}, gene_counts={ 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, @@ -541,17 +530,15 @@ def test_inheritance_filter(self): ] ) - self._set_sv_family_search() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, gene_counts={'ENSG00000171621': {'total': 2, 'families': {'F000014_14': 2}}}, inheritance_filter=sv_affected, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], - ], + ], project_families=SV_PROJECT_FAMILIES, ) inheritance_mode = 'recessive' - self._set_multi_project_search() self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, gene_counts={ 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, @@ -562,16 +549,15 @@ def test_inheritance_filter(self): {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, ], **ALL_SNV_INDEL_PASS_FILTERS, locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) - self._set_single_family_search() self._assert_expected_search( [VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], locus=None, + ], locus=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, **ALL_SNV_INDEL_PASS_FILTERS, ) - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, gene_counts={ @@ -603,7 +589,6 @@ def test_inheritance_filter(self): ], ) - self._set_sv_family_search() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, gene_counts={ @@ -611,7 +596,7 @@ def test_inheritance_filter(self): 'ENSG00000184986': {'total': 1, 'families': {'F000014_14': 1}}, }, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, - ], + ], project_families=SV_PROJECT_FAMILIES, inheritance_filter=sv_affected, ) # Test deletion in trans with hom alt snp/indel @@ -621,18 +606,19 @@ def test_inheritance_filter(self): sample.save() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4], - inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, + inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, {}, - ], + ], project_families=SV_PROJECT_FAMILIES, ) self._set_grch37_search() - self._assert_expected_search([], inheritance_mode=inheritance_mode) + self._assert_expected_search([], inheritance_mode=inheritance_mode, **COMP_HET_ALL_PASS_FILTERS, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( [GRCH37_VARIANT], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}, + **COMP_HET_ALL_PASS_FILTERS, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, is_37=True, ) def test_exclude_previous_search_results(self): @@ -670,13 +656,12 @@ def test_quality_filter(self): quality_filter=quality_filter, check_login=self.check_collaborator_login, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT4], quality_filter=quality_filter, + [SV_VARIANT4], quality_filter=quality_filter, project_families=SV_PROJECT_FAMILIES, ) gcnv_quality_filter = {'min_gq': 40, 'min_qs': 20, 'min_hl': 5} - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT3], quality_filter=gcnv_quality_filter, ) @@ -686,16 +671,14 @@ def test_quality_filter(self): ) sv_quality_filter = {'min_gq_sv': 40} - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, annotations=None, + [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, annotations=None, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, + [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, MITO_VARIANT1, MITO_VARIANT2], quality_filter={'min_gq': 40, 'min_qs': 30, 'vcf_filter': 'pass'}, annotations=None, @@ -706,12 +689,10 @@ def test_quality_filter(self): quality_filter={'min_gq': 60, 'min_qs': 10, 'affected_only': True}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, + [SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT1, VARIANT2, FAMILY_3_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4,MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], quality_filter={'min_ab': 50}, ) @@ -780,28 +761,25 @@ def test_location_search(self): [GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, ) - self._set_sv_family_search() + self.login_manager() # For gene search, return SVs annotated in gene even if they fall outside the gene interval self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, + [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'} + [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT1, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], exclude={ 'rawItems': ','.join([LOCATION_SEARCH['locus']['rawItems'], sv_locus['rawItems']]) }, locus=None, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT2, SV_VARIANT3], exclude=sv_locus, + [SV_VARIANT2, SV_VARIANT3], exclude=sv_locus, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT, VARIANT4], locus={'rawItems': f'{GENE_IDS[1]}\n1:91500851-91525764'}, exclude=None, cached_variant_fields=[ @@ -918,14 +896,13 @@ def test_invalid_search(self): self._assert_expected_search([], inheritance_filter={}) self.assertEqual(str(cm.exception), 'No search data found for families no_individuals') - self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([]) + self._assert_expected_search([], project_families=MULTI_PROJECT_PROJECT_FAMILIES,) self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}) + self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}, project_families=MULTI_PROJECT_PROJECT_FAMILIES) self.assertEqual( str(cm.exception), 'The following samples are incorrectly configured and have different affected statuses in different projects: ' @@ -937,15 +914,13 @@ def test_invalid_search(self): self._assert_expected_search([], locus={'rawItems': build_specific_genes}) self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') - self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus=None) + self._assert_expected_search([], locus=None, project_families=MULTI_PROJECT_PROJECT_FAMILIES) self.assertEqual(str(cm.exception),'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project') - self._set_single_family_search() self.set_cache({'total_results': 20000}) with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], page=1, num_results=2, load_all=True) + self._assert_expected_search([], page=1, num_results=2, load_all=True, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') @mock.patch('seqr.utils.search.utils.LiftOver') @@ -1094,12 +1069,11 @@ def test_frequency_filter(self): [MULTI_FAMILY_VARIANT, GCNV_VARIANT3, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], freqs={'callset': {'ac': 7, 'hh': 0}, 'sv_callset': {'ac': 50}}, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT1], freqs={'sv_callset': {'ac': 1}}, + [SV_VARIANT1], freqs={'sv_callset': {'ac': 1}}, project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], freqs={'callset': {'ac': 1000}, 'gnomad_genomes': {'af': 0.03}, 'gnomad_mito': {'af': 0.005}}, ) @@ -1113,29 +1087,27 @@ def test_frequency_filter(self): freqs={'callset': {'ac': 1000}, 'topmed': {'af': 0.05, 'hh': 1}, 'sv_callset': {'ac': 50}}, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, + [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'ac': 4000}}, + [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'ac': 4000}}, project_families=SV_PROJECT_FAMILIES, ) # seqr af filter is ignored for SNV_INDEL - self._set_single_family_search() self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {'af': 0.2}, **sv_callset_filter}, + freqs={'callset': {'af': 0.2}, **sv_callset_filter}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], - freqs={'callset': {'ac': 6}, 'gnomad_genomes': {'ac': 50}, 'gnomad_mito': {'ac': 10}}, + freqs={'callset': {'ac': 6}, 'gnomad_genomes': {'ac': 50}, 'gnomad_mito': {'ac': 10}}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {}, 'gnomad_genomes': {'af': None}}, + freqs={'callset': {}, 'gnomad_genomes': {'af': None}}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters @@ -1143,11 +1115,12 @@ def test_frequency_filter(self): [VARIANT1, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.002, 'hh': 10}, 'gnomad_mito': {'ac': 1000}}, annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus']}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [VARIANT2, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.002}, 'gnomad_mito': {'ac': 1000}}, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus']}, + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus']}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) def test_annotations_filter(self): @@ -1214,12 +1187,11 @@ def test_annotations_filter(self): ], ) - self._set_sv_family_search() - self._assert_expected_search([SV_VARIANT1], annotations=annotations) + self.login_manager() + self._assert_expected_search([SV_VARIANT1], annotations=annotations, project_families=SV_PROJECT_FAMILIES) annotations['splice_ai'] = '0.005' annotations['structural'] = ['gCNV_DUP', 'DEL'] - self._reset_search_families() self._assert_expected_search( [VARIANT2, MULTI_FAMILY_VARIANT, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], annotations=annotations, cached_variant_fields=[ @@ -1237,11 +1209,9 @@ def test_annotations_filter(self): [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], annotations=annotations, ) - self._set_sv_family_search() - self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations) + self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations, project_families=SV_PROJECT_FAMILIES) annotations = {'other': ['non_coding_transcript_exon_variant__canonical', 'non_coding_transcript_exon_variant']} - self._set_single_family_search() self._assert_expected_search( [VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, cached_variant_fields=[ @@ -1249,9 +1219,8 @@ def test_annotations_filter(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2], @@ -1272,12 +1241,11 @@ def test_annotations_filter(self): annotations['other'] = annotations['other'][:1] annotations['splice_ai'] = '0.005' - self._set_single_family_search() self._assert_expected_search( [VARIANT1, VARIANT3, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, locus=None, cached_variant_fields=[ {'selectedTranscript': None}, {'selectedTranscript': None}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations['extended_splice_site'] = ['extended_intronic_splice_region_variant'] @@ -1288,14 +1256,14 @@ def test_annotations_filter(self): {'selectedTranscript': None}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][0]}, {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'extended_splice_site': ['5_prime_UTR_variant']} self._assert_expected_search( [selected_transcript_variant_2], pathogenicity=None, annotations=annotations, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][1]}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations['extended_splice_site'].append('extended_intronic_splice_region_variant') @@ -1303,12 +1271,12 @@ def test_annotations_filter(self): [selected_transcript_variant_2, VARIANT4], annotations=annotations, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][1]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][0]}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) annotations = {'motif_feature': ['TF_binding_site_variant'], 'regulatory_feature': ['regulatory_region_variant']} self._assert_expected_search( - [VARIANT3, VARIANT4], annotations=annotations, pathogenicity=None, + [VARIANT3, VARIANT4], annotations=annotations, pathogenicity=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_grch37_search() @@ -1423,25 +1391,24 @@ def test_secondary_annotations_filter(self): sv_annotations_1 = {'structural': ['INS', 'LOF']} sv_annotations_2 = {'structural': ['DEL', 'gCNV_DUP'], 'structural_consequence': ['INTRONIC']} - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode='compound_het', locus=None, annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, inheritance_filter={'affected': { 'I000019_na21987': 'N', }}, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], - ], + ], project_families=SV_PROJECT_FAMILIES, ) self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode='recessive', annotations=sv_annotations_2, annotations_secondary=sv_annotations_1, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, - ], + ], project_families=SV_PROJECT_FAMILIES, ) pathogenicity = {'clinvar': ['likely_pathogenic', 'conflicting_p_lp', 'conflicting_no_p', 'vus']} - self._reset_search_families() self._assert_expected_search( [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4], MITO_VARIANT3], inheritance_mode='recessive', annotations=annotations_2, annotations_secondary=annotations_1, pathogenicity=pathogenicity, cached_variant_fields=[ @@ -1584,29 +1551,26 @@ def test_in_silico_filter(self): [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], in_silico=sv_in_silico, ) - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( - [SV_VARIANT4], in_silico=sv_in_silico, + [SV_VARIANT4], in_silico=sv_in_silico, project_families=SV_PROJECT_FAMILIES, ) self._set_grch37_search() self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico) def test_sort(self): - self._set_single_family_search() self._assert_expected_search( [VARIANT4, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1, VARIANT2, MITO_VARIANT2, MITO_VARIANT3, MITO_VARIANT1, VARIANT3, VARIANT1], - sort='protein_consequence', check_login=self.check_collaborator_login, + sort='protein_consequence', check_login=self.check_collaborator_login, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._reset_search_families() - self._set_sv_family_search() + self.login_manager() self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], - sort='protein_consequence', + sort='protein_consequence', project_families=SV_PROJECT_FAMILIES, ) - self._reset_search_families() self._assert_expected_search( [VARIANT4, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, MITO_VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT], sort='protein_consequence', @@ -1622,24 +1586,22 @@ def test_sort(self): [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], sort='pathogenicity', annotations=None, ) - self._set_single_family_search() self._assert_expected_search( [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], - sort='pathogenicity_hgmd', + sort='pathogenicity_hgmd', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] - self._assert_expected_search(sorted_variants, sort='gnomad') + self._assert_expected_search(sorted_variants, sort='gnomad', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( - sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, + sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( - sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, + sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4) + self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self._reset_search_families() self._assert_expected_search( [VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='gnomad_exomes', @@ -1682,47 +1644,43 @@ def test_sort(self): [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT3, GCNV_VARIANT4, VARIANT4, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort=sort, ) - self._set_single_family_search() - self._assert_expected_search([VARIANT2, VARIANT3, VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='prioritized_gene') + self._assert_expected_search([VARIANT2, VARIANT3, VARIANT1, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='prioritized_gene', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self._set_multi_project_search() self._assert_expected_search( [MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, PROJECT_2_VARIANT], sort='family_guid', locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) # size sort only applies to SVs, so has no impact on other variant - self._reset_search_families() self._assert_expected_search( [GCNV_VARIANT1, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='size', locus=None, ) - self._set_sv_family_search() self._assert_expected_search( - [SV_VARIANT4, SV_VARIANT1, SV_VARIANT3, SV_VARIANT2], sort='size', + [SV_VARIANT4, SV_VARIANT1, SV_VARIANT3, SV_VARIANT2], sort='size', project_families=SV_PROJECT_FAMILIES, ) # sort applies to compound hets - self._set_single_family_search() self._assert_expected_search( [[VARIANT4, VARIANT3], VARIANT2, MITO_VARIANT3], sort='revel', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [[VARIANT3, VARIANT4], VARIANT2, MITO_VARIANT3], sort='splice_ai', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._assert_expected_search( [MITO_VARIANT3, [VARIANT4, VARIANT3], VARIANT2], sort='callset_af', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, - ], + ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) def test_multi_data_type_comp_het_sort(self): diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 300bed5f07..ee8ad18f27 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1396,4 +1396,7 @@ def format_cached_variant(variant): DEFAULT_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2', 'F000003_3', 'F000005_5']}] SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] +MULTI_PROJECT_PROJECT_FAMILIES = [ + *SINGLE_FAMILY_PROJECT_FAMILIES, {'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']} +] SV_PROJECT_FAMILIES = [{'projectGuid': 'R0004_non_analyst_project', 'familyGuids': ['F000014_14']}] From 1ede9c64ab119834a719be50c056687d49147829 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 16:55:27 -0400 Subject: [PATCH 108/252] fix exclude previous tests --- clickhouse_search/search_tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 052cd453eb..0cb67a3b60 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -622,28 +622,30 @@ def test_inheritance_filter(self): ) def test_exclude_previous_search_results(self): + self.mock_results_guid.return_value = 'VRS00079516' VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') - self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + self.mock_redis.get.side_effect = [None, None, None, json.dumps({'all_results': [ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], ]})] self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] + exclude = {'previousSearch': True, 'previousSearchHash': 'abc1234'} self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - exclude={'previousSearch': True, 'previousSearchHash': 'abc1234'}, cached_variant_fields=[ + exclude=exclude, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, ], check_login=self.check_collaborator_login, ) - self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + self.mock_redis.get.side_effect = [None, None, json.dumps({'all_results': [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, ]})] self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] self._assert_expected_search( - [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], + [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, inheritance_mode='recessive', cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], ], From 9946575460be90db6d2a00c1238cfba2273567d4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 17:05:55 -0400 Subject: [PATCH 109/252] more test updates --- clickhouse_search/search_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 0cb67a3b60..6c0874e480 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -748,7 +748,7 @@ def test_quality_filter(self): quality_filter = {'min_gq': 1, 'min_ab': 10} self._assert_expected_search([], quality_filter=quality_filter, annotations=None, pathogenicity=None) self._assert_expected_search( - [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, + [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, is_37=True, ) def test_location_search(self): @@ -790,13 +790,13 @@ def test_location_search(self): ) self._add_sample_type_samples('WES', individual__family__guid='F000014_14') - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000014_14'])) self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, + project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT], locus={'rawItems': '7:143268894-143271480'}) + self._assert_expected_search([GRCH37_VARIANT], locus={'rawItems': '7:143268894-143271480'}, is_37=True) def test_variant_id_search(self): self._assert_expected_search([VARIANT1], **VARIANT_ID_SEARCH, check_login=self.check_collaborator_login) @@ -1286,7 +1286,7 @@ def test_annotations_filter(self): annotations['missense'] = ['missense_variant'] self._assert_expected_search( [GRCH37_VARIANT], pathogenicity=None, annotations=annotations, - cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[11][0]}], + cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[11][0]}], is_37=True, ) def test_secondary_annotations_filter(self): @@ -1559,7 +1559,7 @@ def test_in_silico_filter(self): ) self._set_grch37_search() - self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico) + self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico, is_37=True) def test_sort(self): self._assert_expected_search( From 41a33b64e542cd4db96d365436fd2aec610eab35 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 17:37:07 -0400 Subject: [PATCH 110/252] invalid search tests --- clickhouse_search/search_tests.py | 148 ++++++++++++++---------------- 1 file changed, 67 insertions(+), 81 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6c0874e480..098e82a166 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -133,7 +133,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, **search_kwargs): + def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=DEFAULT_PROJECT_FAMILIES, request_body=None, check_login=None, sort='xpos', **search_kwargs): search_hash = random.randint(1000, 10000) self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) @@ -154,12 +154,12 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit if check_login: check_login(url, request_data=request_data) - response = self.client.post(url, content_type='application/json', data=json.dumps(request_data)) + response = self.client.post(f'{url}?sort={sort}', content_type='application/json', data=json.dumps(request_data)) return response, search_hash, search_body def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=DEFAULT_PROJECT_FAMILIES, is_37=False, **kwargs): - response, search_hash, search_body = self._execute_search(project_families=project_families, **kwargs) + response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, @@ -813,117 +813,103 @@ def test_variant_id_search(self): @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawVariantItems': 'chr2-A-C'}) - self.assertEqual(str(cm.exception), 'Invalid variants: chr2-A-C') + self._assert_expected_search_error( + 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, check_login=self.check_collaborator_login, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawVariantItems': 'rs9876,chr2-1234-A-C'}) - self.assertEqual(str(cm.exception), 'Invalid variants: rs9876') + self._assert_expected_search_error('Invalid variants: rs9876', locus={'rawVariantItems': 'rs9876'}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': 'chr27:1234-5678,2:40-400000000, ENSG00012345'}) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345') + self._assert_expected_search_error( + 'Invalid genes/intervals: chr27:1234-5678, chr2:40-400000000, ENSG00012345', + locus={'rawItems': 'chr27:1234-5678,2:40-400000000, ENSG00012345'}, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': '1:1-1000, 2:2000-3000, 3:4000-5000'}) - self.assertEqual(str(cm.exception), 'Too many genes/intervals') + self._assert_expected_search_error('Too many genes/intervals', locus={'rawItems': '1:1-1000, 2:2000-3000, 3:4000-5000'}) build_specific_genes = 'DDX11L1, OR4F29, ENSG00000223972, ENSG00000256186' - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': build_specific_genes}) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + self._assert_expected_search_error('Invalid genes/intervals: OR4F29, ENSG00000256186', locus={'rawItems': build_specific_genes}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], exclude={'rawItems': build_specific_genes}) - self.assertEqual(str(cm.exception), 'Cannot specify both Location and Excluded Genes/Intervals') + self._assert_expected_search_error( + 'Cannot specify both Location and Excluded Genes/Intervals', + locus={'rawItems': build_specific_genes}, exclude={'rawItems': build_specific_genes}, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus=None) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: OR4F29, ENSG00000256186') + self._assert_expected_search_error('Invalid genes/intervals: OR4F29, ENSG00000256186', exclude={'rawItems': build_specific_genes}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], exclude=None) - self.assertEqual(str(cm.exception),'This search returned too many results') + self._assert_expected_search_error('This search returned too many results') - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}) - self.assertEqual(str(cm.exception),'ClinVar pathogenicity vus is both included and excluded') + self._assert_expected_search_error( + 'ClinVar pathogenicity vus is both included and excluded', + pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], exclude=None, inheritance_mode='recessive') - self.assertEqual(str(cm.exception),'Annotations must be specified to search for compound heterozygous variants') + self._assert_expected_search_error('Annotations must be specified to search for compound heterozygous variants', inheritance_mode='recessive') - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive') - self.assertEqual(str(cm.exception),'Location must be specified to search for compound heterozygous variants across many families') + self._assert_expected_search_error( + 'Location must be specified to search for compound heterozygous variants across many families', + annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive', + ) - with self.assertRaises(InvalidSearchException) as se: - self._assert_expected_search([], sort='prioritized_gene', num_results=2) - self.assertEqual(str(se.exception), 'Phenotype sort is only supported for single-family search.') + self._assert_expected_search_error( + 'Phenotype sort is only supported for single-family search.', + sort='prioritized_gene', annotations={'frameshift': ['frameshift_variant']}, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], freqs={}) - self.assertEqual(str(cm.exception),'seqr AC frequency of at least 5000 must be specified to search across multiple families') + self._assert_expected_search_error('seqr AC frequency of at least 5000 must be specified to search across multiple families', freqs={}) - self.results_model.families.set(Family.objects.filter(guid='F000005_5')) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], inheritance_mode='recessive') - self.assertEqual(str(cm.exception),'Inheritance based search is disabled in families with no data loaded for affected individuals') + self._assert_expected_search_error( + 'Inheritance based search is disabled in families with no data loaded for affected individuals', + inheritance_mode='recessive', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000005_5']}], + ) - self.results_model.families.set(Family.objects.filter(guid='F000003_3')) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], pathogenicity={}, annotations={'structural': ['DEL']}) - self.assertEqual(str(cm.exception), 'Unable to search against dataset type "SV"') + no_sv_project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000003_3']}] + self._assert_expected_search_error( + 'Unable to search against dataset type "SV"', pathogenicity={}, annotations={'structural': ['DEL']}, + project_families=no_sv_project_families, + ) - self.search_model.search['annotations_secondary'] = {'frameshift': ['frameshift_variant']} - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], inheritance_mode='recessive', annotations_secondary={'frameshift': ['frameshift_variant']}) - self.assertEqual(str(cm.exception),'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals') + self._assert_expected_search_error( + 'Unable to search for comp-het pairs with dataset type "SV". This may be because inheritance based search is disabled in families with no loaded affected individuals', + inheritance_mode='recessive', annotations={'structural': ['DEL']}, annotations_secondary={'frameshift': ['frameshift_variant']}, project_families=no_sv_project_families, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], inheritance_mode='recessive', inheritance_filter={'affected': {'I000007_na20870': 'N'}}) - self.assertEqual(str(cm.exception),'Inheritance based search is disabled in families with no data loaded for affected individuals') + self._assert_expected_search_error( + 'Inheritance based search is disabled in families with no data loaded for affected individuals', + inheritance_mode='recessive', inheritance_filter={'affected': {'I000007_na20870': 'N'}}, project_families=no_sv_project_families, + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([]) - self.assertEqual(str(cm.exception), 'Inheritance must be specified if custom affected status is set') + self._assert_expected_search_error('Inheritance must be specified if custom affected status is set', inheritance_filter={'affected': {'I000007_na20870': 'N'}}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref'}}) - self.assertEqual(str(cm.exception), 'Invalid custom inheritance') + self._assert_expected_search_error( + 'Invalid custom inheritance', project_families=no_sv_project_families, inheritance_filter={'genotype': {'I000004_hg00731': 'ref_ref'}}, + ) - self.results_model.families.set(Family.objects.filter(family_id='no_individuals')) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], inheritance_filter={}) - self.assertEqual(str(cm.exception), 'No search data found for families no_individuals') + self._assert_expected_search_error( + 'No search data found for families no_individuals', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000013_13']}], + ) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], project_families=MULTI_PROJECT_PROJECT_FAMILIES,) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') + self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES) Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}, project_families=MULTI_PROJECT_PROJECT_FAMILIES) - self.assertEqual( - str(cm.exception), + self._assert_expected_search_error( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', + locus={'rawItems': GENE_IDS[0]}, freqs={'callset': {'ac': 1000}}, project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) self._set_grch37_search() - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus={'rawItems': build_specific_genes}) - self.assertEqual(str(cm.exception), 'Invalid genes/intervals: DDX11L1, ENSG00000223972') + self._assert_expected_search_error('Invalid genes/intervals: DDX11L1, ENSG00000223972', locus={'rawItems': build_specific_genes}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], locus=None, project_families=MULTI_PROJECT_PROJECT_FAMILIES) - self.assertEqual(str(cm.exception),'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project') + self._assert_expected_search_error( + 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project', + project_families=MULTI_PROJECT_PROJECT_FAMILIES, + ) + # TODO test once have export tests in this file self.set_cache({'total_results': 20000}) with self.assertRaises(InvalidSearchException) as cm: self._assert_expected_search([], page=1, num_results=2, load_all=True, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self.assertEqual(str(cm.exception), 'Unable to export more than 1000 variants (20000 requested)') + self._assert_expected_search_error('Unable to export more than 1000 variants (20000 requested)') @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): From e22199d76fb5912b150640bd2a9933db4bf741a4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 18:04:40 -0400 Subject: [PATCH 111/252] codacy and test fixes --- clickhouse_search/search_tests.py | 52 ++++++++++++++++--------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 098e82a166..ab5938279c 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -111,7 +111,7 @@ def setUp(self): patcher = mock.patch('seqr.models.VariantSearchResults._compute_guid') self.mock_results_guid = patcher.start() - self.mock_results_guid.return_value = random.randint(1000, 10000) + self.mock_results_guid.return_value = random.randint(1000, 10000) # nosec self.addCleanup(patcher.stop) # TODO remove @@ -133,7 +133,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=DEFAULT_PROJECT_FAMILIES, request_body=None, check_login=None, sort='xpos', **search_kwargs): + def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, sort='xpos', **search_kwargs): search_hash = random.randint(1000, 10000) self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) @@ -148,7 +148,7 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit search_body['inheritance']['filter'] = inheritance_filter request_data = { - **(request_body or {'projectFamilies': project_families}), 'search': search_body, + **(request_body or {'projectFamilies': project_families or DEFAULT_PROJECT_FAMILIES}), 'search': search_body, } if check_login: @@ -158,14 +158,14 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=DEFAULT_PROJECT_FAMILIES, is_37=False, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=None, is_37=False, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { 'searchedVariants': results_page or expected_results, 'search': { 'search': {**search_body, **(response_search or {})}, - 'projectFamilies': project_families or [], + 'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families, 'totalResults': len(expected_results), }, @@ -913,6 +913,7 @@ def test_invalid_search(self): @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): + # TODO mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] @@ -1019,6 +1020,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3], 'variant_lookup_results__suffix_140593_DUP__38') def test_get_single_variant(self): + # TODO self._set_single_family_search() variant = get_single_variant(self.results_model.families.first(), VARIANT_IDS[0]) self._assert_expected_variants([variant], [VARIANT1]) @@ -1112,14 +1114,13 @@ def test_frequency_filter(self): ) def test_annotations_filter(self): - self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}) + self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}, check_login=self.check_collaborator_login) self._assert_expected_search([], pathogenicity={'hgmd': ['disease_causing', 'likely_disease_causing']}) clinvar_paths = ['likely_pathogenic', 'conflicting_p_lp', 'conflicting_no_p', 'vus', 'benign'] pathogenicity = {'clinvar': clinvar_paths, 'hgmd': []} self._assert_expected_search( [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, - check_login=self.check_collaborator_login, ) self._assert_expected_search( @@ -1210,9 +1211,10 @@ def test_annotations_filter(self): ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) + locus = {'rawItems': f'{GENE_IDS[1]}\n1:11785723-91525764'} self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2], - locus={'rawItems': f'{GENE_IDS[1]}\n1:11785723-91525764'}, pathogenicity=None, annotations=annotations, + locus=locus, pathogenicity=None, annotations=annotations, cached_variant_fields=[{ 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5], }], @@ -1221,7 +1223,7 @@ def test_annotations_filter(self): annotations['other'].append('intron_variant') self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], - annotations=annotations, cached_variant_fields=[ + annotations=annotations, locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][1]}, ], @@ -1354,12 +1356,13 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}]], ) + locus = {'rawItems': 'ENSG00000277258,ENSG00000275023'} self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, 'structural': ['gCNV_DEL'], 'structural_consequence': ['INTRONIC']}, annotations_secondary={**annotations_2, **gcnv_annotations_1}, - locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, cached_variant_fields=[ + locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, @@ -1369,7 +1372,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, 'structural': [], 'structural_consequence': []}, - annotations_secondary=gcnv_annotations_2, cached_variant_fields=[ + annotations_secondary=gcnv_annotations_2, locus=locus, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, @@ -1380,20 +1383,20 @@ def test_secondary_annotations_filter(self): sv_annotations_2 = {'structural': ['DEL', 'gCNV_DUP'], 'structural_consequence': ['INTRONIC']} self.login_manager() + inheritance_filter = {'affected': {'I000019_na21987': 'N'}} self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode='compound_het', locus=None, - annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, inheritance_filter={'affected': { - 'I000019_na21987': 'N', - }}, cached_variant_fields=[ + annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, inheritance_filter=inheritance_filter, + project_families=SV_PROJECT_FAMILIES, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], - ], project_families=SV_PROJECT_FAMILIES, + ], ) self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode='recessive', annotations=sv_annotations_2, annotations_secondary=sv_annotations_1, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000171621'}, {'selectedGeneId': 'ENSG00000171621'}], {}, - ], project_families=SV_PROJECT_FAMILIES, + ], inheritance_filter=inheritance_filter, project_families=SV_PROJECT_FAMILIES, ) pathogenicity = {'clinvar': ['likely_pathogenic', 'conflicting_p_lp', 'conflicting_no_p', 'vus']} @@ -1467,7 +1470,7 @@ def test_secondary_annotations_filter(self): self._assert_expected_search( [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [VARIANT3, VARIANT4], MITO_VARIANT3], annotations={**selected_transcript_annotations, **screen_annotations}, annotations_secondary=annotations_2, - inheritance_mode='recessive', cached_variant_fields=[ + inheritance_mode='recessive', pathogenicity=pathogenicity, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][5]}, [ {'selectedGeneId': 'ENSG00000097046', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][0]}, {'selectedGeneId': 'ENSG00000097046', 'selectedTranscript': None}, @@ -1476,20 +1479,19 @@ def test_secondary_annotations_filter(self): ) self._add_sample_type_samples('WES', individual__family__guid='F000014_14') - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, cached_variant_fields=[ + locus={'rawItems': 'ENSG00000277258,ENSG00000275023'}, pathogenicity=pathogenicity, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, [ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, {'selectedGeneId': 'ENSG00000277258'}, ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}]], + project_families=[*MULTI_PROJECT_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], ) # Search works with a different number of samples within the family - self._reset_search_families() missing_gt_gcnv_variant = { **GCNV_VARIANT4, 'familyGuids': ['F000002_2_x'], @@ -1508,7 +1510,7 @@ def test_secondary_annotations_filter(self): annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, cached_variant_fields=[[ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': None}, {'selectedGeneId': 'ENSG00000277258'}, - ]], + ]], project_families=[{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000002_2_x', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids'][1:]]}], ) def test_in_silico_filter(self): @@ -1780,7 +1782,7 @@ def test_gene_variant_lookup(self): response_json = self._assert_expected_search( [variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], - annotations=annotations, freqs=freqs, locus=locus, project_families=None, + annotations=annotations, freqs=freqs, locus=locus, project_families=[], ) self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY}) self.assertEqual(response_json['totalSampleCounts'], { @@ -1799,7 +1801,7 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - annotations=annotations, freqs=freqs, locus=locus, project_families=None, + annotations=annotations, freqs=freqs, locus=locus, project_families=[], ) self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY, 'ENSG00000177000': mock.ANY}) @@ -1809,7 +1811,7 @@ def test_gene_variant_lookup(self): ) self._assert_expected_search( - [], request_body=request_body, response_search=response_search, project_families=None, + [], request_body=request_body, response_search=response_search, project_families=[], annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='homozygous_recessive', ) @@ -1821,7 +1823,7 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=None, + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=[], ) self.login_collaborator() From bbe9806f6fe286d67ad8e72cc601c0e79e354858 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 18:07:37 -0400 Subject: [PATCH 112/252] fix test --- clickhouse_search/search_tests.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ab5938279c..b570b8a137 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1584,13 +1584,14 @@ def test_sort(self): sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] self._assert_expected_search(sorted_variants, sort='gnomad', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self._assert_expected_search( - sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, - ) - self._assert_expected_search( - sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, - ) - self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) + # TODO test pagination! + # self._assert_expected_search( + # sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + # ) + # self._assert_expected_search( + # sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + # ) + # self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) self._assert_expected_search( [VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], From 312a718030ac309bee9358d07a3677752cf734bd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 18:17:08 -0400 Subject: [PATCH 113/252] test pagination --- clickhouse_search/search_tests.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index b570b8a137..314c4f3a7b 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -133,7 +133,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _execute_search(self, inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, sort='xpos', **search_kwargs): + def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, **search_kwargs): search_hash = random.randint(1000, 10000) self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) @@ -154,7 +154,8 @@ def _execute_search(self, inheritance_mode=None, inheritance_filter=None, qualit if check_login: check_login(url, request_data=request_data) - response = self.client.post(f'{url}?sort={sort}', content_type='application/json', data=json.dumps(request_data)) + query_string = '&'.join([f'{key}={value}' for key, value in {'sort': sort, **(query_params or {})}.items()]) + response = self.client.post(f'{url}?{query_string}', content_type='application/json', data=json.dumps(request_data)) return response, search_hash, search_body @@ -1584,14 +1585,12 @@ def test_sort(self): sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] self._assert_expected_search(sorted_variants, sort='gnomad', project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - # TODO test pagination! - # self._assert_expected_search( - # sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', page=2, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, - # ) - # self._assert_expected_search( - # sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', page=3, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, - # ) - # self._assert_expected_search(sorted_variants, sort='gnomad', load_all=True, num_results=4, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) + self._assert_expected_search( + sorted_variants, results_page=[VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1], sort='gnomad', query_params={'page': 2, 'per_page': 4}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + ) + self._assert_expected_search( + sorted_variants, results_page=[GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', query_params={'page': 3, 'per_page': 4}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + ) self._assert_expected_search( [VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, VARIANT2, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], From c6c4b2dd79df992651b9e88f8c60167729fb4208 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 10 Mar 2026 18:32:11 -0400 Subject: [PATCH 114/252] codacy --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 314c4f3a7b..254f30763e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -134,7 +134,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, **search_kwargs): - search_hash = random.randint(1000, 10000) + search_hash = random.randint(1000, 10000) # nosec self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) From 2da5234be3b4f3fb69bf5772fef9d82c3ff632d1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 09:53:05 -0400 Subject: [PATCH 115/252] move error searches --- clickhouse_search/search_tests.py | 13 ++++- seqr/views/apis/variant_search_api_tests.py | 56 --------------------- 2 files changed, 11 insertions(+), 58 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 254f30763e..f53b364f46 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -148,7 +148,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter search_body['inheritance']['filter'] = inheritance_filter request_data = { - **(request_body or {'projectFamilies': project_families or DEFAULT_PROJECT_FAMILIES}), 'search': search_body, + **(request_body or {'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families }), 'search': search_body, } if check_login: @@ -814,8 +814,17 @@ def test_variant_id_search(self): @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): + url = reverse(query_variants_handler, args=['not_a_hash']) + self.check_require_login(url) + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Invalid search hash: not_a_hash'}) + self.login_collaborator() + + self._assert_expected_search_error('Invalid search: no projects/ families specified', project_families=[]) + self._assert_expected_search_error( - 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, check_login=self.check_collaborator_login, + 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, ) self._assert_expected_search_error('Invalid variants: rs9876', locus={'rawVariantItems': 'rs9876'}) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index f5beb4e8e7..17c4ac5d44 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -398,26 +398,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.check_collaborator_login(url, request_data={'projectFamilies': PROJECT_FAMILIES}) url = reverse(query_variants_handler, args=[SEARCH_HASH]) - # Test invalid inputs - response = self.client.get(url) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.reason_phrase, 'Invalid search hash: {}'.format(SEARCH_HASH)) - mock_error_logger.assert_not_called() - - response = self.client.post(url, content_type='application/json', data=json.dumps({'search': SEARCH})) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.reason_phrase, 'Invalid search: no projects/ families specified') - mock_error_logger.assert_not_called() - - mock_get_variants.side_effect = InvalidSearchException('Invalid search') - mock_error_logger.reset_mock() - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Invalid search') - mock_error_logger.assert_not_called() - mock_get_variants.side_effect = _get_es_variants # Test new search @@ -434,7 +414,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, user=self.collaborator_user) - mock_error_logger.assert_not_called() # include project context info response = self.client.get('{}?loadProjectTagTypes=true'.format(url)) @@ -459,18 +438,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertDictEqual(response_json, expected_response) self._assert_expected_results_family_context(response_json) - # Test pagination - response = self.client.get('{}?page=3'.format(url)) - self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=3, num_results=100, user=self.collaborator_user) - mock_error_logger.assert_not_called() - - # Test sort - response = self.client.get('{}?sort=pathogenicity'.format(url)) - self.assertEqual(response.status_code, 200) - mock_get_variants.assert_called_with(results_model, sort='pathogenicity', page=1, num_results=100, user=self.collaborator_user) - mock_error_logger.assert_not_called() - # Test export export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) response = self.client.get(export_url) @@ -538,7 +505,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) mock_get_variants.assert_called_with(results_model, page=1, load_all=True, user=self.collaborator_user) - mock_error_logger.assert_not_called() # Test gene breakdown gene_counts = { @@ -583,7 +549,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) self.assertDictEqual(response_json, expected_search_response) self._assert_expected_results_context(response_json, has_pa_detail=False, rnaseq=False) - mock_error_logger.assert_not_called() # Test cross-project discovery for analyst users self.login_analyst_user() @@ -600,28 +565,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self._assert_expected_results_context(response_json) mock_get_variants.assert_called_with(results_model, sort='pathogenicity_hgmd', page=1, num_results=100, user=self.analyst_user) - mock_error_logger.assert_not_called() - # Test no results - VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete() - mock_get_variants.side_effect = _get_empty_es_variants - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': {**SEARCH, 'exclude': {'previousSearch': True}}, 'previousSearchHash': 'abc1234', - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json, { - 'searchedVariants': [], - 'search': { - 'search': {**SEARCH, 'exclude': { - 'previousSearch': True, - 'previousSearchHash': 'abc1234', - }}, - 'projectFamilies': PROJECT_FAMILIES, - 'totalResults': 0, - } - }) - mock_error_logger.assert_not_called() @mock.patch('seqr.views.apis.variant_search_api.query_variants') def test_query_all_projects_variants(self, mock_get_variants): From d63ee66ed9e949937399ff3f608ed664fdb9fb43 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 10:31:53 -0400 Subject: [PATCH 116/252] add test for search context --- clickhouse_search/search_tests.py | 56 ++++++++++++++++----- seqr/views/apis/variant_search_api_tests.py | 30 +---------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f53b364f46..ce96e27523 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -42,7 +42,7 @@ from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown @@ -204,6 +204,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun 'searchGeneBreakdown': {str(search_hash): gene_counts}, 'genesById': mock.ANY, }) + # TODO test genesById context return response.json() @@ -1788,23 +1789,17 @@ def test_gene_variant_lookup(self): variant4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997', 'numFamilies': 3} del variant4['familyGuids'] del variant4['genotypes'] - response_json = self._assert_expected_search( + self._assert_expected_search( [variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], annotations=annotations, freqs=freqs, locus=locus, project_families=[], ) - self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY}) - self.assertEqual(response_json['totalSampleCounts'], { - 'MITO': {'WES': 1}, - 'SNV_INDEL': {'WES': 7}, - 'SV': {'WES': 3, 'WGS': 3}, - }) freqs = {'callset': freqs['callset']} variant3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611', 'numFamilies': 4} del variant3['familyGuids'] del variant3['genotypes'] - response_json = self._assert_expected_search( + self._assert_expected_search( [variant3, variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[ {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, @@ -1812,7 +1807,6 @@ def test_gene_variant_lookup(self): ], annotations=annotations, freqs=freqs, locus=locus, project_families=[], ) - self.assertEqual(response_json['genesById'], {'ENSG00000097046': mock.ANY, 'ENSG00000177000': mock.ANY}) self._assert_expected_search_error( 'Compound heterozygous search is not supported when including external projects', @@ -1837,7 +1831,7 @@ def test_gene_variant_lookup(self): self.login_collaborator() project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': mock.ANY}] - response_json = self._assert_expected_search([ + self._assert_expected_search([ {**FAMILY_3_VARIANT, 'selectedMainTranscriptId': 'ENST00000497611'}, {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997'}, ], request_body=request_body, response_search=response_search, project_families=project_families, @@ -1847,8 +1841,6 @@ def test_gene_variant_lookup(self): ], annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', ) - self.assertEqual(response_json['locusListsByGuid'], {'LL00049_pid_genes_autosomal_do': mock.ANY}) - self.assertEqual(response_json['phenotypeGeneScores'], {'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}) locus['rawItems'] = 'ENSG00000171621' other_project_variant = {**PROJECT_4_COMP_HET_VARIANT, 'numFamilies': 1} @@ -1866,7 +1858,45 @@ def test_gene_variant_lookup(self): project_families=project_families, annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', ) + def test_search_context(self): + response_json = self._assert_expected_search( + [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, + GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + check_login=self.check_collaborator_login, + ) + # TODO test pa gene in saved variant test + # expected_pa_gene = {**expected_gene, 'locusListGuids': ['LL00049_pid_genes_autosomal_do'], 'panelAppDetail': mock.ANY} + # self.assertDictEqual( + # response_json['genesById']['ENSG00000227232']['panelAppDetail'], {LOCUS_LIST_GUID: {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}} + # ) + expected_gene = {k: mock.ANY for k in GENE_VARIANT_FIELDS} + expected_gene['locusListGuids'] = [] + self.assertDictEqual(response_json['genesById'], { + 'ENSG00000097046': expected_gene, + 'ENSG00000277258': expected_gene, + 'ENSG00000177000': expected_gene, + 'ENSG00000275023': expected_gene, + }) + self.assertDictEqual(response_json['locusListsByGuid'], {'LL00049_pid_genes_autosomal_do': {'intervals': [mock.ANY, mock.ANY]}}) + self.assertSetEqual( + set(response_json['locusListsByGuid']['LL00049_pid_genes_autosomal_do']['intervals'][0].keys()), + {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} + ) + self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {}) + self.assertDictEqual(response_json['phenotypeGeneScores'],{'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}) + self.assertDictEqual(response_json['rnaSeqData'], {}) + self.assertDictEqual(response_json['savedVariantsByGuid'], {}) + self.assertDictEqual(response_json['totalSampleCounts'], { + 'MITO': {'WES': 1}, + 'SNV_INDEL': {'WES': 7}, + 'SV': {'WES': 3, 'WGS': 3}, + }) + self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {}) + self.assertDictEqual(response_json['variantNotesByGuid'], {}) + self.assertDictEqual(response_json['variantTagsByGuid'], {}) + def test_cached_query_variants(self): + # TODO cache_key_prefix = f'search_results__{self.results_model.guid}' cached_variants = [VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1, VARIANT2] cache_result = self._format_cached_variants(cached_variants) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 17c4ac5d44..f0f00da61c 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -354,20 +354,7 @@ def _assert_expected_results_family_context(self, response_json, locus_list_deta self._assert_expected_rnaseq_response(response_json) def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True): - gene_fields = {'locusListGuids'} - gene_fields.update(GENE_VARIANT_FIELDS) - basic_gene_id = next(gene_id for gene_id in ['ENSG00000268903', 'ENSG00000233653'] if gene_id in response_json['genesById']) - self.assertSetEqual(set(response_json['genesById'][basic_gene_id].keys()), gene_fields) - if has_pa_detail: - gene_fields.add('panelAppDetail') - self.assertSetEqual(set(response_json['genesById']['ENSG00000227232'].keys()), gene_fields) - self.assertListEqual( - response_json['genesById']['ENSG00000227232']['locusListGuids'], [LOCUS_LIST_GUID] - ) - self.assertDictEqual( - response_json['genesById']['ENSG00000227232']['panelAppDetail'], {LOCUS_LIST_GUID: {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}} - ) - + # TODO used in single variant context locus_list_fields = {'intervals'} if locus_list_detail: locus_list_fields.update(LOCUS_LIST_FIELDS) @@ -400,21 +387,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro mock_get_variants.side_effect = _get_es_variants - # Test new search - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), {'F000001_1', 'F000002_2'}) - self._assert_expected_results_context(response_json) - - results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, user=self.collaborator_user) - # include project context info response = self.client.get('{}?loadProjectTagTypes=true'.format(url)) self.assertEqual(response.status_code, 200) From b22c64aaf7580f079421015d6f684b5e3aabc51e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 11:56:38 -0400 Subject: [PATCH 117/252] test context --- .../fixtures/clickhouse_search.json | 30 ++++++++ .../fixtures/clickhouse_transcripts.json | 29 ++++++++ clickhouse_search/search_tests.py | 60 ++++++++++------ clickhouse_search/test_utils.py | 72 +++++++++++++++++++ seqr/views/apis/variant_search_api_tests.py | 2 +- seqr/views/utils/orm_to_json_utils_tests.py | 4 ++ 6 files changed, 176 insertions(+), 21 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index cb9b677255..45ee737d2a 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -88,6 +88,18 @@ "sorted_motif_feature_consequences": [], "sorted_regulatory_feature_consequences": [] } +}, { + "model": "clickhouse_search.variantssnvindel", + "pk": 23, + "fields": { + "key": 23, + "sorted_transcript_consequences": [ + [null, 1, ["missense_variant"], false, null, "ENSG00000227232", false], + [null, 1, ["intron_variant"], false, null, "ENSG00000268903", false] + ], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } }, { "model": "clickhouse_search.keylookupsnvindel", "pk": "1-10439-AC-A", @@ -860,6 +872,24 @@ ["NA21654", 0, 0, 0, 24] ] } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 23, + "fields": { + "key": 23, + "project_guid": "R0001_1kg", + "family_guid": "F000001_1", + "sample_type": "WES", + "xpos": 21003343353, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [2], + "filters": [], + "sign": 1, + "calls": [ + ["NA19675_1", 1, 0, 0.16, 45] + ] + } }, { "model": "clickhouse_search.variantsgrch37snvindel", "pk": 10, diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json index 8955cd937a..d40b40e856 100644 --- a/clickhouse_search/fixtures/clickhouse_transcripts.json +++ b/clickhouse_search/fixtures/clickhouse_transcripts.json @@ -235,5 +235,34 @@ ] ] } +}, { + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 23, + "fields": { + "key": 23, + "variant_id": "21-3343353-GAGA-G", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "transcripts": [ + [ + [null], "T/I", "protein_coding", 1, "aCc/aTc", ["missense_variant"], [1, 12], "ENSG00000227232", + "ENST00000257261.10:c.131C>T", "ENSP00000257261.6:p.Thr44Ile", [null, null], [null, []], "missense_variant", + null, null, "NM_001281501.1", [false], "ENST00000257261", 0, [null, null, null, [ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null + ], null] + ], + [ + [null], null, "protein_coding", null, null, ["intron_variant"], [null, null], "ENSG00000268903", + "ENST00000426137.1:c.-64+100G>A", null, [1, 5], [null, []], "intron_variant", null, null, null, [false], + "ENST00000426137", 0, [null, null, null, [ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null + ], null] + ] + ] + } } ] \ No newline at end of file diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ce96e27523..1b00ee2b35 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -35,14 +35,15 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ + FAMILY_1_VARIANT, DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ + SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown @@ -159,7 +160,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', response_search=None, format_cache_key=None, project_families=None, is_37=False, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, format_cache_key=None, project_families=None, additional_response=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -183,6 +184,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun 'variantNotesByGuid': mock.ANY, 'variantTagsByGuid': mock.ANY, 'totalSampleCounts': mock.ANY, + **(additional_response or {}), }) if not is_37: expected_response['omimIntervals'] = mock.ANY @@ -1861,39 +1863,57 @@ def test_gene_variant_lookup(self): def test_search_context(self): response_json = self._assert_expected_search( [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, - GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - check_login=self.check_collaborator_login, + GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + project_families=[{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000001_1', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids']]}], + check_login=self.check_collaborator_login, additional_response={'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}}, ) - # TODO test pa gene in saved variant test - # expected_pa_gene = {**expected_gene, 'locusListGuids': ['LL00049_pid_genes_autosomal_do'], 'panelAppDetail': mock.ANY} - # self.assertDictEqual( - # response_json['genesById']['ENSG00000227232']['panelAppDetail'], {LOCUS_LIST_GUID: {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}} - # ) + expected_gene = {k: mock.ANY for k in GENE_VARIANT_FIELDS} expected_gene['locusListGuids'] = [] + expected_pa_gene = {**expected_gene, 'locusListGuids': ['LL00049_pid_genes_autosomal_do'], 'panelAppDetail': mock.ANY} self.assertDictEqual(response_json['genesById'], { - 'ENSG00000097046': expected_gene, - 'ENSG00000277258': expected_gene, - 'ENSG00000177000': expected_gene, - 'ENSG00000275023': expected_gene, + **{gene_id: expected_gene for gene_id in [ + 'ENSG00000097046', 'ENSG00000277258', 'ENSG00000177000', 'ENSG00000275023', 'ENSG00000268903', + ]}, + 'ENSG00000227232': expected_pa_gene, }) + self.assertDictEqual( + response_json['genesById']['ENSG00000227232']['panelAppDetail'], + {'LL00049_pid_genes_autosomal_do': {'confidence': '3', 'moi': 'BIALLELIC, autosomal or pseudoautosomal'}}, + ) self.assertDictEqual(response_json['locusListsByGuid'], {'LL00049_pid_genes_autosomal_do': {'intervals': [mock.ANY, mock.ANY]}}) self.assertSetEqual( set(response_json['locusListsByGuid']['LL00049_pid_genes_autosomal_do']['intervals'][0].keys()), {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) - self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {}) - self.assertDictEqual(response_json['phenotypeGeneScores'],{'I000004_hg00731': mock.ANY, 'I000005_hg00732': mock.ANY}) - self.assertDictEqual(response_json['rnaSeqData'], {}) - self.assertDictEqual(response_json['savedVariantsByGuid'], {}) + self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}) + self.assertDictEqual(response_json['phenotypeGeneScores'],{ + 'I000001_na19675': {'ENSG00000268903': mock.ANY}, + 'I000002_na19678': {'ENSG00000268903': mock.ANY}, + 'I000004_hg00731': {'ENSG00000177000': mock.ANY}, + 'I000005_hg00732': {'ENSG00000177000': mock.ANY}, + }) + self.assertDictEqual(response_json['rnaSeqData'], { + 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, + }) + self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { + **{k: mock.ANY for k in [*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId']}, + 'mmeSubmissions': [{'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}], + }}) self.assertDictEqual(response_json['totalSampleCounts'], { 'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}, }) - self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {}) + expected_functional_tag = {k: mock.ANY for k in FUNCTIONAL_FIELDS} + self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {guid: expected_functional_tag for guid in [ + 'VFD0000023_1248367227_r0390_10', 'VFD0000024_1248367227_r0390_10', 'VFD0000025_1248367227_r0390_10', 'VFD0000026_1248367227_r0390_10', + ]}) self.assertDictEqual(response_json['variantNotesByGuid'], {}) - self.assertDictEqual(response_json['variantTagsByGuid'], {}) + expected_tag = {k: mock.ANY for k in TAG_FIELDS} + self.assertDictEqual(response_json['variantTagsByGuid'], { + 'VT1708633_2103343353_r0390_100': expected_tag, 'VT1726961_2103343353_r0390_100': expected_tag, + }) def test_cached_query_variants(self): # TODO diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index ee8ad18f27..8fc1563e39 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -383,6 +383,61 @@ 'CAID': 'CA520798130', } +FAMILY_1_VARIANT = { + 'key': 23, + 'variantId': '21-3343353-GAGA-G', + 'chrom': '21', + 'pos': 3343353, + 'ref': 'GAGA', + 'alt': 'G', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'xpos': 21003343353, + 'rsid': None, + 'familyGuids': ['F000001_1'], + 'genotypes': {}, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 1, 'hom': 0, 'ac_wes': 1, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0}, + 'seqr_affected': {'ac': 1, 'hom': 0}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + }, + 'predictions': { + 'cadd': None, + 'eigen': None, + 'fathmm': None, + 'gnomad_noncoding': None, + 'mpc': None, + 'mut_pred': None, + 'primate_ai': None, + 'splice_ai': None, + 'splice_ai_consequence': None, + 'vest': None, + 'mut_taster': None, + 'polyphen': None, + 'revel': None, + 'sift': None, + 'absplice': None, + 'pext': None, + 'promoter_ai': None, + }, + 'transcripts': { + 'ENSG00000227232': [{'alphamissense': {'pathogenicity': None}, 'aminoAcids': 'T/I','biotype': 'protein_coding', 'canonical': 1, 'codons': 'aCc/aTc', 'consequenceTerms': ['missense_variant'], 'exon': {'index': 1, 'total': 12}, 'geneId': 'ENSG00000227232', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'intron': None, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'majorConsequence': 'missense_variant', 'manePlusClinical': None, 'maneSelect': None, 'refseqTranscriptId': 'NM_001281501.1', 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'transcriptId': 'ENST00000257261', 'transcriptRank': 0, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}}], + 'ENSG00000268903': [{'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000268903', 'hgvsc': 'ENST00000426137.1:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000426137', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 5}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}], + }, + 'mainTranscriptId': 'ENST00000257261', + 'selectedMainTranscriptId': None, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'CAID': None, +} + GRCH37_VARIANT = { 'key': 11, 'variantId': '7-143270172-A-G', @@ -1331,6 +1386,23 @@ 'geneId': 'ENSG00000171621', 'isManeSelect': False, }], +23: [{ + 'alphamissensePathogenicity': None, + 'canonical': 1, + 'consequenceTerms': ['missense_variant'], + 'extendedIntronicSpliceRegionVariant': False, + 'fiveutrConsequence': None, + 'geneId': 'ENSG00000227232', + 'isManeSelect': False, +}, { + 'alphamissensePathogenicity': None, + 'canonical': 1, + 'consequenceTerms': ['intron_variant'], + 'extendedIntronicSpliceRegionVariant': False, + 'fiveutrConsequence': None, + 'geneId': 'ENSG00000268903', + 'isManeSelect': False, +}], } def format_cached_variant(variant): diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index f0f00da61c..0b494e6260 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -392,7 +392,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertEqual(response.status_code, 200) response_json = response.json() expected_search_response = {'projectsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['projectsByGuid']} - expected_search_response.update(EXPECTED_SEARCH_RESPONSE) + expected_search_responsernaSeqData.update(EXPECTED_SEARCH_RESPONSE) self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) self.assertDictEqual(response_json, expected_search_response) self._assert_expected_results_context(response_json) diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index 1b553d3f30..f5a68c23a0 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -142,6 +142,10 @@ def test_json_for_saved_variants_with_tags(self): self.assertListEqual(json['variantTagsByGuid'][tag_guid]['variantGuids'], [variant_guid_1]) for tag_guid in v2_tag_guids: self.assertListEqual(json['variantTagsByGuid'][tag_guid]['variantGuids'], [variant_guid_2]) + self.assertDictEqual( + json['variantTagsByGuid']['VT1726985_2103343353_r0390_100']['structuredMetadata'], + {'4': {'date': '2023-11-15', 'name': 'de Novo'}, 'support': {'date': '2023-11-15', 'name': 'High in Silico Scores'}}, + ) self.assertSetEqual(set(json['variantNotesByGuid'].keys()), set(v2_note_guids)) self.assertSetEqual(set(json['variantNotesByGuid'][v2_note_guids[0]].keys()), VARIANT_NOTE_FIELDS) From 33c98161c3d7f181b5177f0b21383795798bca64 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 12:26:02 -0400 Subject: [PATCH 118/252] test project families context --- clickhouse_search/search_tests.py | 69 +++++++++++++++++++-- seqr/views/apis/variant_search_api_tests.py | 42 ------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 1b00ee2b35..cf297b826f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -43,7 +43,8 @@ from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ - SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS + SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ + FAMILY_NOTE_FIELDS from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown @@ -1861,11 +1862,15 @@ def test_gene_variant_lookup(self): ) def test_search_context(self): + expected_results = [ + VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, + GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, + ] + project_families = [{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000001_1', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids']]}] + additional_response = {'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}} response_json = self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, - GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - project_families=[{**DEFAULT_PROJECT_FAMILIES[0], 'familyGuids': ['F000001_1', *DEFAULT_PROJECT_FAMILIES[0]['familyGuids']]}], - check_login=self.check_collaborator_login, additional_response={'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}}, + expected_results, project_families=project_families, additional_response=additional_response, + check_login=self.check_collaborator_login, ) expected_gene = {k: mock.ANY for k in GENE_VARIANT_FIELDS} @@ -1887,6 +1892,7 @@ def test_search_context(self): {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}) + self.assertDictEqual(response_json['omimIntervals'], {}) # TODO fix! self.assertDictEqual(response_json['phenotypeGeneScores'],{ 'I000001_na19675': {'ENSG00000268903': mock.ANY}, 'I000002_na19678': {'ENSG00000268903': mock.ANY}, @@ -1896,6 +1902,26 @@ def test_search_context(self): self.assertDictEqual(response_json['rnaSeqData'], { 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, }) + self.assertDictEqual( + response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], + {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, + 'tissueType': 'M', 'zScore': 7.08} + ) + self.assertListEqual( + sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), + [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, + 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, + {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, + 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, + {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, + 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, + 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, + 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] + ) self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { **{k: mock.ANY for k in [*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId']}, 'mmeSubmissions': [{'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}], @@ -1915,6 +1941,39 @@ def test_search_context(self): 'VT1708633_2103343353_r0390_100': expected_tag, 'VT1726961_2103343353_r0390_100': expected_tag, }) + # test project context info + self._assert_expected_search( + expected_results, project_families=project_families, query_params={'loadProjectTagTypes': 'true'}, additional_response={ + **additional_response, + 'projectsByGuid': {'R0001_1kg': {field: mock.ANY for field in { + 'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes', + }}}, + }, + ) + + # test family context info + expected_family = {field: mock.ANY for field in [*FAMILY_FIELDS, 'individualGuids']} + expected_individual = {field: mock.ANY for field in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} + expected_igv = {field: mock.ANY for field in IGV_SAMPLE_FIELDS} + expected_note = {field: mock.ANY for field in FAMILY_NOTE_FIELDS} + self._assert_expected_search( + expected_results, project_families=project_families, query_params={'loadFamilyContext': 'true'}, + additional_response={ + **additional_response, + 'familiesByGuid': { + 'F000001_1': {**expected_family, 'tpmGenes': ['ENSG00000227232']}, + 'F000002_2': expected_family, + 'F000003_3': expected_family, + }, + 'individualsByGuid': {guid: expected_individual for guid in [ + 'I000001_na19675', 'I000002_na19678', 'I000003_na19679', 'I000004_hg00731', 'I000005_hg00732', + 'I000006_hg00733', 'I000007_na20870', + ]}, + 'igvSamplesByGuid': {guid: expected_igv for guid in ['S000145_na19675', 'S000146_na20870', 'S000147_na20870']}, + 'familyNotesByGuid': {guid: expected_note for guid in ['FAN000001_1', 'FAN000001_2', 'FAN000001_3']}, + }, + ) + def test_cached_query_variants(self): # TODO cache_key_prefix = f'search_results__{self.results_model.guid}' diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 0b494e6260..a44d9e5dd4 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -306,28 +306,6 @@ def _assert_expected_search_context(self, response_json): 'analysedBy': [{'createdBy': 'Test No Access User', 'dataType': 'SNP', 'lastModifiedDate': '2022-07-22T19:27:08.563+00:00'}], }) - def _assert_expected_rnaseq_response(self, response_json): - self.assertDictEqual( - response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], - {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, - 'tissueType': 'M', 'zScore': 7.08} - ) - self.assertListEqual( - sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), - [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, - 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, - {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, - 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, - {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, - 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, - 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] - ) - def _assert_expected_results_family_context(self, response_json, locus_list_detail=False): self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) @@ -368,15 +346,6 @@ def _assert_expected_results_context(self, response_json, has_pa_detail=True, lo {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) - self.assertSetEqual(set(next(iter(response_json['variantTagsByGuid'].values())).keys()), TAG_FIELDS) - if response_json['variantNotesByGuid']: - self.assertSetEqual(set(next(iter(response_json['variantNotesByGuid'].values())).keys()), VARIANT_NOTE_FIELDS) - if response_json['variantFunctionalDataByGuid']: - self.assertSetEqual(set(next(iter(response_json['variantFunctionalDataByGuid'].values())).keys()), FUNCTIONAL_FIELDS) - - if rnaseq: - self._assert_expected_rnaseq_response(response_json) - @mock.patch('seqr.utils.middleware.logger.error') @mock.patch('seqr.views.apis.variant_search_api.get_variant_query_gene_counts') @mock.patch('seqr.views.apis.variant_search_api.query_variants') @@ -387,17 +356,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro mock_get_variants.side_effect = _get_es_variants - # include project context info - response = self.client.get('{}?loadProjectTagTypes=true'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_response = {'projectsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['projectsByGuid']} - expected_search_responsernaSeqData.update(EXPECTED_SEARCH_RESPONSE) - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_context(response_json) - self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) - # include family context info response = self.client.get('{}?loadFamilyContext=true'.format(url)) self.assertEqual(response.status_code, 200) From 0f1caaf0770a42dd3ccf372cf7ae6cdf603beec7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 12:34:49 -0400 Subject: [PATCH 119/252] omim interval context --- clickhouse_search/search_tests.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index cf297b826f..99c4bde6a8 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1892,7 +1892,7 @@ def test_search_context(self): {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}) - self.assertDictEqual(response_json['omimIntervals'], {}) # TODO fix! + self.assertDictEqual(response_json['omimIntervals'], {}) self.assertDictEqual(response_json['phenotypeGeneScores'],{ 'I000001_na19675': {'ENSG00000268903': mock.ANY}, 'I000002_na19678': {'ENSG00000268903': mock.ANY}, @@ -1974,6 +1974,15 @@ def test_search_context(self): }, ) + self.login_manager() + response_json = self._assert_expected_search( + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, + project_families=SV_PROJECT_FAMILIES, + ) + self.assertDictEqual(response_json['omimIntervals'], { + '3': {'chrom': '1', 'end': 249055991, 'mimNumber': 600315, 'phenotypeDescription': '?Immunodeficiency 16', 'phenotypeInheritance': 'Autosomal recessive', 'phenotypeMimNumber': 615120, 'start': 249044482}, + }) + def test_cached_query_variants(self): # TODO cache_key_prefix = f'search_results__{self.results_model.guid}' From 7f597e3d5ba91377d78327bc5faee259109df549 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 13:27:35 -0400 Subject: [PATCH 120/252] test export --- clickhouse_search/search_tests.py | 41 +++++++---- clickhouse_search/test_utils.py | 65 +++++++++++++++++ seqr/views/apis/variant_search_api.py | 4 +- seqr/views/apis/variant_search_api_tests.py | 80 --------------------- 4 files changed, 94 insertions(+), 96 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 99c4bde6a8..805e4b17f7 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -35,8 +35,8 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - FAMILY_1_VARIANT, DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ - format_cached_variant + FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, \ + SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException @@ -45,7 +45,7 @@ from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ FAMILY_NOTE_FIELDS -from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown +from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler class ClickhouseSearchTestCase(AnvilAuthenticationTestCase): @@ -161,7 +161,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, format_cache_key=None, project_families=None, additional_response=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, format_cache_key=None, project_families=None, additional_response=None, export_data=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -200,17 +200,29 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.mock_redis.set.assert_not_called() if gene_counts: - gene_breakdown_url = reverse(get_variant_gene_breakdown, args=[search_hash]) - gene_breakdown_response = self.client.get(gene_breakdown_url) - self.assertEqual(gene_breakdown_response.status_code, 200) - self.assertDictEqual(gene_breakdown_response.json(), { - 'searchGeneBreakdown': {str(search_hash): gene_counts}, - 'genesById': mock.ANY, - }) - # TODO test genesById context + self._assert_expected_gene_counts(search_hash, gene_counts) + + if export_data: + self._assert_expected_export_data(search_hash, export_data) return response.json() + def _assert_expected_gene_counts(self, search_hash, gene_counts): + url = reverse(get_variant_gene_breakdown, args=[search_hash]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'searchGeneBreakdown': {str(search_hash): gene_counts}, + 'genesById': mock.ANY, + }) + # TODO test genesById context + + def _assert_expected_export_data(self, search_hash, export_data): + url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], export_data) + def _assert_expected_search_error(self, error, **kwargs): response, _, _ = self._execute_search(**kwargs) self.assertEqual(response.status_code, 400) @@ -756,11 +768,12 @@ def test_quality_filter(self): [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, is_37=True, ) + @mock.patch('seqr.views.apis.variant_search_api.MAX_FAMILIES_PER_ROW', 1) def test_location_search(self): self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4], **LOCATION_SEARCH, cached_variant_fields=[ {'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'} - ], check_login=self.check_collaborator_login, + ], check_login=self.check_collaborator_login, export_data=SPLIT_FAMILY_EXPORT_DATA, ) sv_locus = {'rawItems': 'ENSG00000275023, ENSG00000171621'} @@ -1870,7 +1883,7 @@ def test_search_context(self): additional_response = {'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}} response_json = self._assert_expected_search( expected_results, project_families=project_families, additional_response=additional_response, - check_login=self.check_collaborator_login, + check_login=self.check_collaborator_login, export_data=EXPORT_DATA, ) expected_gene = {k: mock.ANY for k in GENE_VARIANT_FIELDS} diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 8fc1563e39..9f1f77ee4e 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1466,6 +1466,71 @@ def format_cached_variant(variant): 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, } +EXPORT_DATA = [ + ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', + 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', + 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', + 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', + 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3', + 'sample_4', 'num_alt_alleles_4', 'filters_4', 'gq_4', 'ab_4'], + ['1', '10439', 'AC', 'A', '', '', '8', '', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', + 'rs112766696', '', '', 'Likely_pathogenic', '', '2', '', '', '', '', '', 'HG00731', '1', '', '99', '0.5', 'HG00732', + '0', '', '0', '0', 'HG00733', '0', '', '20', '0', '', '', '', '', ''], + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', '0.00288997', '0.246152', + '25.9', '0.197', '2.001', '0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', + 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1', 'HG00732', + '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0', '', '', '', '', ''], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '3', '', + '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741', + 'NA20870', '1', '', '99', '0.67857'], + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '', '0.00026519', '0', '0', '29.9', + '0.526', '9.491', '0', '', '', '', '', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', + 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', + '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', '', '', '', '', ''], + ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', + '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], + ['16', '29809156', '', '', 'ENSG00000103495', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '1', '', '', + '', '', '', '', '', ''], + ['17', '38717327', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '2', '', '', '', '', '', 'HG00731', '2', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '0', '', '', '', + '', '', '', '', ''], + ['17', '38721781', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '1', '', '', '', + '', '', '', '', ''], + ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '', '0', '0', '0', '', '', '', '', '', '', + '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', '', '', '', '', '', '', '', + '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', + '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', + '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', + '', 'ENST00000361381.2:c.838A>G', 'ENSP00000354961.2:p.Thr280Ala', '', '', '2', '', '', '', '', '', 'HG00731', '1', + '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + ['M', '14783', 'T', 'C', 'ENSG00000198727', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', '', + 'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '', + '', '', 'HG00731', '2', 'artifact_prone_site'], +] + +SPLIT_FAMILY_EXPORT_DATA = [ + ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', + 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', + 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', + 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', + 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', + '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', '', '', '', '', ''], + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '3', '', '', 'HG00733', + '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857', '', '', '', '', ''], + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '', '0.00026519', '0', '0', '29.9', + '0.526', '9.491', '0', '', '', '', '', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', 'HG00731', '1', + 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', + '0', 'VQSRTrancheSNP99.95to100.00', '30', '0'], +] + DEFAULT_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2', 'F000003_3', 'F000005_5']}] SINGLE_FAMILY_PROJECT_FAMILIES = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000002_2']}] MULTI_PROJECT_PROJECT_FAMILIES = [ diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index fd1f9bdb38..4d6ba59ae1 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -306,7 +306,7 @@ def export_variants_handler(request, search_hash): num_split = ceil(len(variant['familyGuids']) / MAX_FAMILIES_PER_ROW) gens_per_row = ceil(len(variant['genotypes']) / num_split) - gen_keys = list(variant['genotypes'].keys()) + gen_keys = sorted(variant['genotypes'].keys()) for i in range(num_split): split_var = deepcopy(variant) split_var['familyGuids'] = variant['familyGuids'][i*MAX_FAMILIES_PER_ROW:(i+1)*MAX_FAMILIES_PER_ROW] @@ -334,7 +334,7 @@ def export_variants_handler(request, search_hash): row += [_get_field_value(family_tags, config) for config in VARIANT_FAMILY_EXPORT_DATA] row += ['' for i in range(len(VARIANT_FAMILY_EXPORT_DATA) * (max_families_per_variant - len(variant['familyGuids'])))] - genotypes = list(variant['genotypes'].values()) + genotypes = [genotype for _, genotype in sorted(variant['genotypes'].items())] for genotype in genotypes: row += [_get_field_value(genotype, config, variant=variant) for config in VARIANT_SAMPLE_DATA] row += ['' for i in range(len(VARIANT_SAMPLE_DATA) * (max_samples_per_variant - len(genotypes)))] diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index a44d9e5dd4..394632a67a 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -356,86 +356,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro mock_get_variants.side_effect = _get_es_variants - # include family context info - response = self.client.get('{}?loadFamilyContext=true'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_response = { - **EXPECTED_SEARCH_RESPONSE, - **EXPECTED_SEARCH_FAMILY_CONTEXT, - } - self.assertSetEqual(set(response_json.keys()), set(expected_response.keys())) - self.assertDictEqual(response_json, expected_response) - self._assert_expected_results_family_context(response_json) - - # Test export - export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) - response = self.client.get(export_url) - self.assertEqual(response.status_code, 200) - expected_content = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', - 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', - 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', - 'tags_1', 'notes_1', 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', - 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', - '', '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '2', '', '', 'NA19675', '1', 'VQSRTrancheSNP99.95to100.00', '46.0', - '0.702127659574', 'NA19679', '0', 'VQSRTrancheSNP99.95to100.00', '99.0', '0.0', '', '', '', '', ''], - ['3', '835', 'AAAG', 'A', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '1', '', '', '', '', '', 'NA19679', '0', 'artifact_prone_site', '99.0', '0.0', '', '', '', '', '', '', '', '', '', ''], - ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', - '', '', '', '', '', '', '', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', - '0.00288997', '0.246152', '25.9', '0.197', - '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', - 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1.0', - 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0.0', - '0.362681', '28.754', '', '1.378', '0.01', '', '', '', - '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741'], - ] - self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) - - # test export with max families - with mock.patch('seqr.views.apis.variant_search_api.MAX_FAMILIES_PER_ROW', 1): - response = self.client.get(export_url) - self.assertEqual(response.status_code, 200) - expected_content = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', - 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', - 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', - 'tags_1', 'notes_1', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', - 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', '', - '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', 'NA19675', '1', 'VQSRTrancheSNP99.95to100.00', '46.0', '0.702127659574', - '', '', '', '', '', '', '', '', '', '',], - ['21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '0.13', '', '0.007', '', '', '', '', '', '', '', '', '', '', - '', 'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser', '', '', '2', '', '', - 'NA19679', '0', 'VQSRTrancheSNP99.95to100.00', '99.0', '0.0', '', '', '', '', '', '', '', '', '', '',], - ['3', '835', 'AAAG', 'A', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '1', '', '', 'NA19679', '0', 'artifact_prone_site', '99.0', '0.0', '', '', '', '', '', '', '', '', '', '',], - ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', - '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', - '0.00288997', '0.246152', '25.9', '0.197', - '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', - 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', 'HG00731', '2', '', '99', '1.0', - 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0.0', - '0.362681', '28.754', '', '1.378', '0.01', '', '', - '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', - '0.40741'], - ] - self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) - - mock_get_variants.assert_called_with(results_model, page=1, load_all=True, user=self.collaborator_user) - # Test gene breakdown gene_counts = { 'ENSG00000227232': {'total': 2, 'families': {'F000001_1': 2, 'F000002_2': 1}}, From 8b84a2fddd7c412e04278700bcd689953247fee9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 13:40:49 -0400 Subject: [PATCH 121/252] test tag export --- clickhouse_search/search_tests.py | 11 +++++++++-- clickhouse_search/test_utils.py | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 805e4b17f7..b5ebfdf04e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -38,7 +38,7 @@ FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, \ SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults +from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets @@ -1875,6 +1875,10 @@ def test_gene_variant_lookup(self): ) def test_search_context(self): + sv = SavedVariant.objects.get(guid='SV0000001_2103343353_r0390_100') + sv.saved_variant_json['genomeVersion'] = '38' + sv.save() + expected_results = [ VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, @@ -1936,7 +1940,10 @@ def test_search_context(self): 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] ) self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { - **{k: mock.ANY for k in [*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId']}, + **{k: mock.ANY for k in [ + *SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId', 'screenRegionType', + 'sortedMotifFeatureConsequences', 'sortedRegulatoryFeatureConsequences', + ]}, 'mmeSubmissions': [{'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}], }}) self.assertDictEqual(response_json['totalSampleCounts'], { diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 9f1f77ee4e..96cc672c7d 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1500,8 +1500,9 @@ def format_cached_variant(variant): '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '1', '', '', '', '', '', '', '', ''], ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '', '0', '0', '0', '', '', '', '', '', '', - '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', '', '', '', '', '', '', '', - '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], + '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', + 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '', '', '', '', '', '', ''], ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], From e48a41eec33fbbccdd494b19db46bcc007d4f458 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 13:51:34 -0400 Subject: [PATCH 122/252] test updates --- clickhouse_search/search_tests.py | 9 +++-- seqr/views/apis/variant_search_api_tests.py | 37 --------------------- 2 files changed, 6 insertions(+), 40 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index b5ebfdf04e..c61f18ff89 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -44,7 +44,7 @@ from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ - FAMILY_NOTE_FIELDS + FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler @@ -211,11 +211,13 @@ def _assert_expected_gene_counts(self, search_hash, gene_counts): url = reverse(get_variant_gene_breakdown, args=[search_hash]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), { + response_json = response.json() + self.assertDictEqual(response_json, { 'searchGeneBreakdown': {str(search_hash): gene_counts}, 'genesById': mock.ANY, }) - # TODO test genesById context + for gene in response_json['genesById'].values(): + self.assertSetEqual(set(gene.keys()), GENE_VARIANT_DISPLAY_FIELDS) def _assert_expected_export_data(self, search_hash, export_data): url = reverse(export_variants_handler, args=[search_hash]) @@ -2077,6 +2079,7 @@ def test_trigger_delete_project(self): 5: {'ac_wes': 1, 'ac_wgs': 1, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 2, 'hom_affected': 0}, 6: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, 22: {'ac_wes': 0, 'ac_wgs': 3, 'hom_wes': 0, 'hom_wgs': 1, 'ac_affected': 3, 'hom_affected': 1}, + 23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, }) project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 394632a67a..6759487789 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -363,43 +363,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro } mock_get_gene_counts.return_value = gene_counts - gene_breakdown_url = reverse(get_variant_gene_breakdown, args=[SEARCH_HASH]) - response = self.client.get(gene_breakdown_url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'searchGeneBreakdown', 'genesById'}) - self.assertDictEqual(response_json['searchGeneBreakdown'], {SEARCH_HASH: gene_counts}) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000227232', 'ENSG00000268903'}) - self.assertSetEqual(set(response_json['genesById']['ENSG00000227232'].keys()), GENE_VARIANT_DISPLAY_FIELDS) - - # Test compound hets - mock_get_variants.side_effect = _get_compound_het_es_variants - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_response = deepcopy(EXPECTED_SEARCH_RESPONSE) - expected_search_response.update({ - 'searchedVariants': COMP_HET_VARAINTS, - 'savedVariantsByGuid': {k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, - 'genesById': {'ENSG00000233653': EXPECTED_GENE}, - 'variantTagsByGuid': { - 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726985_2103343353_r0390_100': expected_aip_tag, - }, - 'variantFunctionalDataByGuid': {}, - 'phenotypeGeneScores': {}, - 'rnaSeqData': {}, - 'mmeSubmissionsByGuid': {}, - }) - expected_search_response['search']['totalResults'] = 1 - del expected_search_response['familiesByGuid'] - expected_search_response.pop('transcriptsById', None) - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_context(response_json, has_pa_detail=False, rnaseq=False) - # Test cross-project discovery for analyst users self.login_analyst_user() mock_get_variants.side_effect = _get_es_variants From 53508b1d27e0989e4b7a3eec8a8b6d301b681440 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 14:35:10 -0400 Subject: [PATCH 123/252] caceh mock clean up --- clickhouse_search/search_tests.py | 32 +++++++++++---------- seqr/views/apis/variant_search_api_tests.py | 12 -------- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index c61f18ff89..6aa722f82e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -106,9 +106,10 @@ class ClickhouseSearchTests(ClickhouseSearchTestCase): fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): + self.MOCK_CACHE = {} patcher = mock.patch('seqr.utils.redis_utils.redis.StrictRedis') self.mock_redis = patcher.start().return_value - self.mock_redis.get.return_value = None + self.mock_redis.get.side_effect = self.MOCK_CACHE.get self.addCleanup(patcher.stop) patcher = mock.patch('seqr.models.VariantSearchResults._compute_guid') @@ -126,8 +127,8 @@ def setUp(self): super().setUp() - def set_cache(self, cached): - self.mock_redis.get.return_value = json.dumps(cached) + def set_cache(self, cache_key, cached): + self.MOCK_CACHE[cache_key] = json.dumps(cached) def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' @@ -226,9 +227,10 @@ def _assert_expected_export_data(self, search_hash, export_data): self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], export_data) def _assert_expected_search_error(self, error, **kwargs): - response, _, _ = self._execute_search(**kwargs) + response, search_hash, _ = self._execute_search(**kwargs) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': error}) + return search_hash def _assert_expected_variants(self, variants, expected_results, cache_key=None, results_page=None, format_cached_variants=None, sort='xpos', **kwargs): encoded_variants = self._encode_variants(variants) @@ -643,9 +645,9 @@ def test_inheritance_filter(self): def test_exclude_previous_search_results(self): self.mock_results_guid.return_value = 'VRS00079516' VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') - self.mock_redis.get.side_effect = [None, None, None, json.dumps({'all_results': [ + self.set_cache('search_results__abc1234__gnomad', {'all_results': [ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], - ]})] + ]}) self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] exclude = {'previousSearch': True, 'previousSearchHash': 'abc1234'} @@ -659,9 +661,9 @@ def test_exclude_previous_search_results(self): ], check_login=self.check_collaborator_login, ) - self.mock_redis.get.side_effect = [None, None, json.dumps({'all_results': [ + self.set_cache('search_results__abc1234__gnomad', {'all_results': [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, - ]})] + ]}) self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, @@ -842,10 +844,16 @@ def test_invalid_search(self): self._assert_expected_search_error('Invalid search: no projects/ families specified', project_families=[]) - self._assert_expected_search_error( + search_hash = self._assert_expected_search_error( 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, ) + self.set_cache(f'search_results__VRS{search_hash:07d}__xpos', {'total_results': 20000}) + export_url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(export_url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Unable to export more than 1000 variants (20000 requested)'}) + self._assert_expected_search_error('Invalid variants: rs9876', locus={'rawVariantItems': 'rs9876'}) self._assert_expected_search_error( @@ -934,12 +942,6 @@ def test_invalid_search(self): project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) - # TODO test once have export tests in this file - self.set_cache({'total_results': 20000}) - with self.assertRaises(InvalidSearchException) as cm: - self._assert_expected_search([], page=1, num_results=2, load_all=True, project_families=SINGLE_FAMILY_PROJECT_FAMILIES) - self._assert_expected_search_error('Unable to export more than 1000 variants (20000 requested)') - @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): # TODO diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 6759487789..7724f5ee5c 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -354,15 +354,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.check_collaborator_login(url, request_data={'projectFamilies': PROJECT_FAMILIES}) url = reverse(query_variants_handler, args=[SEARCH_HASH]) - mock_get_variants.side_effect = _get_es_variants - - # Test gene breakdown - gene_counts = { - 'ENSG00000227232': {'total': 2, 'families': {'F000001_1': 2, 'F000002_2': 1}}, - 'ENSG00000268903': {'total': 1, 'families': {'F000002_2': 1}} - } - mock_get_gene_counts.return_value = gene_counts - # Test cross-project discovery for analyst users self.login_analyst_user() mock_get_variants.side_effect = _get_es_variants @@ -377,9 +368,6 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro self.assertDictEqual(response_json, expected_search_results) self._assert_expected_results_context(response_json) - mock_get_variants.assert_called_with(results_model, sort='pathogenicity_hgmd', page=1, num_results=100, user=self.analyst_user) - - @mock.patch('seqr.views.apis.variant_search_api.query_variants') def test_query_all_projects_variants(self, mock_get_variants): url = reverse(query_variants_handler, args=[SEARCH_HASH]) From 07cda35c331eff3acf219ad6788d526090d6ac34 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 11 Mar 2026 12:41:35 -0600 Subject: [PATCH 124/252] chore: clinvar send invalid enumerated values to slack (#5344) * chore: clinvar update enumerated values * Cleanup * missing filter * cleanup test cases --- .../commands/reload_clinvar_all_variants.py | 44 +++++++++---- .../reload_clinvar_all_variants_tests.py | 64 ++++++++++++++++++- 2 files changed, 94 insertions(+), 14 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 1e137a25ee..8f14c1a289 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -99,7 +99,7 @@ def parse_positions(classified_record_node: xml.etree.ElementTree.Element) -> di } return positions -def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element) -> [str, list[str]]: +def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element, allele_id: int, unenumerated_value_alerts: list) -> [str, list[str]]: pathogenicity_node = classified_record_node.find( 'Classifications/GermlineClassification/Description', ) @@ -133,15 +133,23 @@ def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.Element assertions = [a.strip() for a in pathogenicity_string.split(';')] enumerated_assertions = set(CLINVAR_ASSERTIONS) + filtered_assertions = [] for assertion in assertions: if assertion not in enumerated_assertions: - raise CommandError(f'Found an un-enumerated clinvar assertion: {assertion}') - - return pathogenicity, assertions + unenumerated_value_alerts.append({ + 'type': 'Assertion', + 'allele_id': allele_id, + 'value': assertion, + }) + else: + filtered_assertions.append(assertion) + return pathogenicity, filtered_assertions def parse_conflicting_pathogenicities( classified_record_node: xml.etree.ElementTree.Element, pathogenicity: str, + allele_id: int, + unenumerated_value_alerts: list, ) -> list[[str, int]]: if pathogenicity == CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY: conflicting_pathogenicities_node = classified_record_node.find( @@ -159,10 +167,17 @@ def parse_conflicting_pathogenicities( conflicting_pathogenicities_node.text ) enumerated_pathogenicities = set(CLINVAR_PATHOGENICITIES) - for (pathogenicity, _) in conflicting_pathogenicities: + filtered_pathogenicities = [] + for (pathogenicity, count) in conflicting_pathogenicities: if pathogenicity not in enumerated_pathogenicities: - raise CommandError(f'Found an un-enumerated conflicting pathogenicity: {pathogenicity}') - return conflicting_pathogenicities + unenumerated_value_alerts.append({ + 'type': 'Conflicting Pathogenicity', + 'allele_id': allele_id, + 'value': pathogenicity, + }) + else: + filtered_pathogenicities.append((pathogenicity, count)) + return filtered_pathogenicities def parse_gold_stars(classified_record_node: xml) -> Optional[int]: review_status_node = classified_record_node.find( @@ -189,7 +204,7 @@ def parse_submitters_and_conditions(classified_record_node: xml) -> [list[str], }) return submitters, conditions -def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]: +def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, unenumerated_value_alerts: Optional[list] = None) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]: # Cannot use regular bool-falseyness here, as: # "An element with no child elements (even if it exists and has text) will be falsey." classified_record_node = elem.find('ClassifiedRecord') @@ -202,8 +217,8 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str) if not positions: return None, None, None - pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node) - conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity) + pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node, allele_id, unenumerated_value_alerts) + conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity, allele_id, unenumerated_value_alerts) gold_stars = parse_gold_stars(classified_record_node) submitters, conditions = parse_submitters_and_conditions(classified_record_node) # Note: this manipulation to an enumerated pathogenicty happens after we parse conflicting pathogenicities. @@ -247,6 +262,7 @@ def handle(self, *args, **options): ClinvarAllVariantsSnvIndel: [], ClinvarAllVariantsMito: [], } + unenumerated_value_alerts = [] new_version = None with tempfile.TemporaryDirectory() as tmpdir: with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \ @@ -276,7 +292,7 @@ def handle(self, *args, **options): ) # Handle parsing variants if event == 'end' and elem.tag == 'VariationArchive' and new_version: - for obj in extract_variant_info(elem, new_version): + for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): for model, batch in model_to_batch.items(): if isinstance(obj, model): batch.append(obj) @@ -316,4 +332,10 @@ def handle(self, *args, **options): version=new_version ) slack_message = f'Successfully updated Clinvar ClickHouse tables to {new_version}.' + if unenumerated_value_alerts: + slack_message += f'\n\nFound {len(unenumerated_value_alerts)} unenumerated value(s) during parsing:' + for alert in unenumerated_value_alerts[:10]: + slack_message += f"\n- {alert['type']}: '{alert['value']}' (Allele ID: {alert['allele_id']})" + if len(unenumerated_value_alerts) > 10: + slack_message += f"\n• ... and {len(unenumerated_value_alerts) - 10} more" safe_post_to_slack(SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, slack_message) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index 6462ba065a..c22bff5763 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -200,12 +200,10 @@ def test_batching(self, mock_logger, mock_safe_post_to_slack): @responses.activate def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): + # Test errors that should still raise (not unenumerated values) for description, review_status, conflicting_pathogenicities, error_message in [ - ("Pathogenic-ey", None, None, 'Found an un-enumerated clinvar assertion: Pathogenic-ey'), - ("Pathogenic; but unknown assertion", None, None, 'Found an un-enumerated clinvar assertion: but unknown assertion'), ("Pathogenic", "unhandled", None, 'Found unexpected review status unhandled'), ("Conflicting classifications of pathogenicity", None, "Pathogenic;", 'Failed to correctly parse conflicting pathogenicity counts: Pathogenic;'), - ("Conflicting classifications of pathogenicity", None, "Pathogenic(18); unhandled(1)", 'Found an un-enumerated conflicting pathogenicity: unhandled'), ("Conflicting classifications of pathogenicity", None, None, 'Failed to find the conflicting pathogenicities node'), ]: data = f''' @@ -239,6 +237,66 @@ def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): with self.assertRaisesMessage(CommandError, error_message): call_command('reload_clinvar_all_variants') + # Test that unenumerated values are now collected instead of raising + unenumerated_test_cases = [ + { + 'allele_id': 1, + 'description': 'Pathogenic-ey', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'Pathogenic-ey', + }, + { + 'allele_id': 2, + 'description': 'Pathogenic; but unknown assertion', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'but unknown assertion', + }, + { + 'allele_id': 3, + 'description': 'Conflicting classifications of pathogenicity', + 'explanation': 'Pathogenic(18); unhandled(1)', + 'alert_type': 'Conflicting Pathogenicity', + 'alert_value': 'unhandled', + }, + ] + data = WEEKLY_XML_RELEASE_HEADER + for case in unenumerated_test_cases: + data += f''' + + + + + + + + + + {case['description']} + {f"{case['explanation']}" if case['explanation'] else ""} + + + + + ''' + data += '' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(data.encode()), + stream=True, + ) + DataVersions.objects.all().delete() + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + call_command('reload_clinvar_all_variants') + # Command should succeed and collect all unenumerated values in single slack message + slack_message = mock_safe_post_to_slack.call_args[0][1] + self.assertIn(f'Found {len(unenumerated_test_cases)} unenumerated value(s) during parsing:', slack_message) + for case in unenumerated_test_cases: + self.assertIn(f"- {case['alert_type']}: '{case['alert_value']}' (Allele ID: {case['allele_id']})", slack_message) + # Variants with missing/equivalent alleles and positions are skipped for simple_allele_attrs, sequence_location_attrs in [ # Case 1: Missing AlleleId in From 19c94be0afd0d7abb4469d82f68b0c8ad03e0973 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 15:41:00 -0400 Subject: [PATCH 125/252] test discovery tags --- .../fixtures/clickhouse_transcripts.json | 14 +++++++ clickhouse_search/search_tests.py | 39 +++++++++++++++++-- seqr/views/apis/variant_search_api_tests.py | 22 ----------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json index d40b40e856..54383fa7bd 100644 --- a/clickhouse_search/fixtures/clickhouse_transcripts.json +++ b/clickhouse_search/fixtures/clickhouse_transcripts.json @@ -264,5 +264,19 @@ ] ] } +}, { + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": null, + "lifted_over_chrom": null, + "lifted_over_pos": null, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "transcripts": [] + } } ] \ No newline at end of file diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6aa722f82e..45250c6227 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -103,7 +103,7 @@ def setUpTestData(cls): class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' - fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): self.MOCK_CACHE = {} @@ -136,8 +136,8 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) - def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, **search_kwargs): - search_hash = random.randint(1000, 10000) # nosec + def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs): + search_hash = search_hash or random.randint(1000, 10000) # nosec self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) @@ -1998,6 +1998,7 @@ def test_search_context(self): }, ) + # Test OMIM intervals self.login_manager() response_json = self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], gene_counts=SV_GENE_COUNTS, @@ -2007,6 +2008,38 @@ def test_search_context(self): '3': {'chrom': '1', 'end': 249055991, 'mimNumber': 600315, 'phenotypeDescription': '?Immunodeficiency 16', 'phenotypeInheritance': 'Autosomal recessive', 'phenotypeMimNumber': 615120, 'start': 249044482}, }) + # Test cross-project discovery for analyst users + self.set_cache('search_results__VRS0009876__xpos',{'total_results': 1, 'all_results': [{ + 'key': 100, 'familyGuids': ['F000002_2'], 'xpos': 1248367227, 'genomeVersion': '38', 'sortedTranscriptConsequences': [], + }]}) + response, _, _ = self._execute_search(search_hash=9876) + self.assertEqual(response.status_code, 200) + variants = response.json()['searchedVariants'] + self.assertEqual(len(variants), 1) + self.assertFalse('discoveryTags' in variants[0]) + + self.login_analyst_user() + response, _, _ = self._execute_search(search_hash=9876) + self.assertEqual(response.status_code, 200) + variants = response.json()['searchedVariants'] + self.assertEqual(len(variants), 1) + self.assertListEqual(variants[0]['discoveryTags'], [{ + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0003_tes', + 'familyGuid': 'F000012_12', + 'projectGuid': 'R0003_test', + }, + 'tagGuid': 'VT1726961_2103343353_r0003_tes', + 'name': 'Tier 1 - Novel gene and phenotype', + 'category': 'CMG Discovery Tags', + 'color': '#03441E', + 'searchHash': None, + 'metadata': None, + 'lastModifiedDate': '2018-05-29T16:32:51.449Z', + 'createdBy': None, + }]) + self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) + def test_cached_query_variants(self): # TODO cache_key_prefix = f'search_results__{self.results_model.guid}' diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 7724f5ee5c..edcc7fd6d4 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -346,28 +346,6 @@ def _assert_expected_results_context(self, response_json, has_pa_detail=True, lo {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) - @mock.patch('seqr.utils.middleware.logger.error') - @mock.patch('seqr.views.apis.variant_search_api.get_variant_query_gene_counts') - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_error_logger): - url = reverse(query_variants_handler, args=['abc']) - self.check_collaborator_login(url, request_data={'projectFamilies': PROJECT_FAMILIES}) - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - - # Test cross-project discovery for analyst users - self.login_analyst_user() - mock_get_variants.side_effect = _get_es_variants - response = self.client.get('{}?sort=pathogenicity'.format(url)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_search_results = deepcopy(EXPECTED_SEARCH_RESPONSE) - expected_search_results['searchedVariants'] = VARIANTS_WITH_DISCOVERY_TAGS - expected_search_results['savedVariantsByGuid']['SV0000002_1248367227_r0390_100']['discoveryTags'] = DISCOVERY_TAGS - expected_search_results['familiesByGuid'].update({'F000012_12': mock.ANY}) - self.assertSetEqual(set(response_json.keys()), set(expected_search_results.keys())) - self.assertDictEqual(response_json, expected_search_results) - self._assert_expected_results_context(response_json) - @mock.patch('seqr.views.apis.variant_search_api.query_variants') def test_query_all_projects_variants(self, mock_get_variants): url = reverse(query_variants_handler, args=[SEARCH_HASH]) From aab0f63936b52081d70e3b2fc67c9dfebc020fa4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 16:35:03 -0400 Subject: [PATCH 126/252] test all project search --- .../fixtures/clickhouse_search.json | 2 +- clickhouse_search/search_tests.py | 38 ++++++ clickhouse_search/test_utils.py | 7 +- seqr/views/apis/variant_search_api_tests.py | 114 ------------------ 4 files changed, 44 insertions(+), 117 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 45ee737d2a..3f38de9c53 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -887,7 +887,7 @@ "filters": [], "sign": 1, "calls": [ - ["NA19675_1", 1, 0, 0.16, 45] + ["NA19675", 1, 0, 0.16, 45] ] } }, { diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 45250c6227..027ca43697 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -383,6 +383,44 @@ def test_multi_project_search(self): ] ) + def test_all_project_search(self): + self.login_collaborator() + request_body = {'allGenomeProjectFamilies': '38'} + project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': [ + 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', + 'F000009_9', 'F000010_10', 'F000013_13', + ]}] + additional_response = {'familiesByGuid': {'F000001_1': mock.ANY}} + self._assert_expected_search( + [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, + GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + request_body=request_body, project_families=project_families, additional_response=additional_response, + ) + + self.login_manager() + project_families += SV_PROJECT_FAMILIES + results = [ + VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, + GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, + ] + self._assert_expected_search( + results, request_body=request_body, project_families=project_families, additional_response=additional_response, + ) + + request_body['unsolvedFamiliesOnly'] = True + project_families[0]['familyGuids'].remove('F000007_7') + project_families[0]['familyGuids'].remove('F000010_10') + self._assert_expected_search( + results, request_body=request_body, project_families=project_families, additional_response=additional_response, + ) + + request_body['trioFamiliesOnly'] = True + self._assert_expected_search( + [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, + GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + request_body=request_body, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + ) + def test_both_sample_types_search(self): Sample.objects.filter(dataset_type='MITO').update(is_active=False) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 96cc672c7d..b9eea18d4d 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -397,13 +397,16 @@ 'xpos': 21003343353, 'rsid': None, 'familyGuids': ['F000001_1'], - 'genotypes': {}, + 'genotypes': {'I000001_na19675': { + 'ab': 0.16, 'dp': 45, 'familyGuid': 'F000001_1', 'filters': [], 'gq': 0, 'individualGuid': 'I000001_na19675', + 'numAlt': 1, 'sampleId': 'NA19675', 'sampleType': 'WES', + }}, 'clinvar': None, 'hgmd': None, 'screenRegionType': None, 'populations': { 'seqr': {'ac': 1, 'hom': 0, 'ac_wes': 1, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0}, - 'seqr_affected': {'ac': 1, 'hom': 0}, + 'seqr_affected': {'ac': 0, 'hom': 0}, 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index edcc7fd6d4..6616b793f1 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -346,120 +346,6 @@ def _assert_expected_results_context(self, response_json, has_pa_detail=True, lo {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_all_projects_variants(self, mock_get_variants): - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - self.check_require_login(url) - - expected_searched_families = set() - def _get_variants(results_model, **kwargs): - results_model.save() - self.assertSetEqual(expected_searched_families, {f.guid for f in results_model.families.all()}) - matched_variants = [ - deepcopy(variant) for variant in ALL_VARIANTS - if any(family_guid in expected_searched_families for family_guid in variant['familyGuids']) - ] - return matched_variants, len(matched_variants) - - mock_get_variants.side_effect = _get_variants - - body = {'allGenomeProjectFamilies': '37', 'search': SEARCH} - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - empty_search_response = { - 'searchedVariants': [], 'search': { - 'search': SEARCH, - 'projectFamilies': [], - 'totalResults': 0, - }} - self.assertDictEqual(response_json, empty_search_response) - results_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - mock_get_variants.assert_called_with(results_model, sort='xpos', page=1, num_results=100, user=self.no_access_user) - - VariantSearchResults.objects.filter(search_hash=SEARCH_HASH).delete() - self.login_data_manager_user() - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), empty_search_response) - - VariantSearchResults.objects.filter(search_hash=SEARCH_HASH).delete() - self.login_collaborator() - expected_searched_families = { - 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', - 'F000009_9', 'F000010_10', 'F000013_13'} - body['projectFamilies'] = [{'projectGuid': PROJECT_GUID, 'familyGuids': ['F000001_1']}] - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - - result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) - mock_get_variants.assert_called_with(result_model, sort='xpos', page=1, num_results=100, - user=self.collaborator_user) - - result_model.delete() - body['includeNoAccessProjects'] = True - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_no_access_resonse = deepcopy(EXPECTED_SEARCH_RESPONSE) - expected_no_access_resonse['search']['search']['no_access_project_genome_version'] = '37' - self.assertDictEqual(response_json, expected_no_access_resonse) - self._assert_expected_results_context(response_json) - self.assertSetEqual(set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - result_model = VariantSearchResults.objects.get(search_hash=SEARCH_HASH) - self.assertSetEqual(expected_searched_families, {f.guid for f in result_model.families.all()}) - mock_get_variants.assert_called_with( - result_model, sort='xpos', page=1, num_results=100, user=self.collaborator_user, - ) - - result_model.delete() - expected_searched_families.remove('F000007_7') - expected_searched_families.remove('F000010_10') - body['unsolvedFamiliesOnly'] = True - del body['includeNoAccessProjects'] - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), set(EXPECTED_SEARCH_RESPONSE.keys())) - self.assertDictEqual(response_json, EXPECTED_SEARCH_RESPONSE) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - - VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete() - body['trioFamiliesOnly'] = True - expected_searched_families = {'F000002_2'} - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self.assertEqual(response.status_code, 200) - response_json = response.json() - expected_variants = EXPECTED_SEARCH_RESPONSE['searchedVariants'][:1] + EXPECTED_SEARCH_RESPONSE['searchedVariants'][2:] - expected_response = { - **EXPECTED_SEARCH_RESPONSE, - 'searchedVariants': expected_variants, - 'search': {**EXPECTED_SEARCH_RESPONSE['search'], 'totalResults': len(expected_variants)}, - 'mmeSubmissionsByGuid': {}, - 'variantFunctionalDataByGuid': {}, - 'savedVariantsByGuid': { - k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() - if k != 'SV0000001_2103343353_r0390_100' - }, - 'variantTagsByGuid': { - k: v for k, v in EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'].items() - if k not in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'] - }, - } - self.assertSetEqual(set(response_json.keys()), set(expected_response.keys())) - self.assertDictEqual(response_json, expected_response) - self._assert_expected_results_context(response_json) - self.assertSetEqual( - set(response_json['search']['projectFamilies'][0]['familyGuids']), expected_searched_families) - @mock.patch('seqr.views.apis.variant_search_api.query_variants') def test_query_all_project_families_variants(self, mock_get_variants): url = reverse(query_variants_handler, args=['abc']) From cf1a66ec0c7a3c56d55159fdfe6ed4ef076cae7b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 16:39:32 -0400 Subject: [PATCH 127/252] more test merging --- clickhouse_search/search_tests.py | 7 ++++- seqr/views/apis/variant_search_api_tests.py | 29 --------------------- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 027ca43697..fba2028ce9 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -975,11 +975,16 @@ def test_invalid_search(self): self._set_grch37_search() self._assert_expected_search_error('Invalid genes/intervals: DDX11L1, ENSG00000223972', locus={'rawItems': build_specific_genes}) - self._assert_expected_search_error( + search_hash = self._assert_expected_search_error( 'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: 37 - 1kg project nåme with uniçøde; 38 - Test Reprocessed Project', project_families=MULTI_PROJECT_PROJECT_FAMILIES, ) + # Test export disabled in demo projects + export_url = reverse(export_variants_handler, args=[search_hash]) + response = self.client.get(export_url) + self.assertEqual(response.status_code, 403) + @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): # TODO diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 6616b793f1..861f7efdf9 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -346,35 +346,6 @@ def _assert_expected_results_context(self, response_json, has_pa_detail=True, lo {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} ) - @mock.patch('seqr.views.apis.variant_search_api.query_variants') - def test_query_all_project_families_variants(self, mock_get_variants): - url = reverse(query_variants_handler, args=['abc']) - self.check_collaborator_login(url, request_data={'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}]}) - url = reverse(query_variants_handler, args=[SEARCH_HASH]) - - mock_get_variants.side_effect = _get_es_variants - - response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}], - })) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertDictEqual(response_json['search'], { - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': 'R0003_test', 'familyGuids': ['F000011_11']}], - 'totalResults': 5, - }) - - mock_get_variants.assert_called_with( - VariantSearchResults.objects.get(search_hash=SEARCH_HASH), sort='xpos', page=1, num_results=100, - user=self.collaborator_user) - - # Test export disabled in demo projects - export_url = reverse(export_variants_handler, args=[SEARCH_HASH]) - response = self.client.get(export_url) - self.assertEqual(response.status_code, 403) - def test_search_context(self): search_context_url = reverse(search_context_handler) self.check_collaborator_login(search_context_url, request_data={'familyGuid': 'F000001_1'}) From 9010c0c380933685ff610a8507824efcc2ec6b78 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 17:06:12 -0400 Subject: [PATCH 128/252] move single variants handle test --- .../fixtures/clickhouse_search.json | 6 ++ clickhouse_search/search_tests.py | 73 ++++++++++------ seqr/views/apis/variant_search_api_tests.py | 86 ------------------- 3 files changed, 51 insertions(+), 114 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 3f38de9c53..d7e8bf15d1 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -137,6 +137,12 @@ "key": 22 } }, { + "model": "clickhouse_search.keylookupsnvindel", + "pk": "21-3343353-GAGA-G", + "fields": { + "key": 23 + } +},{ "model": "clickhouse_search.clinvarallvariantssnvindel", "pk": "1-10439-AC-A", "fields": { diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index fba2028ce9..7b38faff75 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -44,8 +44,15 @@ from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ - FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS -from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler + FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS +from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler, \ + query_single_variant_handler + + +SEARCH_RESPONSE_KEYS = { + 'genesById', 'locusListsByGuid', 'mmeSubmissionsByGuid', 'phenotypeGeneScores', 'rnaSeqData', 'savedVariantsByGuid', + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', 'totalSampleCounts', +} class ClickhouseSearchTestCase(AnvilAuthenticationTestCase): @@ -176,16 +183,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun } if expected_results: expected_response.update({ - 'genesById': mock.ANY, - 'locusListsByGuid': mock.ANY, - 'mmeSubmissionsByGuid': mock.ANY, - 'phenotypeGeneScores': mock.ANY, - 'rnaSeqData': mock.ANY, - 'savedVariantsByGuid': mock.ANY, - 'variantFunctionalDataByGuid': mock.ANY, - 'variantNotesByGuid': mock.ANY, - 'variantTagsByGuid': mock.ANY, - 'totalSampleCounts': mock.ANY, + **{key: mock.ANY for key in SEARCH_RESPONSE_KEYS}, **(additional_response or {}), }) if not is_37: @@ -1094,29 +1092,48 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3], 'variant_lookup_results__suffix_140593_DUP__38') def test_get_single_variant(self): - # TODO - self._set_single_family_search() - variant = get_single_variant(self.results_model.families.first(), VARIANT_IDS[0]) - self._assert_expected_variants([variant], [VARIANT1]) + url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') + url = url_template.format('21-3343353-GAGA-G', 'F000001_1') + self.check_collaborator_login(url) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(self.results_model.families.first(), VARIANT_IDS[1]) - self.assertEqual(str(cm.exception), 'Variant 1-91511686-TCA-G not found') + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'searchedVariants': [FAMILY_1_VARIANT], + 'familiesByGuid': {'F000001_1': mock.ANY}, + 'projectsByGuid': {'R0001_1kg': mock.ANY}, + 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': { + key: mock.ANY for key in ['paLocusList', 'intervals', *LOCUS_LIST_FIELDS] + }}, + **{key: mock.ANY for key in [ + 'omimIntervals', 'individualsByGuid', 'igvSamplesByGuid', 'familyNotesByGuid', *SEARCH_RESPONSE_KEYS, + ]}, + }) - variant = get_single_variant(self.results_model.families.first(), 'M-4429-G-A') - self._assert_expected_variants([variant], [MITO_VARIANT1]) + response = self.client.get(url_template.format(VARIANT_IDS[1], 'F000002_2')) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Variant 1-91511686-TCA-G not found'}) + + response = self.client.get(url_template.format('M-4429-G-A', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [MITO_VARIANT1]) - variant = get_single_variant(self.results_model.families.first(), 'suffix_140608_DUP') - self._assert_expected_variants([variant], [GCNV_VARIANT4]) + response = self.client.get(url_template.format('suffix_140608_DUP', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [GCNV_VARIANT4]) - self._set_sv_family_search() - variant = get_single_variant(self.results_model.families.first(), 'phase2_DEL_chr14_4640') - self._assert_expected_variants([variant], [SV_VARIANT4]) + self.login_manager() + response = self.client.get(url_template.format('phase2_DEL_chr14_4640', 'F000014_14')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [SV_VARIANT4]) self._set_grch37_search() - variant = get_single_variant(self.results_model.families.first(), '7-143270172-A-G') - self._assert_expected_variants([variant], [GRCH37_VARIANT]) + response = self.client.get(url_template.format('7-143270172-A-G', 'F000002_2')) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.json()['searchedVariants'], [GRCH37_VARIANT]) + self.mock_redis.get.assert_not_called() + self.mock_redis.set.assert_not_called() def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 861f7efdf9..ed09c24077 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -306,46 +306,6 @@ def _assert_expected_search_context(self, response_json): 'analysedBy': [{'createdBy': 'Test No Access User', 'dataType': 'SNP', 'lastModifiedDate': '2022-07-22T19:27:08.563+00:00'}], }) - def _assert_expected_results_family_context(self, response_json, locus_list_detail=False): - self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) - - family_fields = {'individualGuids'} - family_fields.update(FAMILY_FIELDS) - if len(response_json['familiesByGuid']) > 1: - self.assertSetEqual(set(response_json['familiesByGuid']['F000002_2'].keys()), family_fields) - - family_fields.add('tpmGenes') - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) - - self.assertEqual(len(response_json['individualsByGuid']), len(response_json['familiesByGuid'])*3) - individual_fields = {'igvSampleGuids'} - individual_fields.update(INDIVIDUAL_FIELDS) - self.assertSetEqual(set(response_json['individualsByGuid']['I000001_na19675'].keys()), individual_fields) - - self.assertEqual(len(response_json['igvSamplesByGuid']), 1) - self.assertSetEqual(set(response_json['igvSamplesByGuid']['S000145_na19675'].keys()), IGV_SAMPLE_FIELDS) - - self.assertEqual(len(response_json['familyNotesByGuid']), 3) - self.assertSetEqual(set(response_json['familyNotesByGuid']['FAN000001_1'].keys()), FAMILY_NOTE_FIELDS) - - self._assert_expected_rnaseq_response(response_json) - - def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True): - # TODO used in single variant context - locus_list_fields = {'intervals'} - if locus_list_detail: - locus_list_fields.update(LOCUS_LIST_FIELDS) - if has_pa_detail: - locus_list_fields.update({'paLocusList'}) - self.assertSetEqual(set(response_json['locusListsByGuid'][LOCUS_LIST_GUID].keys()), locus_list_fields) - intervals = response_json['locusListsByGuid'][LOCUS_LIST_GUID]['intervals'] - self.assertEqual(len(intervals), 2) - self.assertSetEqual( - set(intervals[0].keys()), - {'locusListGuid', 'locusListIntervalGuid', 'genomeVersion', 'chrom', 'start', 'end'} - ) - def test_search_context(self): search_context_url = reverse(search_context_handler) self.check_collaborator_login(search_context_url, request_data={'familyGuid': 'F000001_1'}) @@ -423,52 +383,6 @@ def test_search_context(self): response_json = response.json() self._assert_expected_search_context(response_json) - - @mock.patch('seqr.views.apis.variant_search_api.get_single_variant') - def test_query_single_variant(self, mock_get_variant): - mock_get_variant.return_value = SINGLE_FAMILY_VARIANT - - url = '{}?familyGuid=F000001_1'.format(reverse(query_single_variant_handler, args=['21-3343353-GAGA-G'])) - self.check_collaborator_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self._assert_expected_single_variant_results_context(response.json(), searchedVariants=[SINGLE_FAMILY_VARIANT]) - - mock_get_variant.assert_called_with(mock.ANY, '21-3343353-GAGA-G') - searched_family = mock_get_variant.call_args.args[0] - self.assertEqual(searched_family.guid, 'F000001_1') - - mock_get_variant.side_effect = InvalidSearchException('Variant not found') - response = self.client.get(url) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Variant not found') - - def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response): - omit_fields = {'search', *(omit_fields or [])} - - expected_search_response = deepcopy({**EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) - expected_search_response.update(expected_response) - expected_search_response.update({ - k: EXPECTED_SEARCH_CONTEXT_RESPONSE[k] for k in ['projectsByGuid', 'familiesByGuid', 'locusListsByGuid'] - }) - for k in omit_fields: - expected_search_response.pop(k) - expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') - expected_search_response['variantTagsByGuid'] = { - k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] - for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} - } - expected_search_response['variantNotesByGuid'] = {} - expected_search_response['genesById'] = { - k: v for k, v in expected_search_response['genesById'].items() if k in {'ENSG00000227232', 'ENSG00000268903'} - } - self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) - self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_family_context(response_json, locus_list_detail=True) - self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) - self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1'}) - @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_variant_lookup(self, mock_variant_lookup): response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) From 38911e59cf5c0ebb94e07b890d1d94146d4bc3a9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 17:07:30 -0400 Subject: [PATCH 129/252] clean up --- clickhouse_search/search_tests.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 7b38faff75..eca20019ff 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -169,7 +169,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, format_cache_key=None, project_families=None, additional_response=None, export_data=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, project_families=None, additional_response=None, export_data=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -190,13 +190,9 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun expected_response['omimIntervals'] = mock.ANY self.assertDictEqual(response.json(), expected_response) - cache_key = format_cache_key() if format_cache_key else f'search_results__VRS{search_hash:07d}__{sort}' - if cache_key: - cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) - self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) - else: - self.mock_redis.get.assert_not_called() - self.mock_redis.set.assert_not_called() + cache_key = f'search_results__VRS{search_hash:07d}__{sort}' + cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) + self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) if gene_counts: self._assert_expected_gene_counts(search_hash, gene_counts) From ea1d9cd8b19ae349f6630af7931ca2d924fddb75 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 17:22:59 -0400 Subject: [PATCH 130/252] fix test --- clickhouse_search/test_utils.py | 4 ++-- seqr/views/apis/variant_search_api_tests.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index b9eea18d4d..2b9466fc42 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1504,8 +1504,8 @@ def format_cached_variant(variant): '', '', '', '', ''], ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '', '0', '0', '0', '', '', '', '', '', '', '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '', '', '', '', '', '', '', '', ''], + 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19675', '1', '', '0', '0.16', '', '', + '', '', '', '', '', '', '', '', '', '', '', '', ''], ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index ed09c24077..5bd4278178 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -385,6 +385,7 @@ def test_search_context(self): @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_variant_lookup(self, mock_variant_lookup): + # TODO response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(response_variant)] @@ -553,6 +554,7 @@ def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_sv_variant_lookup(self, mock_variant_lookup): + # TODO lookup_variants = [ {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}}, SV_LOOKUP_VARIANT, From ea0a403d5336eaa6a131327308f012d34d58d612 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 11 Mar 2026 17:46:03 -0400 Subject: [PATCH 131/252] clean up --- clickhouse_search/search_tests.py | 355 ++++++++++- seqr/views/apis/summary_data_api_tests.py | 2 +- seqr/views/apis/variant_search_api_tests.py | 429 +------------ seqr/views/utils/test_utils.py | 671 -------------------- 4 files changed, 355 insertions(+), 1102 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index eca20019ff..51412df1c4 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -46,7 +46,7 @@ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS from seqr.views.apis.variant_search_api import query_variants_handler, get_variant_gene_breakdown, export_variants_handler, \ - query_single_variant_handler + query_single_variant_handler, variant_lookup_handler SEARCH_RESPONSE_KEYS = { @@ -981,23 +981,221 @@ def test_invalid_search(self): @mock.patch('seqr.utils.search.utils.LiftOver') def test_variant_lookup(self, mock_liftover): - # TODO mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] - variants = variant_lookup(self.user, '1-10439-AC-A', '38') + url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38&affectedOnly=true' + self.check_require_login(url) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + expected_variant = { + **VARIANT_LOOKUP_VARIANT, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A', 'F2_1-10439-AC-A'], + 'liftedFamilyGuids': ['F2_1-10439-AC-A'], + 'genotypes': { + 'I0_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I1_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I2_F0_1-10439-AC-A': [ + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F1_1-10439-AC-A': [ + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], + 'sampleType': 'WGS'}, + }, + } + del expected_variant['familyGenotypes'] + expected_individuals = { + 'I0_F0_1-10439-AC-A': { + 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], + 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I0_F1_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F1_1-10439-AC-A', 'individualGuid': 'I0_F1_1-10439-AC-A', 'sex': 'M', + 'features': [{'category': 'HP:0001626', 'label': '1 terms'}, {'category': 'Other', 'label': '1 terms'}], + 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + }, + 'I0_F2_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], + 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_1-10439-AC-A': { + 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], + 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_1-10439-AC-A': { + 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, + {'category': 'HP:0001626', 'label': '1 terms'}], + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None, + affected_only=True, hom_only=False) + self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT], 'variant_lookup_results__1-10439-AC-A__38') + + + response_variant['transcripts'] = VARIANTS[0]['transcripts'] + expected_variant['transcripts'] = VARIANTS[0]['transcripts'] + expected_gene_body = { + 'genesById': {'ENSG00000227232': EXPECTED_GENE, 'ENSG00000268903': EXPECTED_GENE}, + } + expected_body.update(expected_gene_body) + # TODO test these for real + EXPECTED_LIRICAL_DATA = [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, + 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, + ] + EXPECTED_EXOMISER_DATA = [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, + 'scores': {'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, + 'variant_score': 0.999200702}}, + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, + 'scores': {'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, + 'variant_score': 1}}, + ] + EXPECTED_SEARCH_RESPONSE = { + 'searchedVariants': ALL_VARIANTS, + 'savedVariantsByGuid': { + 'SV0000001_2103343353_r0390_100': mock.ANY, + 'SV0000002_1248367227_r0390_100': mock.ANY, + }, + 'genesById': { + 'ENSG00000227232': expected_pa_gene, 'ENSG00000268903': EXPECTED_GENE, 'ENSG00000233653': EXPECTED_GENE, + 'ENSG00000177000': mock.ANY, 'ENSG00000097046': mock.ANY, + }, + 'search': { + 'search': SEARCH, + 'projectFamilies': [{'projectGuid': PROJECT_GUID, 'familyGuids': mock.ANY}], + 'totalResults': 5, + }, + 'variantTagsByGuid': { + 'VT1708633_2103343353_r0390_100': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, + 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726961_2103343353_r0390_100': EXPECTED_TAG, + 'VT1726985_2103343353_r0390_100': expected_aip_tag, + }, + 'variantNotesByGuid': { + 'VN0714935_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, + 'VN0714937_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, + }, + 'variantFunctionalDataByGuid': { + 'VFD0000023_1248367227_r0390_10': expected_functional_tag, + 'VFD0000024_1248367227_r0390_10': expected_functional_tag, + 'VFD0000025_1248367227_r0390_10': expected_functional_tag, + 'VFD0000026_1248367227_r0390_10': expected_functional_tag, + }, + 'locusListsByGuid': {LOCUS_LIST_GUID: {'intervals': mock.ANY}}, + 'rnaSeqData': { + 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, + 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, + }, + 'phenotypeGeneScores': { + 'I000001_na19675': {'ENSG00000268903': {'exomiser': EXPECTED_EXOMISER_DATA}}, + 'I000002_na19678': {'ENSG00000268903': {'lirical': EXPECTED_LIRICAL_DATA}}, + }, + 'mmeSubmissionsByGuid': {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}, + 'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}, + 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, + } + if 'transcriptsById' in EXPECTED_SEARCH_RESPONSE: + expected_body['transcriptsById'] = EXPECTED_SEARCH_RESPONSE['transcriptsById'] + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), expected_body) + + response_variant['variantId'] = '1-248367227-TC-T' + response_variant['genomeVersion'] = '37' + self.login_manager() + response = self.client.get(url.replace("38", "37") + '&homOnly=true') + self.assertEqual(response.status_code, 200) + + individual_guid_map = [ + ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), + ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), + ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), + ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), + ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), + ] + expected_variant.update({ + 'lookupFamilyGuids': ['F000002_2', 'F000011_11', 'F000014_14'], + 'liftedFamilyGuids': ['F000014_14'], + 'genotypes': { + individual_guid: [ + {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in + expected_variant['genotypes'][anon_individual_guid] + ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { + **expected_variant['genotypes'][anon_individual_guid], **genotype, + 'individualGuid': individual_guid, + } for individual_guid, anon_individual_guid, genotype in individual_guid_map + }, + 'genomeVersion': '37', + 'variantId': '1-248367227-TC-T', + }) + expected_individuals = { + individual_guid: { + **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, + **{k: v for k, v in expected_individuals[anon_individual_guid].items() + if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, + } for individual_guid, anon_individual_guid, _ in individual_guid_map + } + expected_individuals.update( + {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) + expected_body = self._expected_lookup_body( + expected_individuals, [expected_variant], include_context=True, + project_guids=[PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], + ) + expected_body.update({ + k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in + {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} + }) + expected_body['savedVariantsByGuid'] = { + k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if + k in ['SV0000002_1248367227_r0390_100'] + } + expected_body.update(expected_gene_body) + expected_body['genesById']['ENSG00000227232'] = expected_pa_gene + expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop( + 'MS000001_na19675') + expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY + expected_body['variantTagsByGuid']['VT1726970_2103343353_r0004_tes'] = EXPECTED_TAG + expected_body['variantTagsByGuid']['VT1726961_2103343353_r0005_tes'] = EXPECTED_TAG + for k in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100']: + del expected_body['variantTagsByGuid'][k] + expected_body['rnaSeqData']['I000019_na21987'] = {'outliers': {}, + 'spliceOutliers': {'ENSG00000268903': mock.ANY}} + + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with( + self.manager_user, '1-10439-AC-A', '37', sample_type=None, affected_only=True, hom_only=True, + ) self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT], 'variant_lookup_results__1-10439-AC-A__38') with self.assertRaises(ObjectDoesNotExist) as cm: variant_lookup(self.user, '1-91511686-TCA-G', '38') self.assertEqual(str(cm.exception), 'Variant not present in seqr') - self.set_cache([VARIANT_LOOKUP_VARIANT]) + self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [VARIANT_LOOKUP_VARIANT]) cached_variants = variant_lookup(self.user, '1-91511686-TCA-G', '38') self.assertListEqual([VARIANT_LOOKUP_VARIANT], cached_variants) - self.mock_redis.get.assert_called_with('variant_lookup_results__1-91511686-TCA-G__38') - self.set_cache(None) variants = variant_lookup(self.user, '7-143270172-A-G', '37') grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, @@ -1063,6 +1261,112 @@ def test_variant_lookup(self, mock_liftover): variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + expected_gcnv_variant = { + **GCNV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], + 'genotypes': { + 'I0_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_variant = { + **SV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], + 'genotypes': { + 'I0_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_individuals = { + 'I0_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I0_F0_suffix_140608_DUP': { + 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, + {'category': 'HP:0001626', 'label': '1 terms'}], + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I1_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', + 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_phase2_DEL_chr14_4640': { + 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I2_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', + 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + expected_body = self._expected_lookup_body(expected_individuals, [expected_gcnv_variant, expected_sv_variant]) + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with( + self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False, + ) + + self.login_manager() + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + expected_gcnv_variant.update({ + 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], + 'genotypes': GCNV_VARIANT4['genotypes'], + }) + expected_sv_variant.update({ + 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], + 'genotypes': SV_VARIANT4['genotypes'], + }) + expected_individuals = { + i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', + 'I000021_na21654', + ] + } + expected_body = self._expected_lookup_body( + expected_individuals, + [expected_gcnv_variant, expected_sv_variant], + project_guids=[PROJECT_GUID, 'R0004_non_analyst_project'], + family_guids=['F000002_2', 'F000014_14'], + include_context=True, + ) + del expected_body['transcriptsById'] + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', + affected_only=False, hom_only=False, ) + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT], cache_key) @@ -1087,6 +1391,45 @@ def test_variant_lookup(self, mock_liftover): variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3], 'variant_lookup_results__suffix_140593_DUP__38') + def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): + exclude_keys = {'searchedVariants', 'search', 'transcriptsById'} + if not include_context: + exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) + EXPECTED_SEARCH_FAMILY_CONTEXT = { + 'familiesByGuid': {'F000001_1': mock.ANY, 'F000002_2': mock.ANY}, + 'individualsByGuid': mock.ANY, + 'igvSamplesByGuid': mock.ANY, + 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY}, + 'familyNotesByGuid': mock.ANY, + } + PROJECT_TAG_TYPE_FIELDS = {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'} + expected_body = { + **{k: {} for k in EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, + **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, + 'projectsByGuid': { + p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids or [] + }, + 'familiesByGuid': { + f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids or [] + }, + 'individualsByGuid': individuals_by_guid, + 'variants': variants, + } + if is_build_38: + expected_body['omimIntervals'] = {} + if include_context: + EXPECTED_TRANSCRIPTS_RESPONSE = { + 'transcriptsById': { + 'ENST00000624735': {'isManeSelect': False, 'refseqId': None, 'transcriptId': 'ENST00000624735'}}, + } + expected_body.update({ + **EXPECTED_TRANSCRIPTS_RESPONSE, + 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], + }) + if 'totalSampleCounts' in EXPECTED_SEARCH_RESPONSE: + expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + return expected_body + def test_get_single_variant(self): url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') url = url_template.format('21-3343353-GAGA-G', 'F000001_1') diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 32bc54acc2..771091921c 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -7,7 +7,7 @@ from seqr.views.apis.summary_data_api import mme_details, success_story, saved_variants_page, hpo_summary_data, \ bulk_update_family_external_analysis, individual_metadata, send_vlm_email -from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, PARSED_VARIANTS, SAVED_VARIANT_FIELDS +from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, SAVED_VARIANT_FIELDS from seqr.models import FamilyAnalysedBy, SavedVariant, VariantTag from settings import AIRTABLE_URL diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 5bd4278178..119b662664 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -6,17 +6,11 @@ from django.db import transaction from django.urls.base import reverse -from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, \ - SV_LOOKUP_VARIANT, SV_VARIANT4, GCNV_VARIANT4 -from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch -from seqr.utils.search.utils import InvalidSearchException -from seqr.views.apis.variant_search_api import query_variants_handler, query_single_variant_handler, vlm_lookup_handler, \ - export_variants_handler, search_context_handler, get_saved_search_handler, create_saved_search_handler, \ - update_saved_search_handler, delete_saved_search_handler, get_variant_gene_breakdown, variant_lookup_handler -from seqr.views.utils.test_utils import AuthenticationTestCase, VARIANTS,\ - GENE_VARIANT_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS, FAMILY_FIELDS, \ - PA_LOCUS_LIST_FIELDS, INDIVIDUAL_FIELDS, FUNCTIONAL_FIELDS, IGV_SAMPLE_FIELDS, FAMILY_NOTE_FIELDS, ANALYSIS_GROUP_FIELDS, \ - VARIANT_NOTE_FIELDS, TAG_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_DETAIL_FIELDS, DYNAMIC_ANALYSIS_GROUP_FIELDS +from seqr.models import LocusList, Project, VariantSearch +from seqr.views.apis.variant_search_api import vlm_lookup_handler, search_context_handler, get_saved_search_handler, \ + create_saved_search_handler, update_saved_search_handler, delete_saved_search_handler +from seqr.views.utils.test_utils import AuthenticationTestCase, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, \ + ANALYSIS_GROUP_FIELDS, DYNAMIC_ANALYSIS_GROUP_FIELDS LOCUS_LIST_GUID = 'LL00049_pid_genes_autosomal_do' PROJECT_GUID = 'R0001_1kg' @@ -24,115 +18,8 @@ SEARCH = {'filters': {}, 'inheritance': None} PROJECT_FAMILIES = [{'projectGuid': PROJECT_GUID, 'familyGuids': ['F000001_1', 'F000002_2']}] -ALL_VARIANTS = VARIANTS + [VARIANT2, VARIANT3] - -VARIANTS_WITH_DISCOVERY_TAGS = deepcopy(ALL_VARIANTS) -DISCOVERY_TAGS = [{ - 'savedVariant': { - 'variantGuid': 'SV0000006_1248367227_r0003_tes', - 'familyGuid': 'F000012_12', - 'projectGuid': 'R0003_test', - }, - 'tagGuid': 'VT1726961_2103343353_r0003_tes', - 'name': 'Tier 1 - Novel gene and phenotype', - 'category': 'CMG Discovery Tags', - 'color': '#03441E', - 'searchHash': None, - 'metadata': None, - 'lastModifiedDate': '2018-05-29T16:32:51.449Z', - 'createdBy': None, -}] -VARIANTS_WITH_DISCOVERY_TAGS[2]['discoveryTags'] = DISCOVERY_TAGS - -SINGLE_FAMILY_VARIANT = deepcopy(VARIANTS[0]) -SINGLE_FAMILY_VARIANT['familyGuids'] = ['F000001_1'] - PROJECT_CONTEXT_FIELDS = {'locusListGuids', 'datasetTypes', 'analysisGroupsLoaded', 'projectGuid', 'name'} -PROJECT_TAG_TYPE_FIELDS = {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'} - -EXPECTED_TAG = {k: mock.ANY for k in TAG_FIELDS} -expected_functional_tag = {k: mock.ANY for k in FUNCTIONAL_FIELDS} -expected_aip_tag = { - 'structuredMetadata': { - '4': {'date': '2023-11-15', 'name': 'de Novo'}, - 'support': {'date': '2023-11-15', 'name': 'High in Silico Scores'}, - }, - **EXPECTED_TAG, -} -del expected_aip_tag['metadata'] -EXPECTED_GENE = {k: mock.ANY for k in GENE_VARIANT_FIELDS} -EXPECTED_GENE['locusListGuids'] = [] -expected_pa_gene = deepcopy(EXPECTED_GENE) -expected_pa_gene['locusListGuids'] = ['LL00049_pid_genes_autosomal_do'] -expected_pa_gene['panelAppDetail'] = mock.ANY -EXPECTED_SAVED_VARIANT = {k: mock.ANY for k in SAVED_VARIANT_DETAIL_FIELDS} -EXPECTED_SAVED_VARIANT.update( {'key': mock.ANY, 'mainTranscriptId': mock.ANY}) -expected_detail_saved_variant = deepcopy(EXPECTED_SAVED_VARIANT) -expected_detail_saved_variant['mmeSubmissions'] = [ - {'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': 'SV0000001_2103343353_r0390_100'}, -] -expected_detail_saved_variant['mainTranscriptId'] = mock.ANY - -EXPECTED_EXOMISER_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, - 'scores': {'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, - 'variant_score': 0.999200702}}, - {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, - 'scores': {'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, - 'variant_score': 1}}, -] - -EXPECTED_LIRICAL_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, - 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, -] - -EXPECTED_SEARCH_RESPONSE = { - 'searchedVariants': ALL_VARIANTS, - 'savedVariantsByGuid': { - 'SV0000001_2103343353_r0390_100': expected_detail_saved_variant, - 'SV0000002_1248367227_r0390_100': EXPECTED_SAVED_VARIANT, - }, - 'genesById': { - 'ENSG00000227232': expected_pa_gene, 'ENSG00000268903': EXPECTED_GENE, 'ENSG00000233653': EXPECTED_GENE, - 'ENSG00000177000': mock.ANY, 'ENSG00000097046': mock.ANY, - }, - 'search': { - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': PROJECT_GUID, 'familyGuids': mock.ANY}], - 'totalResults': 5, - }, - 'variantTagsByGuid': { - 'VT1708633_2103343353_r0390_100': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726961_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726985_2103343353_r0390_100': expected_aip_tag, - }, - 'variantNotesByGuid': { - 'VN0714935_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - 'VN0714937_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - }, - 'variantFunctionalDataByGuid': { - 'VFD0000023_1248367227_r0390_10': expected_functional_tag, 'VFD0000024_1248367227_r0390_10': expected_functional_tag, - 'VFD0000025_1248367227_r0390_10': expected_functional_tag, 'VFD0000026_1248367227_r0390_10': expected_functional_tag, - }, - 'locusListsByGuid': {LOCUS_LIST_GUID: {'intervals': mock.ANY}}, - 'rnaSeqData': { - 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, - }, - 'phenotypeGeneScores': { - 'I000001_na19675': {'ENSG00000268903': {'exomiser': EXPECTED_EXOMISER_DATA}}, - 'I000002_na19678': {'ENSG00000268903': {'lirical': EXPECTED_LIRICAL_DATA}}, - }, - 'mmeSubmissionsByGuid': {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}, - 'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}, - 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, -} - -EXPECTED_TRANSCRIPTS_RESPONSE = { - 'transcriptsById': {'ENST00000624735': {'isManeSelect': False, 'refseqId': None, 'transcriptId': 'ENST00000624735'}}, -} - EXPECTED_SEARCH_CONTEXT_RESPONSE = { 'savedSearchesByGuid': { 'VS0079516_': mock.ANY, 'VS0079525_': mock.ANY, 'VS0079517_': mock.ANY, 'VS0145435_': mock.ANY, @@ -143,14 +30,6 @@ 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY, 'LL00005_retina_proteome': mock.ANY}, } -EXPECTED_SEARCH_FAMILY_CONTEXT = { - 'familiesByGuid': {'F000001_1': mock.ANY, 'F000002_2': mock.ANY}, - 'individualsByGuid': mock.ANY, - 'igvSamplesByGuid': mock.ANY, - 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY}, - 'familyNotesByGuid': mock.ANY, -} - MOCK_TOKEN = 'mock_token' # nosec MOCK_CLIENT_ID = 'mock_client_id' VLM_CLIENTS_RESPONSE = [ @@ -261,21 +140,6 @@ } } -def _get_es_variants(results_model, **kwargs): - results_model.save() - return deepcopy(ALL_VARIANTS), len(ALL_VARIANTS) - - -def _get_empty_es_variants(results_model, **kwargs): - results_model.save() - return [], 0 - - -COMP_HET_VARAINTS = [[VARIANTS[2], VARIANTS[1]]] -def _get_compound_het_es_variants(results_model, **kwargs): - results_model.save() - return deepcopy(COMP_HET_VARAINTS), 1 - @mock.patch('seqr.views.utils.permissions_utils.safe_redis_get_json', lambda *args: None) class VariantSearchAPITest(AuthenticationTestCase): @@ -383,289 +247,6 @@ def test_search_context(self): response_json = response.json() self._assert_expected_search_context(response_json) - @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') - def test_variant_lookup(self, mock_variant_lookup): - # TODO - response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(response_variant)] - - url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38&affectedOnly=true' - self.check_require_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A', 'F2_1-10439-AC-A'], - 'liftedFamilyGuids': ['F2_1-10439-AC-A'], - 'genotypes': { - 'I0_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I1_F0_1-10439-AC-A': [ - {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I2_F0_1-10439-AC-A': [ - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, - {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I0_F1_1-10439-AC-A': [ - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, - {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, - ], - 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, - }, - } - del expected_variant['familyGenotypes'] - expected_individuals = { - 'I0_F0_1-10439-AC-A': { - 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I0_F1_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F1_1-10439-AC-A', 'individualGuid': 'I0_F1_1-10439-AC-A', 'sex': 'M', - 'features': [{'category': 'HP:0001626', 'label': '1 terms'}, {'category': 'Other', 'label': '1 terms'}], - 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', - }, - 'I0_F2_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I1_F0_1-10439-AC-A': { - 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I2_F0_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', - 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, - {'category': 'HP:0001626', 'label': '1 terms'}], - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - } - expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None, affected_only=True, hom_only=False) - - response_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_gene_body = { - 'genesById': {'ENSG00000227232': EXPECTED_GENE, 'ENSG00000268903': EXPECTED_GENE}, - } - expected_body.update(expected_gene_body) - if 'transcriptsById' in EXPECTED_SEARCH_RESPONSE: - expected_body['transcriptsById'] = EXPECTED_SEARCH_RESPONSE['transcriptsById'] - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), expected_body) - - response_variant['variantId'] = '1-248367227-TC-T' - response_variant['genomeVersion'] = '37' - self.login_manager() - response = self.client.get(url.replace("38", "37") + '&homOnly=true') - self.assertEqual(response.status_code, 200) - - individual_guid_map = [ - ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), - ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), - ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), - ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), - ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), - ] - expected_variant.update({ - 'lookupFamilyGuids': ['F000002_2', 'F000011_11', 'F000014_14'], - 'liftedFamilyGuids': ['F000014_14'], - 'genotypes': { - individual_guid: [ - {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] - ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { - **expected_variant['genotypes'][anon_individual_guid], **genotype, 'individualGuid': individual_guid, - } for individual_guid, anon_individual_guid, genotype in individual_guid_map - }, - 'genomeVersion': '37', - 'variantId': '1-248367227-TC-T', - }) - expected_individuals = { - individual_guid: { - **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, - **{k: v for k, v in expected_individuals[anon_individual_guid].items() - if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, - } for individual_guid, anon_individual_guid, _ in individual_guid_map - } - expected_individuals.update({individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) - expected_body = self._expected_lookup_body( - expected_individuals, [expected_variant], include_context=True, - project_guids=[PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], - ) - expected_body.update({ - k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} - }) - expected_body['savedVariantsByGuid']= { - k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] - } - expected_body.update(expected_gene_body) - expected_body['genesById']['ENSG00000227232'] = expected_pa_gene - expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop('MS000001_na19675') - expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY - expected_body['variantTagsByGuid']['VT1726970_2103343353_r0004_tes'] = EXPECTED_TAG - expected_body['variantTagsByGuid']['VT1726961_2103343353_r0005_tes'] = EXPECTED_TAG - for k in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100']: - del expected_body['variantTagsByGuid'][k] - expected_body['rnaSeqData']['I000019_na21987'] = {'outliers': {}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}} - - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.manager_user, '1-10439-AC-A', '37', sample_type=None, affected_only=True, hom_only=True, - ) - - def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): - exclude_keys = {'searchedVariants', 'search', 'transcriptsById'} - if not include_context: - exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) - expected_body = { - **{k: {} for k in EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, - **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, - 'projectsByGuid': { - p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids or [] - }, - 'familiesByGuid': { - f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids or [] - }, - 'individualsByGuid': individuals_by_guid, - 'variants': variants, - } - if is_build_38: - expected_body['omimIntervals'] = {} - if include_context: - expected_body.update({ - **EXPECTED_TRANSCRIPTS_RESPONSE, - 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], - }) - if 'totalSampleCounts' in EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] - return expected_body - - @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') - def test_sv_variant_lookup(self, mock_variant_lookup): - # TODO - lookup_variants = [ - {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}}, - SV_LOOKUP_VARIANT, - ] - mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(v) for v in lookup_variants] - - url = f'{reverse(variant_lookup_handler)}?variantId=phase2_DEL_chr14_4640&genomeVersion=37&sampleType=WGS' - self.check_require_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant = { - **GCNV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], - 'genotypes': { - 'I0_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_sv_variant = { - **SV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], - 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_individuals = { - 'I0_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I0_F0_suffix_140608_DUP': { - 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', - 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, {'category': 'HP:0001626', 'label': '1 terms'}], - 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I1_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I1_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', - 'sex': 'M','features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I2_F0_phase2_DEL_chr14_4640': { - 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I2_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', - 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - } - expected_body = self._expected_lookup_body(expected_individuals,[expected_gcnv_variant, expected_sv_variant]) - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False, - ) - - self.login_manager() - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant.update({ - 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], - 'genotypes': GCNV_VARIANT4['genotypes'], - }) - expected_sv_variant.update({ - 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], - 'genotypes': SV_VARIANT4['genotypes'], - }) - expected_individuals = { - i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in [ - 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654', - ] - } - expected_body = self._expected_lookup_body( - expected_individuals, - [expected_gcnv_variant, expected_sv_variant], - project_guids=[PROJECT_GUID, 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000014_14'], - include_context=True, - ) - del expected_body['transcriptsById'] - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False,) @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_SECRET', 'abc123') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_ID', MOCK_CLIENT_ID) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 4af77065fb..ed5f4e9f61 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -767,677 +767,6 @@ def _get_list_param(call, param): GENE_DETAIL_FIELDS = {'notes', 'mgiMarkerId'} GENE_DETAIL_FIELDS.update(GENE_VARIANT_FIELDS) -VARIANTS = [ - { - 'alt': 'G', - 'ref': 'GAGA', - 'chrom': '21', - 'pos': 3343400, - 'xpos': 21003343400, - 'genomeVersion': '38', - 'liftedOverChrom': '21', - 'liftedOverPos': 3343353, - 'liftedOverGenomeVersion': '37', - 'variantId': '21-3343400-GAGA-G', - 'mainTranscriptId': 'ENST00000623083', - 'transcripts': { - 'ENSG00000227232': [ - { - 'aminoAcids': 'G/S', - 'geneSymbol': 'WASH7P', - 'biotype': 'protein_coding', - 'category': 'missense', - 'cdnaEnd': 1075, - 'cdnaStart': 1075, - 'codons': 'Ggt/Agt', - 'consequenceTerms': ['missense_variant'], - 'hgvs': 'ENSP00000485442.1:p.Gly359Ser', - 'hgvsc': 'ENST00000623083.3:c.1075G>A', - 'hgvsp': 'ENSP00000485442.1:p.Gly359Ser', - 'majorConsequence': 'missense_variant', - 'majorConsequenceRank': 11, - 'proteinId': 'ENSP00000485442', - 'transcriptId': 'ENST00000623083', - 'transcriptRank': 0 - } - ], - 'ENSG00000268903': [ - { - 'aminoAcids': 'G/S', - 'biotype': 'protein_coding', - 'category': 'missense', - 'cdnaEnd': 1338, - 'cdnaStart': 1338, - 'codons': 'Ggt/Agt', - 'consequenceTerms': ['missense_variant'], - 'geneId': 'ENSG00000268903', - 'hgvs': 'ENSP00000485351.1:p.Gly368Ser', - 'hgvsc': 'ENST00000624735.1:c.1102G>A', - 'hgvsp': 'ENSP00000485351.1:p.Gly368Ser', - 'majorConsequence': 'missense_variant', - 'majorConsequenceRank': 11, - 'proteinId': 'ENSP00000485351', - 'transcriptId': 'ENST00000624735', - 'transcriptRank': 1 - } - ] - }, - 'familyGuids': ['F000001_1', 'F000002_2'], - 'populations': { - 'callset': {'af': 0.13, 'ac': 4192, 'an': '32588'}, - 'gnomad_genomes': {'af': 0.007}, - }, - 'genotypes': { - 'NA19675': { - 'sampleId': 'NA19675', - 'ab': 0.702127659574, - 'gq': 46.0, - 'numAlt': 1, - 'dp': '50', - 'ad': '14,33', - 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'NA19679': { - 'sampleId': 'NA19679', - 'ab': 0.0, - 'gq': 99.0, - 'numAlt': 0, - 'dp': '45', - 'ad': '45,0', - 'filters': ['VQSRTrancheSNP99.95to100.00'], - } - } - }, - { - 'alt': 'A', - 'ref': 'AAAG', - 'chrom': '3', - 'pos': 835, - 'xpos': 3000000835, - 'genomeVersion': '37', - 'liftedOverGenomeVersion': '', - 'variantId': '3-835-AAAG-A', - 'transcripts': {}, - 'familyGuids': ['F000001_1'], - 'genotypes': { - 'NA19679': { - 'filters': ['artifact_prone_site'], - 'sampleId': 'NA19679', - 'ab': 0.0, - 'gq': 99.0, - 'numAlt': 0, - 'dp': '45', - 'ad': '45,0' - } - } - }, - { - 'alt': 'T', - 'ref': 'TC', - 'chrom': '12', - 'pos': 48367227, - 'xpos': 1248367227, - 'genomeVersion': '37', - 'liftedOverGenomeVersion': '', - 'variantId': '1-248367227-TC-T', - 'transcripts': {'ENSG00000233653': {}}, - 'familyGuids': ['F000002_2'], - 'genotypes': {} - } -] - -SINGLE_VARIANT = { - 'alt': 'A', - 'ref': 'G', - 'chrom': '1', - 'pos': 46394160, - 'xpos': 1046394160, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'variantId': '1-46394160-G-A', - 'transcripts': {'ENSG00000233653': {}}, - 'familyGuids': ['F000002_2'], - 'genotypes': {} -} - -TRANSCRIPT_1 = { - 'aminoAcids': 'LL/L', - 'biotype': 'protein_coding', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 10, - 'codons': 'ctTCTc/ctc', - 'geneSymbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'PROSITE_profiles:PS50850', - ], - 'canonical': 1, - 'transcriptRank': 0, - 'cdnaEnd': 421, - 'lofFilter': None, - 'hgvs': 'ENSP00000258436.5:p.Leu126del', - 'hgvsc': 'ENST00000258436.5:c.375_377delTCT', - 'cdnaStart': 419, - 'transcriptId': 'ENST00000258436', - 'proteinId': 'ENSP00000258436', - 'category': 'missense', - 'geneId': 'ENSG00000135953', - 'hgvsp': 'ENSP00000258436.5:p.Leu126del', - 'majorConsequence': 'inframe_deletion', - 'consequenceTerms': [ - 'inframe_deletion' - ] -} -TRANSCRIPT_2 = { - 'aminoAcids': 'P/X', - 'biotype': 'protein_coding', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 4, - 'codons': 'Ccc/cc', - 'geneSymbol': 'OR2M3', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Prints_domain:PR00237', - ], - 'canonical': 1, - 'transcriptRank': 0, - 'cdnaEnd': 897, - 'lofFilter': None, - 'hgvs': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'hgvsc': 'ENST00000456743.1:c.862delC', - 'cdnaStart': 897, - 'transcriptId': 'ENST00000456743', - 'proteinId': 'ENSP00000389625', - 'category': 'lof', - 'geneId': 'ENSG00000228198', - 'hgvsp': 'ENSP00000389625.1:p.Leu288SerfsTer10', - 'majorConsequence': 'frameshift_variant', - 'consequenceTerms': [ - 'frameshift_variant' - ] -} -TRANSCRIPT_3 = { - 'aminoAcids': 'LL/L', - 'biotype': 'nonsense_mediated_decay', - 'lof': None, - 'lofFlags': None, - 'majorConsequenceRank': 10, - 'codons': 'ctTCTc/ctc', - 'geneSymbol': 'MFSD9', - 'domains': [ - 'Transmembrane_helices:TMhelix', - 'Gene3D:1', - ], - 'canonical': None, - 'transcriptRank': 1, - 'cdnaEnd': 143, - 'lofFilter': None, - 'hgvs': 'ENSP00000413641.1:p.Leu48del', - 'hgvsc': 'ENST00000428085.1:c.141_143delTCT', - 'cdnaStart': 141, - 'transcriptId': 'ENST00000428085', - 'proteinId': 'ENSP00000413641', - 'category': 'missense', - 'geneId': 'ENSG00000135953', - 'hgvsp': 'ENSP00000413641.1:p.Leu48del', - 'majorConsequence': 'frameshift_variant', - 'consequenceTerms': [ - 'frameshift_variant', - 'inframe_deletion', - 'NMD_transcript_variant' - ] -} - -PARSED_VARIANTS = [ - { - 'alt': 'T', - 'chrom': '1', - 'bothsidesSupport': None, - 'clinvar': {'clinicalSignificance': 'Pathogenic/Likely_pathogenic', 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': '2023-03-05'}, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'familyGuids': ['F000003_3'], - 'cpxIntervals': None, - 'algorithms': None, - 'genotypes': { - 'I000007_na20870': { - 'ab': 1, 'ad': None, 'gq': 99, 'sampleId': 'NA20870', 'numAlt': 2, 'dp': 74, 'pl': None, - 'sampleType': 'WES', - } - }, - 'genomeVersion': '37', - 'genotypeFilters': '', - 'hgmd': {'accession': None, 'class': 'DM'}, - 'mainTranscriptId': TRANSCRIPT_3['transcriptId'], - 'selectedMainTranscriptId': None, - 'originalAltAlleles': ['T'], - 'populations': { - 'callset': {'an': 32, 'ac': 2, 'hom': 3, 'af': 0.063, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': 30946, 'ac': 4, 'hom': 0, 'af': 0.00012925741614425127, 'hemi': 0, - 'filter_af': 0.0004590314436538903, 'het': 0, 'id': None, 'max_hl': None}, - 'exac': {'an': 121308, 'ac': 8, 'hom': 0, 'af': 0.00006589, 'hemi': 0, 'filter_af': 0.0006726888333653661, - 'het': 0, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': 245930, 'ac': 16, 'hom': 0, 'af': 0.00006505916317651364, 'hemi': 0, - 'filter_af': 0.0009151523074911753, 'het': 0, 'id': None, 'max_hl': None}, - 'topmed': {'an': 125568, 'ac': 21, 'hom': 0, 'af': 0.00016724, 'hemi': 0, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'hom': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, 'hom': None, - 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': 32, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 248367227, - 'predictions': {'splice_ai': 0.75, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': 'D', - 'vest': '0.335', 'mut_pred': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': '25.9', 'primate_ai': None, - 'mpc': None, 'strvctvre': None, 'splice_ai_consequence': None, 'gnomad_noncoding': 1.01272,}, - 'ref': 'TC', - 'rsid': None, - 'screenRegionType': 'dELS', - 'transcripts': { - 'ENSG00000135953': [TRANSCRIPT_3], - 'ENSG00000228198': [TRANSCRIPT_2], - }, - 'variantId': '1-248367227-TC-T', - 'xpos': 1248367227, - 'end': None, - 'svType': None, - 'svTypeDetail': None, - 'numExon': None, - 'rg37LocusEnd': None, - '_sort': [1248367227], - }, - { - 'alt': 'G', - 'chrom': '2', - 'bothsidesSupport': None, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': '2023-03-05'}, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'familyGuids': ['F000002_2', 'F000003_3'], - 'cpxIntervals': None, - 'algorithms': None, - 'genotypes': { - 'I000004_hg00731': { - 'ab': 0, 'ad': None, 'gq': 99, 'sampleId': 'HG00731', 'numAlt': 2, 'dp': 67, 'pl': None, - 'sampleType': 'WES', - }, - 'I000005_hg00732': { - 'ab': 0, 'ad': None, 'gq': 96, 'sampleId': 'HG00732', 'numAlt': 1, 'dp': 42, 'pl': None, - 'sampleType': 'WES', - }, - 'I000006_hg00733': { - 'ab': 0, 'ad': None, 'gq': 96, 'sampleId': 'HG00733', 'numAlt': 0, 'dp': 42, 'pl': None, - 'sampleType': 'WES', - }, - 'I000007_na20870': { - 'ab': 0.70212764, 'ad': None, 'gq': 46, 'sampleId': 'NA20870', 'numAlt': 1, 'dp': 50, 'pl': None, - 'sampleType': 'WES', - } - }, - 'genotypeFilters': '', - 'genomeVersion': '37', - 'hgmd': {'accession': None, 'class': None}, - 'mainTranscriptId': TRANSCRIPT_1['transcriptId'], - 'selectedMainTranscriptId': TRANSCRIPT_2['transcriptId'], - 'originalAltAlleles': ['G'], - 'populations': { - 'callset': {'an': 32, 'ac': 1, 'hom': None, 'af': 0.031, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': 0, 'ac': 0, 'hom': 0, 'af': 0.0, 'hemi': 0, 'filter_af': None, 'het': 0, - 'id': None, 'max_hl': None}, - 'exac': {'an': 121336, 'ac': 6, 'hom': 0, 'af': 0.00004942, 'hemi': 0, 'filter_af': 0.000242306760358614, - 'het': 0, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': 245714, 'ac': 6, 'hom': 0, 'af': 0.000024418633044922146, 'hemi': 0, - 'filter_af': 0.00016269686320447742, 'het': 0, 'id': None, 'max_hl': None}, - 'topmed': {'an': 0, 'ac': 0, 'hom': 0, 'af': 0.0, 'hemi': 0, 'filter_af': None, 'het': None, 'id': None, - 'max_hl': None}, - 'sv_callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'hom': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': 32, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 103343353, - 'predictions': { - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, 'gnomad_noncoding': None, - 'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, 'polyphen': None, - 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': 1, 'vest': None, 'mut_pred': None, - 'mpc': None, 'strvctvre': None, 'splice_ai_consequence': None, - }, - 'ref': 'GAGA', - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000135953': [TRANSCRIPT_1], - 'ENSG00000228198': [TRANSCRIPT_2], - }, - 'variantId': '2-103343353-GAGA-G', - 'xpos': 2103343353, - 'end': None, - 'svType': None, - 'svTypeDetail': None, - 'numExon': None, - 'rg37LocusEnd': None, - '_sort': [2103343353], - }, -] - -PARSED_SV_VARIANT = { - 'alt': None, - 'chrom': '1', - 'bothsidesSupport': True, - 'familyGuids': ['F000002_2'], - 'cpxIntervals': None, - 'algorithms': None, - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'numAlt': -1, 'geneIds': ['ENSG00000228198'], - 'cn': 1, 'end': None, 'start': None, 'numExon': None, 'defragged': False, 'qs': 33, 'gq': None, - 'prevCall': False, 'prevOverlap': False, 'newCall': True, 'prevNumAlt': None, - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'numAlt': -1, 'sampleType': None, 'geneIds': None, 'gq': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'isRef': True, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'numAlt': -1, 'geneIds': None, 'gq': None, - 'cn': 2, 'end': 49045890, 'start': 49045987, 'numExon': 1, 'defragged': False, 'qs': 80, - 'prevCall': False, 'prevOverlap': True, 'newCall': False, 'prevNumAlt': None, - }, - }, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': None}, - 'hgmd': {'accession': None, 'class': None}, - 'genomeVersion': '37', - 'genotypeFilters': '', - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'originalAltAlleles': [], - 'populations': { - 'callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'exac': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'topmed': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': 10088, 'ac': 7, 'hom': None, 'af': 0.000693825, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': None, 'hemi': 0, 'hom': 0, 'het': 0, 'id': None, - 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 49045487, - 'predictions': {'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, 'gnomad_noncoding': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': None, - 'vest': None, 'mut_pred': None, 'mpc': None, 'strvctvre': 0.374, 'splice_ai_consequence': None}, - 'ref': None, - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000228198': [ - { - 'geneId': 'ENSG00000228198' - }, - ], - 'ENSG00000135953': [ - { - 'geneId': 'ENSG00000135953' - }, - ], - }, - 'variantId': 'prefix_19107_DEL', - 'xpos': 1049045487, - 'end': 49045899, - 'svType': 'INS', - 'svTypeDetail': None, - 'svSourceDetail': {'chrom': '9'}, - 'numExon': 2, - 'rg37LocusEnd': None, - '_sort': [1049045387], -} - -PARSED_SV_WGS_VARIANT = { - 'alt': None, - 'chrom': '2', - 'bothsidesSupport': None, - 'familyGuids': ['F000014_14'], - 'cpxIntervals': [{'chrom': '2', 'end': 3000, 'start': 1000, 'type': 'DUP'}, - {'chrom': '20', 'end': 13000, 'start': 11000, 'type': 'INV'}], - 'algorithms': 'wham, manta', - 'commonLowHeteroplasmy': None, - 'highConstraintRegion': None, - 'mitomapPathogenic': None, - 'genotypes': { - 'I000018_na21234': { - 'gq': 33, 'sampleId': 'NA21234', 'numAlt': 1, 'geneIds': None, - 'cn': -1, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': 'WGS', - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': 2, - }, - 'I000019_na21987': { - 'gq': None, 'sampleId': 'NA21987', 'numAlt': -1, 'isRef': True, 'geneIds': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': None, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - 'I000021_na21654': { - 'gq': None, 'sampleId': 'NA21654', 'numAlt': -1, 'isRef': True, 'geneIds': None, - 'cn': 2, 'end': None, 'start': None, 'numExon': None, 'defragged': None, 'qs': None, 'sampleType': None, - 'prevCall': None, 'prevOverlap': None, 'newCall': None, 'prevNumAlt': None, - }, - }, - 'clinvar': {'clinicalSignificance': None, 'alleleId': None, 'variationId': None, 'goldStars': None, 'version': None}, - 'hgmd': {'accession': None, 'class': None}, - 'genomeVersion': '38', - 'genotypeFilters': '', - 'liftedOverChrom': '2', - 'liftedOverGenomeVersion': '37', - 'liftedOverPos': 49272526, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'originalAltAlleles': [], - 'populations': { - 'callset': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'exac': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'topmed': {'an': None, 'ac': None, 'hom': None, 'af': None, 'hemi': None, 'filter_af': None, 'het': None, - 'id': None, 'max_hl': None}, - 'sv_callset': {'an': 10088, 'ac': 7, 'hom': None, 'af': 0.000693825, 'hemi': None, 'filter_af': None, - 'het': None, 'id': None, 'max_hl': None}, - 'gnomad_svs': {'ac': 22, 'af': 0.00679, 'an': 3240, 'filter_af': None, 'hemi': 0, 'hom': 0, 'het': 0, - 'id': 'gnomAD-SV_v2.1_BND_1_1', 'max_hl': None}, - 'gnomad_mito': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'helix_heteroplasmy': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - }, - 'pos': 49045387, - 'predictions': {'splice_ai': None, 'eigen': None, 'revel': None, 'mut_taster': None, 'fathmm': None, - 'vest': None, 'mut_pred': None, - 'hmtvar': None, 'apogee': None, 'haplogroup_defining': None, 'mitotip': None, - 'polyphen': None, 'dann': None, 'sift': None, 'cadd': None, 'primate_ai': None, - 'mpc': None, 'strvctvre': None, 'gnomad_noncoding': None, 'splice_ai_consequence': None}, - 'ref': None, - 'rsid': None, - 'screenRegionType': None, - 'transcripts': { - 'ENSG00000228198': [ - { - 'geneSymbol': 'OR4F5', - 'majorConsequence': 'DUP_PARTIAL', - 'geneId': 'ENSG00000228198' - }, - ], - 'ENSG00000228199': [ - { - 'geneId': 'ENSG00000228199', - 'geneSymbol': 'FBXO28', - 'majorConsequence': 'MSV_EXON_OVERLAP' - } - ], - 'ENSG00000228201': [ - { - 'geneId': 'ENSG00000228201', - 'geneSymbol': 'FAM131C', - 'majorConsequence': 'INTRAGENIC_EXON_DUP' - } - ] - }, - 'variantId': 'prefix_19107_CPX', - 'xpos': 2049045387, - 'end': 12345678, - 'endChrom': '20', - 'svType': 'CPX', - 'svTypeDetail': 'dupINV', - 'numExon': None, - 'rg37LocusEnd': {'contig': '20', 'position': 12326326}, - '_sort': [2049045387], -} - -PARSED_MITO_VARIANT = { - '_sort': [25000010195], - 'algorithms': None, - 'alt': 'A', - 'bothsidesSupport': None, - 'chrom': 'M', - 'clinvar': {'alleleId': None, 'clinicalSignificance': 'Likely_pathogenic', 'goldStars': None, 'variationId': None, 'version': None}, - 'commonLowHeteroplasmy': False, - 'cpxIntervals': None, - 'end': 10195, - 'familyGuids': ['F000002_2'], - 'genomeVersion': '37', - 'genotypeFilters': '', - 'genotypes': - {'I000004_hg00731': - {'contamination': 0.0, 'dp': 5139.0, 'gq': 60.0, 'hl': 1.0, 'mitoCn': 319.03225806451616, 'numAlt': 2, - 'sampleId': 'HG00731', 'sampleType': 'WES'}}, - 'hgmd': {'accession': None, 'class': None}, - 'highConstraintRegion': True, - 'mainTranscriptId': 'ENST00000361227', - 'mitomapPathogenic': True, - 'numExon': None, - 'originalAltAlleles': [], - 'populations': - {'callset': {'ac': 0, 'af': 0.0, 'an': 2520, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'exac': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_exomes': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_genomes': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito': {'ac': 1368, 'af': 0.024246292, 'an': 56421, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None,'max_hl': None}, - 'gnomad_svs': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'helix': {'ac': 1312, 'af': 0.0033268193, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'sv_callset': {'ac': None, 'af': None, 'an': None, 'filter_af': None, - 'hemi': None, 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'topmed': {'ac': None, 'af': None, 'an': None, 'filter_af': None, 'hemi': None, - 'het': None, 'hom': None, 'id': None, 'max_hl': None}, - 'callset_heteroplasmy': {'ac': 1, 'af': 0.0003968253968253968, 'an': 2520, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': None}, - 'gnomad_mito_heteroplasmy': {'ac': 3, 'af': 5.317169e-05, 'an': 56421, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': 1.0}, - 'helix_heteroplasmy': {'ac': 5, 'af': 4.081987e-05, 'an': None, 'filter_af': None, 'hemi': None, 'het': None, - 'hom': None, 'id': None, 'max_hl': 0.90441}, - }, - 'pos': 10195, - 'predictions': {'hmtvar': 0.71, 'apogee': 0.42, 'cadd': None, 'dann': None, 'eigen': None, 'fathmm': 'T', - 'haplogroup_defining': None, 'mitotip': None, 'mpc': None, 'mut_taster': 'N', 'polyphen': None, - 'primate_ai': None, 'revel': None, 'sift': 'D', 'splice_ai': None, 'splice_ai_consequence': None, - 'vest': None, 'mut_pred': None, 'strvctvre': None, 'gnomad_noncoding': None,}, - 'ref': 'C', - 'rg37LocusEnd': None, - 'rsid': None, - 'screenRegionType': None, - 'selectedMainTranscriptId': None, - 'svType': None, - 'svTypeDetail': None, - 'transcripts': {'ENSG00000198840': [ - {'aminoAcids': 'P/H', 'biotype': 'protein_coding', 'canonical': 1, 'category': 'missense', 'cdnaEnd': 137, - 'cdnaStart': 137, 'codons': 'cCc/cAc', 'consequenceTerms': [ - 'missense_variant'], - 'domains': ['Gene3D:1', 'ENSP_mappings:5xtc', 'ENSP_mappings:5xtd', 'Pfam:PF00507', 'PANTHER:PTHR11058', - 'PANTHER:PTHR11058'], 'geneId': 'ENSG00000198840', 'geneSymbol': 'MT-ND3', 'hgvs': 'p.Pro46His', - 'hgvsc': 'ENST00000361227.2:c.137C>A', 'hgvsp': 'ENSP00000355206.2:p.Pro46His', - 'majorConsequence': 'missense_variant', 'majorConsequenceRank': 11, - 'polyphenPrediction': 'probably_damaging', 'proteinId': 'ENSP00000355206', 'proteinStart': 46, - 'siftPrediction': 'deleterious_low_confidence', 'transcriptId': 'ENST00000361227', 'transcriptRank': 0}]}, - 'variantId': 'M-10195-C-A', - 'xpos': 25000010195 -} - -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT = deepcopy(PARSED_VARIANTS) -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT[1].update({ - 'familyGuids': ['F000003_3'], - 'mainTranscriptId': TRANSCRIPT_2['transcriptId'], - 'selectedMainTranscriptId': None, -}) -PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT[1]['transcripts']['ENSG00000135953'][0]['majorConsequence'] = 'frameshift_variant' -for variant in PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT: - variant['_sort'][0] += 100 - variant['familyGuids'].append('F000011_11') - variant['genotypes'].update({ - 'I000015_na20885': { - 'ab': 0.631, 'ad': None, 'gq': 99, 'sampleId': 'NA20885', 'numAlt': 1, 'dp': 50, 'pl': None, - 'sampleType': 'WES', - }, - }) - GOOGLE_API_TOKEN_URL = 'https://oauth2.googleapis.com/token' # nosec GOOGLE_ACCESS_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' # nosec From b22d12813a042d48ce13a9458656429edd1d2474 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Mar 2026 12:03:45 -0400 Subject: [PATCH 132/252] more variant lookup clean up --- clickhouse_search/search_tests.py | 284 ++++++++++---------------- seqr/views/apis/variant_search_api.py | 2 +- 2 files changed, 110 insertions(+), 176 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 51412df1c4..531281a670 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -39,9 +39,7 @@ SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant -from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.apis.data_manager_api import trigger_delete_project -from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ FAMILY_NOTE_FIELDS, GENE_VARIANT_DISPLAY_FIELDS, LOCUS_LIST_FIELDS @@ -139,9 +137,12 @@ def set_cache(self, cache_key, cached): def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' - self.mock_redis.set.assert_called_with(cache_key, mock.ANY) - self.assertEqual(json.loads(self.mock_redis.set.call_args.args[1]), expected_results) - self.mock_redis.expire.assert_called_with(cache_key, timedelta(weeks=2)) + self.mock_redis.set.assert_any_call(cache_key, mock.ANY) + call_index = next(i for i, call in enumerate(self.mock_redis.set.call_args_list) if call.args[0] == cache_key) + self.assertEqual(json.loads(self.mock_redis.set.call_args_list[call_index].args[1]), expected_results) + self.mock_redis.expire.assert_any_call(cache_key, timedelta(weeks=2)) + self.mock_redis.set.reset_mock() + self.mock_redis.expire.reset_mock() def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs): search_hash = search_hash or random.randint(1000, 10000) # nosec @@ -226,20 +227,6 @@ def _assert_expected_search_error(self, error, **kwargs): self.assertDictEqual(response.json(), {'error': error}) return search_hash - def _assert_expected_variants(self, variants, expected_results, cache_key=None, results_page=None, format_cached_variants=None, sort='xpos', **kwargs): - encoded_variants = self._encode_variants(variants) - self.assertListEqual(encoded_variants, results_page or expected_results) - if cache_key: - cached_variants = format_cached_variants(expected_results, **kwargs) if format_cached_variants else expected_results - self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) - else: - self.mock_redis.get.assert_not_called() - self.mock_redis.set.assert_not_called() - - @staticmethod - def _encode_variants(variants): - return json.loads(json.dumps(variants, cls=DjangoJSONEncoderWithSets)) - def _format_cached_variants(self, variants, cached_variant_fields=None): cached_variants = [ self._get_cached_variant(variant, (cached_variant_fields[i] if cached_variant_fields else None)) @@ -984,7 +971,7 @@ def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] - url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38&affectedOnly=true' + url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38' self.check_require_login(url) response = self.client.get(url) @@ -1046,84 +1033,10 @@ def test_variant_lookup(self, mock_liftover): } expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None, - affected_only=True, hom_only=False) - self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT], 'variant_lookup_results__1-10439-AC-A__38') - + self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key='variant_lookup_results__1-10439-AC-A__38') - response_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_variant['transcripts'] = VARIANTS[0]['transcripts'] - expected_gene_body = { - 'genesById': {'ENSG00000227232': EXPECTED_GENE, 'ENSG00000268903': EXPECTED_GENE}, - } - expected_body.update(expected_gene_body) - # TODO test these for real - EXPECTED_LIRICAL_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, - 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, - ] - EXPECTED_EXOMISER_DATA = [ - {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, - 'scores': {'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, - 'variant_score': 0.999200702}}, - {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, - 'scores': {'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, - 'variant_score': 1}}, - ] - EXPECTED_SEARCH_RESPONSE = { - 'searchedVariants': ALL_VARIANTS, - 'savedVariantsByGuid': { - 'SV0000001_2103343353_r0390_100': mock.ANY, - 'SV0000002_1248367227_r0390_100': mock.ANY, - }, - 'genesById': { - 'ENSG00000227232': expected_pa_gene, 'ENSG00000268903': EXPECTED_GENE, 'ENSG00000233653': EXPECTED_GENE, - 'ENSG00000177000': mock.ANY, 'ENSG00000097046': mock.ANY, - }, - 'search': { - 'search': SEARCH, - 'projectFamilies': [{'projectGuid': PROJECT_GUID, 'familyGuids': mock.ANY}], - 'totalResults': 5, - }, - 'variantTagsByGuid': { - 'VT1708633_2103343353_r0390_100': EXPECTED_TAG, 'VT1726945_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726970_2103343353_r0004_tes': EXPECTED_TAG, 'VT1726961_2103343353_r0390_100': EXPECTED_TAG, - 'VT1726985_2103343353_r0390_100': expected_aip_tag, - }, - 'variantNotesByGuid': { - 'VN0714935_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - 'VN0714937_2103343353_r0390_100': {k: mock.ANY for k in VARIANT_NOTE_FIELDS}, - }, - 'variantFunctionalDataByGuid': { - 'VFD0000023_1248367227_r0390_10': expected_functional_tag, - 'VFD0000024_1248367227_r0390_10': expected_functional_tag, - 'VFD0000025_1248367227_r0390_10': expected_functional_tag, - 'VFD0000026_1248367227_r0390_10': expected_functional_tag, - }, - 'locusListsByGuid': {LOCUS_LIST_GUID: {'intervals': mock.ANY}}, - 'rnaSeqData': { - 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, - 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, - }, - 'phenotypeGeneScores': { - 'I000001_na19675': {'ENSG00000268903': {'exomiser': EXPECTED_EXOMISER_DATA}}, - 'I000002_na19678': {'ENSG00000268903': {'lirical': EXPECTED_LIRICAL_DATA}}, - }, - 'mmeSubmissionsByGuid': {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}, - 'familiesByGuid': {'F000001_1': {'tpmGenes': ['ENSG00000227232']}}, - 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3}}, - } - if 'transcriptsById' in EXPECTED_SEARCH_RESPONSE: - expected_body['transcriptsById'] = EXPECTED_SEARCH_RESPONSE['transcriptsById'] - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), expected_body) - - response_variant['variantId'] = '1-248367227-TC-T' - response_variant['genomeVersion'] = '37' self.login_manager() - response = self.client.get(url.replace("38", "37") + '&homOnly=true') + response = self.client.get(url) self.assertEqual(response.status_code, 200) individual_guid_map = [ @@ -1145,8 +1058,6 @@ def test_variant_lookup(self, mock_liftover): 'individualGuid': individual_guid, } for individual_guid, anon_individual_guid, genotype in individual_guid_map }, - 'genomeVersion': '37', - 'variantId': '1-248367227-TC-T', }) expected_individuals = { individual_guid: { @@ -1159,59 +1070,77 @@ def test_variant_lookup(self, mock_liftover): {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) expected_body = self._expected_lookup_body( expected_individuals, [expected_variant], include_context=True, - project_guids=[PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], - ) - expected_body.update({ - k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in - {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} - }) - expected_body['savedVariantsByGuid'] = { - k: v for k, v in EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if - k in ['SV0000002_1248367227_r0390_100'] - } - expected_body.update(expected_gene_body) - expected_body['genesById']['ENSG00000227232'] = expected_pa_gene - expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop( - 'MS000001_na19675') - expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY - expected_body['variantTagsByGuid']['VT1726970_2103343353_r0004_tes'] = EXPECTED_TAG - expected_body['variantTagsByGuid']['VT1726961_2103343353_r0005_tes'] = EXPECTED_TAG - for k in ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100']: - del expected_body['variantTagsByGuid'][k] - expected_body['rnaSeqData']['I000019_na21987'] = {'outliers': {}, - 'spliceOutliers': {'ENSG00000268903': mock.ANY}} - + project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], is_build_38=True, + ) + # TODO test variant with search context! + # expected_body.update({ + # 'mmeSubmissionsByGuid': {'MS000018_P0004517': mock.ANY}, + # 'rnaSeqData': {'I000019_na21987': {'outliers': {}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}}, + # 'savedVariantsByGuid': { + # 'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0004_non': mock.ANY, + # }, + # 'variantTagsByGuid': { + # 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726970_2103343353_r0004_tes': mock.ANY, + # 'VT1726985_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0005_tes': mock.ANY, + # }, + # 'variantNotesByGuid': { + # 'VN0714935_2103343353_r0390_100': mock.ANY, + # 'VN0714937_2103343353_r0390_100': mock.ANY, + # }, + # }) self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.manager_user, '1-10439-AC-A', '37', sample_type=None, affected_only=True, hom_only=True, - ) - self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT], 'variant_lookup_results__1-10439-AC-A__38') + self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key='variant_lookup_results__1-10439-AC-A__38') - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, '1-91511686-TCA-G', '38') - self.assertEqual(str(cm.exception), 'Variant not present in seqr') + url = url.replace('1-10439-AC-A', '1-91511686-TCA-G') + response = self.client.get(url) + self.assertEqual(response.status_code, 404) + self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [VARIANT_LOOKUP_VARIANT]) - cached_variants = variant_lookup(self.user, '1-91511686-TCA-G', '38') - self.assertListEqual([VARIANT_LOOKUP_VARIANT], cached_variants) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), expected_body) - variants = variant_lookup(self.user, '7-143270172-A-G', '37') + url_37 = url.replace('1-91511686-TCA-G', '7-143270172-A-G').replace('38', '37') + response = self.client.get(url_37) + self.assertEqual(response.status_code, 200) grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() ], key=lambda x: x['sampleId'], reverse=True)}, } + expected_grch37_variant = { + **GRCH37_VARIANT, + 'familyGuids': [], + 'lookupFamilyGuids': GRCH37_VARIANT['familyGuids'], + } + grch37_individuals = {guid: mock.ANY for guid in ['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731']} + expected_body = self._expected_lookup_body( + grch37_individuals, [expected_grch37_variant], include_context=True, + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + ) + self.assertDictEqual(response.json(), expected_body) cache_key = 'variant_lookup_results__7-143270172-A-G__37' - self._assert_expected_variants(variants, [grch37_lookup_variant], cache_key) + self.assert_cached_results([grch37_lookup_variant], cache_key=cache_key) - variants = variant_lookup(self.user, '7-143270172-A-G', '37', hom_only=True, affected_only=True) - self._assert_expected_variants(variants, [grch37_lookup_variant], f'{cache_key}__affected__hom') + response = self.client.get(url_37 + '&homOnly=true&affectedOnly=true') + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), expected_body) + self.assert_cached_results([grch37_lookup_variant], cache_key=f'{cache_key}__affected__hom') # Lookup works if variant is only present on a different build - variants = variant_lookup(self.user, '7-143260172-A-G', '38') - self._assert_expected_variants(variants, [grch37_lookup_variant], 'variant_lookup_results__7-143260172-A-G__38') + url = url.replace('1-91511686-TCA-G', '7-143260172-A-G') + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + expected_body = self._expected_lookup_body( + grch37_individuals, [expected_grch37_variant], include_context=True, + project_guids=['R0001_1kg'], family_guids=['F000002_2'], is_build_38=True, + ) + expected_body['genesById'] = {'ENSG00000176227': mock.ANY, 'ENSG00000271079': mock.ANY} + self.assertDictEqual(response.json(), expected_body) + self.assert_cached_results([grch37_lookup_variant], cache_key='variant_lookup_results__7-143260172-A-G__38') mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) @@ -1225,7 +1154,7 @@ def test_variant_lookup(self, mock_liftover): del liftover_variant['liftedFamilyGuids'] variants = variant_lookup(self.user, '1-439-AC-A', '37') cache_key = 'variant_lookup_results__1-439-AC-A__37' - self._assert_expected_variants(variants, [liftover_variant], cache_key) + self.assert_cached_results([liftover_variant], cache_key=cache_key) mock_liftover.assert_called_with('hg19', 'hg38') mock_convert_coordinate.assert_called_with('chr1', 439) @@ -1237,17 +1166,18 @@ def test_variant_lookup(self, mock_liftover): }, } variants = variant_lookup(self.user, '1-10439-AC-A', '38', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant], 'variant_lookup_results__1-10439-AC-A__38__hom') + self.assert_cached_results([hom_only_lookup_variant], cache_key='variant_lookup_results__1-10439-AC-A__38__hom') variants = variant_lookup(self.user, '1-439-AC-A', '37', hom_only=True) - self._assert_expected_variants(variants, [hom_only_lookup_variant], f'{cache_key}__hom') + self.assert_cached_results([hom_only_lookup_variant], cache_key=f'{cache_key}__hom') variants = variant_lookup(self.user, 'M-4429-G-A', '38') - self._assert_expected_variants(variants, [{ + self.assert_cached_results([{ **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() ]}, - }], 'variant_lookup_results__M-4429-G-A__38') + }], cache_key='variant_lookup_results__M-4429-G-A__38') + with self.assertRaises(ObjectDoesNotExist) as cm: variant_lookup(self.user, 'M-4429-G-A', '38', hom_only=True) @@ -1369,7 +1299,7 @@ def test_variant_lookup(self, mock_liftover): variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT], cache_key) + self.assert_cached_results([SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT], cache_key=cache_key) affected_only_lookup_variant = { **GCNV_LOOKUP_VARIANT, @@ -1378,56 +1308,42 @@ def test_variant_lookup(self, mock_liftover): }, } variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', affected_only=True) - self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, affected_only_lookup_variant], f'{cache_key}__affected') + self.assert_cached_results([SV_LOOKUP_VARIANT, affected_only_lookup_variant], cache_key=f'{cache_key}__affected') # reciprocal overlap does not meet the threshold for smaller events variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') cache_key = 'variant_lookup_results__suffix_140608_DUP__38' - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT], cache_key) + self.assert_cached_results([GCNV_LOOKUP_VARIANT], cache_key=cache_key) variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES', affected_only=True) - self._assert_expected_variants(variants, [affected_only_lookup_variant], f'{cache_key}__affected') + self.assert_cached_results([affected_only_lookup_variant], cache_key=f'{cache_key}__affected') variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') - self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3], 'variant_lookup_results__suffix_140593_DUP__38') + self.assert_cached_results([GCNV_LOOKUP_VARIANT_3], cache_key='variant_lookup_results__suffix_140593_DUP__38') def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): - exclude_keys = {'searchedVariants', 'search', 'transcriptsById'} + exclude_keys = set() if not include_context: exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) - EXPECTED_SEARCH_FAMILY_CONTEXT = { - 'familiesByGuid': {'F000001_1': mock.ANY, 'F000002_2': mock.ANY}, - 'individualsByGuid': mock.ANY, - 'igvSamplesByGuid': mock.ANY, - 'locusListsByGuid': {LOCUS_LIST_GUID: mock.ANY}, - 'familyNotesByGuid': mock.ANY, - } - PROJECT_TAG_TYPE_FIELDS = {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'} + context_fields = {*SEARCH_RESPONSE_KEYS, 'igvSamplesByGuid', 'locusListsByGuid', 'familyNotesByGuid'} - exclude_keys + if is_build_38: + context_fields.add('omimIntervals') expected_body = { - **{k: {} for k in EXPECTED_SEARCH_RESPONSE if k not in exclude_keys}, - **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, + **{k: {} for k in context_fields}, 'projectsByGuid': { - p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids or [] - }, - 'familiesByGuid': { - f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids or [] + p: {k: mock.ANY for k in {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'}} + for p in project_guids or [] }, + 'familiesByGuid': {f: mock.ANY for f in family_guids or []}, 'individualsByGuid': individuals_by_guid, + 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if is_build_38 else {}, 'variants': variants, } - if is_build_38: - expected_body['omimIntervals'] = {} if include_context: - EXPECTED_TRANSCRIPTS_RESPONSE = { - 'transcriptsById': { - 'ENST00000624735': {'isManeSelect': False, 'refseqId': None, 'transcriptId': 'ENST00000624735'}}, - } + # TODO clean up? expected_body.update({ - **EXPECTED_TRANSCRIPTS_RESPONSE, - 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], + 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, }) - if 'totalSampleCounts' in EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = {'SV': {'WGS': 3}} if is_build_38 else EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] return expected_body def test_get_single_variant(self): @@ -2314,10 +2230,28 @@ def test_search_context(self): self.assertDictEqual(response_json['mmeSubmissionsByGuid'], {'MS000001_na19675': {k: mock.ANY for k in MATCHMAKER_SUBMISSION_FIELDS}}) self.assertDictEqual(response_json['omimIntervals'], {}) self.assertDictEqual(response_json['phenotypeGeneScores'],{ - 'I000001_na19675': {'ENSG00000268903': mock.ANY}, - 'I000002_na19678': {'ENSG00000268903': mock.ANY}, - 'I000004_hg00731': {'ENSG00000177000': mock.ANY}, - 'I000005_hg00732': {'ENSG00000177000': mock.ANY}, + 'I000001_na19675': {'ENSG00000268903': {'exomiser': [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 2, 'scores': { + 'exomiser_score': 0.969347946, 'phenotype_score': 0.443567539, 'variant_score': 0.999200702, + }}, + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 1, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, + 'I000002_na19678': {'ENSG00000268903': {'lirical': [ + {'diseaseId': 'OMIM:219800', 'diseaseName': 'Cystinosis, nephropathic', 'rank': 1, + 'scores': {'compositeLR': 0.003, 'post_test_probability': 0}}, + ]}}, + 'I000004_hg00731': {'ENSG00000177000': {'exomiser': [ + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 5, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, + 'I000005_hg00732': {'ENSG00000177000': {'exomiser': [ + {'diseaseId': 'OMIM:618460', 'diseaseName': 'Khan-Khan-Katsanis syndrome', 'rank': 3, 'scores': { + 'exomiser_score': 0.977923765, 'phenotype_score': 0.603998205, 'variant_score': 1, + }}, + ]}}, }) self.assertDictEqual(response_json['rnaSeqData'], { 'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}, diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 4d6ba59ae1..fd86ce0e35 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -617,7 +617,7 @@ def _update_lookup_variant(variant, response, individual_guid_map): genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype - for i, (unmapped_family_guid, genotypes) in enumerate(variant.pop('familyGenotypes').items()): + for i, (unmapped_family_guid, genotypes) in enumerate(sorted(variant.pop('familyGenotypes').items())): family_guid = f'F{i}_{variant["variantId"]}' variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): From 74ea1f8162bf0e1b2478e8c3074f2a4626b67025 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 12 Mar 2026 16:12:35 -0400 Subject: [PATCH 133/252] better helper function --- clickhouse_search/search_tests.py | 184 ++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 63 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 531281a670..e160c413bd 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,3 +1,4 @@ +from collections import defaultdict from datetime import timedelta from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist @@ -972,7 +973,7 @@ def test_variant_lookup(self, mock_liftover): mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38' - self.check_require_login(url) + self.check_require_login(reverse(variant_lookup_handler)) response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -1033,7 +1034,8 @@ def test_variant_lookup(self, mock_liftover): } expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) self.assertDictEqual(response.json(), expected_body) - self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key='variant_lookup_results__1-10439-AC-A__38') + cache_key = 'variant_lookup_results__1-10439-AC-A__38' + self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key=cache_key) self.login_manager() response = self.client.get(url) @@ -1090,7 +1092,38 @@ def test_variant_lookup(self, mock_liftover): # }, # }) self.assertDictEqual(response.json(), expected_body) - self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key='variant_lookup_results__1-10439-AC-A__38') + self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key=cache_key) + + hom_only_lookup_variant = { + **VARIANT_LOOKUP_VARIANT, + 'familyGenotypes': { + 'F000002_2': [gt for gt in VARIANT_LOOKUP_VARIANT['familyGenotypes']['F000002_2'] if gt['sampleType'] == 'WGS'], + 'F000011_11': VARIANT_LOOKUP_VARIANT['familyGenotypes']['F000011_11'], + }, + } + del hom_only_lookup_variant['liftedFamilyGuids'] + hom_only_expected_variant = { + **expected_variant, + 'lookupFamilyGuids': expected_variant['lookupFamilyGuids'][:-1], + 'genotypes': { + **expected_variant['genotypes'], + **{guid: next(gt for gt in expected_variant['genotypes'][guid] if gt['sampleType'] == 'WGS') + for guid in ['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733']}, + } + } + del hom_only_expected_variant['genotypes']['I000018_na21234'] + del hom_only_expected_variant['liftedFamilyGuids'] + expected_individuals = {guid: individual for guid, individual in expected_individuals.items() if guid not in { + 'I000018_na21234', 'I000019_na21987', 'I000021_na21654', + }} + hom_only_expected_body = self._expected_lookup_body( + expected_individuals, [hom_only_expected_variant], include_context=True, + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], is_build_38=True, + ) + response = self.client.get(url + '&homOnly=true') + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), hom_only_expected_body) + self.assert_cached_results([hom_only_lookup_variant], cache_key=f'{cache_key}__hom') url = url.replace('1-10439-AC-A', '1-91511686-TCA-G') response = self.client.get(url) @@ -1102,82 +1135,63 @@ def test_variant_lookup(self, mock_liftover): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), expected_body) - url_37 = url.replace('1-91511686-TCA-G', '7-143270172-A-G').replace('38', '37') - response = self.client.get(url_37) - self.assertEqual(response.status_code, 200) - grch37_lookup_variant = { - **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() - ], key=lambda x: x['sampleId'], reverse=True)}, - } - expected_grch37_variant = { - **GRCH37_VARIANT, - 'familyGuids': [], - 'lookupFamilyGuids': GRCH37_VARIANT['familyGuids'], - } - grch37_individuals = {guid: mock.ANY for guid in ['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731']} - expected_body = self._expected_lookup_body( - grch37_individuals, [expected_grch37_variant], include_context=True, + cache_key = 'variant_lookup_results__7-143270172-A-G__37' + self._assert_expected_lookup( + '7-143270172-A-G', GRCH37_VARIANT, cache_key, genome_version='37', project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] ) - self.assertDictEqual(response.json(), expected_body) - cache_key = 'variant_lookup_results__7-143270172-A-G__37' - self.assert_cached_results([grch37_lookup_variant], cache_key=cache_key) - response = self.client.get(url_37 + '&homOnly=true&affectedOnly=true') - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), expected_body) - self.assert_cached_results([grch37_lookup_variant], cache_key=f'{cache_key}__affected__hom') + self._assert_expected_lookup( + '7-143270172-A-G', GRCH37_VARIANT, f'{cache_key}__affected__hom', genome_version='37', + hom_only=True, affected_only=True, project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) # Lookup works if variant is only present on a different build - url = url.replace('1-91511686-TCA-G', '7-143260172-A-G') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_body = self._expected_lookup_body( - grch37_individuals, [expected_grch37_variant], include_context=True, - project_guids=['R0001_1kg'], family_guids=['F000002_2'], is_build_38=True, + self._assert_expected_lookup( + '7-143260172-A-G', GRCH37_VARIANT, 'variant_lookup_results__7-143260172-A-G__38', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'], genesById={ + 'ENSG00000176227': mock.ANY, 'ENSG00000271079': mock.ANY, + }, ) - expected_body['genesById'] = {'ENSG00000176227': mock.ANY, 'ENSG00000271079': mock.ANY} - self.assertDictEqual(response.json(), expected_body) - self.assert_cached_results([grch37_lookup_variant], cache_key='variant_lookup_results__7-143260172-A-G__38') mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) + # TODO do as part of VARIANT_LOOKUP_VARIANT creation liftover_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGenotypes': { - family_guid: gts - for family_guid, gts in VARIANT_LOOKUP_VARIANT['familyGenotypes'].items() if family_guid != 'F000014_14' + **VARIANT1, + 'familyGuids': [VARIANT1['familyGuids'][0], 'F000011_11'], + 'genotypes': { + **VARIANT1_BOTH_SAMPLE_TYPES['genotypes'], + 'I000015_na20885': [ + {**PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], 'sampleType': 'WES'}, + PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], + ], }, } - del liftover_variant['liftedFamilyGuids'] - variants = variant_lookup(self.user, '1-439-AC-A', '37') - cache_key = 'variant_lookup_results__1-439-AC-A__37' - self.assert_cached_results([liftover_variant], cache_key=cache_key) + self._assert_expected_lookup( + '1-439-AC-A', liftover_variant, 'variant_lookup_results__1-439-AC-A__37', genome_version='37', + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + ) mock_liftover.assert_called_with('hg19', 'hg38') mock_convert_coordinate.assert_called_with('chr1', 439) - hom_only_lookup_variant = { - **liftover_variant, - 'familyGenotypes': { - **liftover_variant['familyGenotypes'], - 'F000002_2': [gt for gt in liftover_variant['familyGenotypes']['F000002_2'] if gt['sampleType'] == 'WGS'], - }, - } - variants = variant_lookup(self.user, '1-10439-AC-A', '38', hom_only=True) - self.assert_cached_results([hom_only_lookup_variant], cache_key='variant_lookup_results__1-10439-AC-A__38__hom') - variants = variant_lookup(self.user, '1-439-AC-A', '37', hom_only=True) - self.assert_cached_results([hom_only_lookup_variant], cache_key=f'{cache_key}__hom') - - variants = variant_lookup(self.user, 'M-4429-G-A', '38') - self.assert_cached_results([{ - **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() - ]}, - }], cache_key='variant_lookup_results__M-4429-G-A__38') + self._assert_expected_lookup( + '1-439-AC-A', { + **liftover_variant, 'genotypes': hom_only_expected_variant['genotypes'], + }, 'variant_lookup_results__1-439-AC-A__37__hom', genome_version='37', hom_only=True, + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + ) + self._assert_expected_lookup( + 'M-4429-G-A', MITO_VARIANT1, 'variant_lookup_results__M-4429-G-A__38', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733',], + ) with self.assertRaises(ObjectDoesNotExist) as cm: variant_lookup(self.user, 'M-4429-G-A', '38', hom_only=True) @@ -1321,6 +1335,50 @@ def test_variant_lookup(self, mock_liftover): variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') self.assert_cached_results([GCNV_LOOKUP_VARIANT_3], cache_key='variant_lookup_results__suffix_140593_DUP__38') + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, **kwargs): + url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' + if hom_only: + url += '&homOnly=true' + if affected_only: + url += '&affectedOnly=true' + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + context_fields = {*SEARCH_RESPONSE_KEYS, 'igvSamplesByGuid', 'locusListsByGuid', 'familyNotesByGuid'} + if genome_version == '38': + context_fields.add('omimIntervals') + expected_body = { + **{k: {} for k in context_fields}, + 'projectsByGuid': {p: mock.ANY for p in project_guids or []}, + 'familiesByGuid': {f: mock.ANY for f in family_guids or []}, + 'individualsByGuid': {i: mock.ANY for i in individual_guids or []}, + 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, + 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, + 'variants': [{ + **variant, + 'familyGuids': [], + 'lookupFamilyGuids': variant['familyGuids'], + }], + **kwargs, + } + self.assertDictEqual(response.json(), expected_body) + + family_genotypes = defaultdict(list) + for gts in variant['genotypes'].values(): + if not isinstance(gts, list): + gts = [gts] + for gt in gts: + family_genotypes[gt['familyGuid']].append({k: v for k, v in gt.items() if k != 'individualGuid'}) + lookup_variant = { + **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, + 'familyGenotypes': { + family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x['sampleId']), reverse=True) + for family_guid, gts in family_genotypes.items() + }, + } + self.assert_cached_results([lookup_variant], cache_key=cache_key) + def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): exclude_keys = set() if not include_context: From e0fe5bbd9c49a9fd96b9e3e22fab28d97b32a843 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 09:27:05 -0400 Subject: [PATCH 134/252] sv lookup test --- .../fixtures/clickhouse_search.json | 8 +- clickhouse_search/search_tests.py | 345 +++++++++--------- 2 files changed, 170 insertions(+), 183 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index d7e8bf15d1..de16fe36db 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -1454,9 +1454,9 @@ "geneId_ids": [76], "sign": 1, "calls": [ - ["NA21234", 2, 0, 99, false, true, null], ["NA21987", 1, 1, 99, false, false, 2], - ["NA21654", 1, 1, 99, false, false, 2] + ["NA21654", 1, 1, 99, false, false, 2], + ["NA21234", 2, 0, 99, false, true, null] ] } }, { @@ -1625,9 +1625,9 @@ "filters": [], "sign": 1, "calls": [ - ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false], + ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false], ["HG00732", null, null, null, null, null, null, null, [], null, null, null], - ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] + ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] ] } }, { diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e160c413bd..a509ee6113 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,7 +1,6 @@ from collections import defaultdict from datetime import timedelta from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist from django.core.management import call_command from django.db import connections from django.urls.base import reverse @@ -24,7 +23,7 @@ TopmedGRCh37Dict, GnomadmitoMv, GnomadmitoDict, GnomadmitoheteroplasmyMv, GnomadmitoheteroplasmyDict, HelixmitoMv, \ HelixmitoDict, HelixmitoheteroplasmyMv, HelixmitoheteroplasmyDict, ScreenDict, MitomapMv, MitomapDict, Absplice2Mv, \ Absplice2Dict, PromoterAIMv, PromoterAIDict, PextSnvIndelMv, PextSnvIndelDict, PextMitoMv, PextMitoDict -from clickhouse_search.models.search_models import EntriesSnvIndel, VariantsSnvIndel +from clickhouse_search.models.search_models import EntriesSnvIndel, VariantsSnvIndel, EntriesSv, EntriesGcnv from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, CACHED_CONSEQUENCES_BY_KEY, \ VARIANT_ID_SEARCH, VARIANT_IDS, LOCATION_SEARCH, GENE_IDS, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT, \ SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, COMP_HET_ALL_PASS_FILTERS, \ @@ -971,16 +970,31 @@ def test_invalid_search(self): def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] + EntriesSv.objects.using('clickhouse_write').filter(family_guid='F000002_2_x').delete() + EntriesGcnv.objects.using('clickhouse_write').filter(family_guid='F000002_2_x').delete() - url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38' self.check_require_login(reverse(variant_lookup_handler)) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) + # TODO do with/ replace VARIANT_LOOKUP_VARIANT + lookup_variant = { + **VARIANT1, + 'liftedFamilyGuids': ['F000014_14'], + 'familyGuids': [VARIANT1['familyGuids'][0], 'F000011_11'], + 'genotypes': { + **VARIANT1_BOTH_SAMPLE_TYPES['genotypes'], + 'I000015_na20885': [ + {**PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], 'sampleType': 'WES'}, + PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], + ], + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', + 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531, 'filters': [], 'individualGuid': 'I000018_na21234', + }, + }, + } expected_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A', 'F2_1-10439-AC-A'], + **VARIANT1, + 'familyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A'], 'liftedFamilyGuids': ['F2_1-10439-AC-A'], 'genotypes': { 'I0_F0_1-10439-AC-A': [ @@ -1003,7 +1017,6 @@ def test_variant_lookup(self, mock_liftover): 'sampleType': 'WGS'}, }, } - del expected_variant['familyGenotypes'] expected_individuals = { 'I0_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], @@ -1032,16 +1045,91 @@ def test_variant_lookup(self, mock_liftover): 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, } - expected_body = self._expected_lookup_body(expected_individuals, [expected_variant], is_build_38=True) - self.assertDictEqual(response.json(), expected_body) cache_key = 'variant_lookup_results__1-10439-AC-A__38' - self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key=cache_key) + self._assert_expected_lookup( + '1-10439-AC-A', expected_variant, cache_key, cached_variants=[lookup_variant], skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, expected_individuals=expected_individuals, locusListsByGuid={}, + ) - self.login_manager() - response = self.client.get(url) - self.assertEqual(response.status_code, 200) + expected_gcnv_variant = { + **GCNV_VARIANT4, + 'familyGuids': ['F0_suffix_140608_DUP'], + 'genotypes': { + 'I0_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_variant = { + **SV_VARIANT4, + 'familyGuids': ['F0_phase2_DEL_chr14_4640'], + 'genotypes': { + 'I0_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_individuals = { + 'I0_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I0_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', + 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I1_F0_phase2_DEL_chr14_4640': { + 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', + 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I2_F0_suffix_140608_DUP': { + 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, + {'category': 'HP:0001626', 'label': '1 terms'}], + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + } + self._assert_expected_lookup( + 'phase2_DEL_chr14_4640', expected_sv_variant, 'variant_lookup_results__phase2_DEL_chr14_4640__38', + additional_variant=expected_gcnv_variant, cached_variants=[SV_VARIANT4, GCNV_VARIANT4], skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById={ + 'ENSG00000184986': mock.ANY, 'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY, + }, + ) - individual_guid_map = [ + self.login_manager() + individual_guid_map = [ # TODO clean up ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), @@ -1070,10 +1158,10 @@ def test_variant_lookup(self, mock_liftover): } expected_individuals.update( {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) - expected_body = self._expected_lookup_body( - expected_individuals, [expected_variant], include_context=True, + url = self._assert_expected_lookup( + '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000011_11', 'F000014_14'], is_build_38=True, + family_guids=['F000002_2', 'F000011_11', 'F000014_14'], ) # TODO test variant with search context! # expected_body.update({ @@ -1091,49 +1179,39 @@ def test_variant_lookup(self, mock_liftover): # 'VN0714937_2103343353_r0390_100': mock.ANY, # }, # }) - self.assertDictEqual(response.json(), expected_body) - self.assert_cached_results([VARIANT_LOOKUP_VARIANT], cache_key=cache_key) hom_only_lookup_variant = { - **VARIANT_LOOKUP_VARIANT, - 'familyGenotypes': { - 'F000002_2': [gt for gt in VARIANT_LOOKUP_VARIANT['familyGenotypes']['F000002_2'] if gt['sampleType'] == 'WGS'], - 'F000011_11': VARIANT_LOOKUP_VARIANT['familyGenotypes']['F000011_11'], - }, - } - del hom_only_lookup_variant['liftedFamilyGuids'] - hom_only_expected_variant = { - **expected_variant, - 'lookupFamilyGuids': expected_variant['lookupFamilyGuids'][:-1], + **lookup_variant, 'genotypes': { - **expected_variant['genotypes'], + **lookup_variant['genotypes'], **{guid: next(gt for gt in expected_variant['genotypes'][guid] if gt['sampleType'] == 'WGS') for guid in ['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733']}, - } + }, } - del hom_only_expected_variant['genotypes']['I000018_na21234'] - del hom_only_expected_variant['liftedFamilyGuids'] - expected_individuals = {guid: individual for guid, individual in expected_individuals.items() if guid not in { - 'I000018_na21234', 'I000019_na21987', 'I000021_na21654', - }} - hom_only_expected_body = self._expected_lookup_body( - expected_individuals, [hom_only_expected_variant], include_context=True, - project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], is_build_38=True, - ) - response = self.client.get(url + '&homOnly=true') - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), hom_only_expected_body) - self.assert_cached_results([hom_only_lookup_variant], cache_key=f'{cache_key}__hom') + del hom_only_lookup_variant['liftedFamilyGuids'] + del hom_only_lookup_variant['genotypes']['I000018_na21234'] + self._assert_expected_lookup( + '1-10439-AC-A', hom_only_lookup_variant, f'{cache_key}__hom', hom_only=True, + project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], + individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + ) url = url.replace('1-10439-AC-A', '1-91511686-TCA-G') response = self.client.get(url) self.assertEqual(response.status_code, 404) self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) - self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [VARIANT_LOOKUP_VARIANT]) + self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [{ + **VARIANT1, + 'familyGenotypes': {'F000002_2': list(VARIANT1['genotypes'].values())}, + }]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), expected_body) + self.assertDictEqual(response.json()['variants'][0], { + **VARIANT1, + 'familyGuids': [], + 'lookupFamilyGuids': VARIANT1['familyGuids'], + }) cache_key = 'variant_lookup_results__7-143270172-A-G__37' self._assert_expected_lookup( @@ -1181,108 +1259,33 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-439-AC-A', { - **liftover_variant, 'genotypes': hom_only_expected_variant['genotypes'], + **liftover_variant, 'genotypes': hom_only_lookup_variant['genotypes'], }, 'variant_lookup_results__1-439-AC-A__37__hom', genome_version='37', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], ) - self._assert_expected_lookup( + url = self._assert_expected_lookup( 'M-4429-G-A', MITO_VARIANT1, 'variant_lookup_results__M-4429-G-A__38', project_guids=['R0001_1kg'], family_guids=['F000002_2'], individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733',], ) - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, 'M-4429-G-A', '38', hom_only=True) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') - - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') - self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') + url = url.replace('38', '37') + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'MITO variants are not available for GRCh37'}) - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') - self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') + url = url.replace('M-4429-G-A', 'phase2_DEL_chr14_4640') + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'SV variants are not available for GRCh37'}) + url = url.replace('37', '38') response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant = { - **GCNV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], - 'genotypes': { - 'I0_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_suffix_140608_DUP': { - k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_sv_variant = { - **SV_VARIANT4, - 'familyGuids': [], - 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], - 'genotypes': { - 'I0_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I1_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - 'I2_F0_phase2_DEL_chr14_4640': { - k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() - if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }, - }, - } - expected_individuals = { - 'I0_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I0_F0_suffix_140608_DUP': { - 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', - 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, - {'category': 'HP:0001626', 'label': '1 terms'}], - 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I1_F0_phase2_DEL_chr14_4640': { - 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I1_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', - 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - 'I2_F0_phase2_DEL_chr14_4640': { - 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], - 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', - }, - 'I2_F0_suffix_140608_DUP': { - 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', - 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', - }, - } - expected_body = self._expected_lookup_body(expected_individuals, [expected_gcnv_variant, expected_sv_variant]) - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with( - self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', affected_only=False, hom_only=False, - ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {'error': 'Sample type must be specified to look up a structural variant'}) - self.login_manager() response = self.client.get(url) self.assertEqual(response.status_code, 200) expected_gcnv_variant.update({ @@ -1335,74 +1338,58 @@ def test_variant_lookup(self, mock_liftover): variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') self.assert_cached_results([GCNV_LOOKUP_VARIANT_3], cache_key='variant_lookup_results__suffix_140593_DUP__38') - def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, **kwargs): + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' if hom_only: url += '&homOnly=true' if affected_only: url += '&affectedOnly=true' + if sample_type: + url += f'&sampleType={sample_type}' response = self.client.get(url) self.assertEqual(response.status_code, 200) context_fields = {*SEARCH_RESPONSE_KEYS, 'igvSamplesByGuid', 'locusListsByGuid', 'familyNotesByGuid'} + if skip_fields: + context_fields -= skip_fields if genome_version == '38': context_fields.add('omimIntervals') + variants = [variant, *([additional_variant] if additional_variant else [])] expected_body = { **{k: {} for k in context_fields}, 'projectsByGuid': {p: mock.ANY for p in project_guids or []}, 'familiesByGuid': {f: mock.ANY for f in family_guids or []}, - 'individualsByGuid': {i: mock.ANY for i in individual_guids or []}, + 'individualsByGuid': expected_individuals or {i: mock.ANY for i in individual_guids or []}, 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, 'variants': [{ - **variant, + **v, 'familyGuids': [], - 'lookupFamilyGuids': variant['familyGuids'], - }], + 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []), + } for v in variants], **kwargs, } self.assertDictEqual(response.json(), expected_body) - family_genotypes = defaultdict(list) - for gts in variant['genotypes'].values(): - if not isinstance(gts, list): - gts = [gts] - for gt in gts: - family_genotypes[gt['familyGuid']].append({k: v for k, v in gt.items() if k != 'individualGuid'}) - lookup_variant = { - **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': { - family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x['sampleId']), reverse=True) - for family_guid, gts in family_genotypes.items() - }, - } - self.assert_cached_results([lookup_variant], cache_key=cache_key) - - def _expected_lookup_body(self, individuals_by_guid, variants, is_build_38=False, include_context=False, project_guids=None, family_guids=None): - exclude_keys = set() - if not include_context: - exclude_keys.update({'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid'}) - context_fields = {*SEARCH_RESPONSE_KEYS, 'igvSamplesByGuid', 'locusListsByGuid', 'familyNotesByGuid'} - exclude_keys - if is_build_38: - context_fields.add('omimIntervals') - expected_body = { - **{k: {} for k in context_fields}, - 'projectsByGuid': { - p: {k: mock.ANY for k in {'projectGuid', 'genomeVersion', 'variantTagTypes', 'variantFunctionalTagTypes'}} - for p in project_guids or [] - }, - 'familiesByGuid': {f: mock.ANY for f in family_guids or []}, - 'individualsByGuid': individuals_by_guid, - 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if is_build_38 else {}, - 'variants': variants, - } - if include_context: - # TODO clean up? - expected_body.update({ - 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, + parsed_cached_variants = [] + for v in (cached_variants or variants): + family_genotypes = defaultdict(list) + for individual_guid, gts in v['genotypes'].items(): + if not isinstance(gts, list): + gts = [gts] + for gt in gts: + family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] + family_genotypes[family_guid].append({k: v for k, v in gt.items() if k != 'individualGuid'}) + parsed_cached_variants.append({ + **{k: v for k, v in v.items() if k not in {'familyGuids', 'genotypes'}}, + 'familyGenotypes': { + family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) + for family_guid, gts in family_genotypes.items() + }, }) - return expected_body + self.assert_cached_results(parsed_cached_variants, cache_key=cache_key) + return url def test_get_single_variant(self): url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') From acf5b8682c52315b6cd09a1c73b1389f52d15c44 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 09:42:25 -0400 Subject: [PATCH 135/252] fix lookup test --- clickhouse_search/search_tests.py | 70 ++++++++----------------------- 1 file changed, 18 insertions(+), 52 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a509ee6113..721db9ac88 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1119,13 +1119,14 @@ def test_variant_lookup(self, mock_liftover): 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, } + sv_genes = { + 'ENSG00000184986': mock.ANY, 'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY, + } self._assert_expected_lookup( 'phase2_DEL_chr14_4640', expected_sv_variant, 'variant_lookup_results__phase2_DEL_chr14_4640__38', additional_variant=expected_gcnv_variant, cached_variants=[SV_VARIANT4, GCNV_VARIANT4], skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', - }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById={ - 'ENSG00000184986': mock.ANY, 'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY, - }, + }, expected_individuals=expected_sv_individuals, sample_type='WGS', locusListsByGuid={}, genesById=sv_genes, ) self.login_manager() @@ -1196,6 +1197,8 @@ def test_variant_lookup(self, mock_liftover): individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], ) + # TODO test affected_only + url = url.replace('1-10439-AC-A', '1-91511686-TCA-G') response = self.client.get(url) self.assertEqual(response.status_code, 404) @@ -1286,57 +1289,20 @@ def test_variant_lookup(self, mock_liftover): self.assertEqual(response.status_code, 400) self.assertEqual(response.json(), {'error': 'Sample type must be specified to look up a structural variant'}) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - expected_gcnv_variant.update({ - 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], - 'genotypes': GCNV_VARIANT4['genotypes'], - }) - expected_sv_variant.update({ - 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], - 'genotypes': SV_VARIANT4['genotypes'], - }) - expected_individuals = { - i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in [ - 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', - 'I000021_na21654', - ] - } - expected_body = self._expected_lookup_body( - expected_individuals, - [expected_gcnv_variant, expected_sv_variant], - project_guids=[PROJECT_GUID, 'R0004_non_analyst_project'], - family_guids=['F000002_2', 'F000014_14'], - include_context=True, - ) - del expected_body['transcriptsById'] - self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', - affected_only=False, hom_only=False, ) - - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' - self.assert_cached_results([SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT], cache_key=cache_key) - - affected_only_lookup_variant = { - **GCNV_LOOKUP_VARIANT, - 'familyGenotypes': { - family_guid: gts for family_guid, gts in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if family_guid != 'F000002_2_x' - }, - } - variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS', affected_only=True) - self.assert_cached_results([SV_LOOKUP_VARIANT, affected_only_lookup_variant], cache_key=f'{cache_key}__affected') + self._assert_expected_lookup( + 'phase2_DEL_chr14_4640', SV_VARIANT4, 'variant_lookup_results__phase2_DEL_chr14_4640__38', + additional_variant=GCNV_VARIANT4, sample_type='WGS', genesById=sv_genes, + project_guids=['R0001_1kg', 'R0004_non_analyst_project'], family_guids=['F000002_2', 'F000014_14'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654'], + ) # reciprocal overlap does not meet the threshold for smaller events - variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') - cache_key = 'variant_lookup_results__suffix_140608_DUP__38' - self.assert_cached_results([GCNV_LOOKUP_VARIANT], cache_key=cache_key) - - variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES', affected_only=True) - self.assert_cached_results([affected_only_lookup_variant], cache_key=f'{cache_key}__affected') - - variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') - self.assert_cached_results([GCNV_LOOKUP_VARIANT_3], cache_key='variant_lookup_results__suffix_140593_DUP__38') + self._assert_expected_lookup( + 'suffix_140608_DUP', GCNV_VARIANT4, 'variant_lookup_results__suffix_140608_DUP__38', + sample_type='WES', genesById={'ENSG00000275023': mock.ANY, 'ENSG00000277258': mock.ANY}, + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], + ) def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' From 715286dda4bcc7112d3220795adc0ad8e712963f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 09:47:41 -0400 Subject: [PATCH 136/252] clean up --- clickhouse_search/search_tests.py | 29 ++++++------------ clickhouse_search/test_utils.py | 50 ------------------------------- 2 files changed, 9 insertions(+), 70 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 721db9ac88..9a8c8935ef 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -33,10 +33,10 @@ VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ - MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ - MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ - FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, \ - SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, format_cached_variant + MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, \ + MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, \ + DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ + format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant from seqr.views.apis.data_manager_api import trigger_delete_project @@ -975,7 +975,6 @@ def test_variant_lookup(self, mock_liftover): self.check_require_login(reverse(variant_lookup_handler)) - # TODO do with/ replace VARIANT_LOOKUP_VARIANT lookup_variant = { **VARIANT1, 'liftedFamilyGuids': ['F000014_14'], @@ -1240,20 +1239,10 @@ def test_variant_lookup(self, mock_liftover): mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) - # TODO do as part of VARIANT_LOOKUP_VARIANT creation - liftover_variant = { - **VARIANT1, - 'familyGuids': [VARIANT1['familyGuids'][0], 'F000011_11'], - 'genotypes': { - **VARIANT1_BOTH_SAMPLE_TYPES['genotypes'], - 'I000015_na20885': [ - {**PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], 'sampleType': 'WES'}, - PROJECT_2_VARIANT1['genotypes']['I000015_na20885'], - ], - }, - } + del lookup_variant['liftedFamilyGuids'] + del lookup_variant['genotypes']['I000018_na21234'] self._assert_expected_lookup( - '1-439-AC-A', liftover_variant, 'variant_lookup_results__1-439-AC-A__37', genome_version='37', + '1-439-AC-A', lookup_variant, 'variant_lookup_results__1-439-AC-A__37', genome_version='37', project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], ) @@ -1262,7 +1251,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-439-AC-A', { - **liftover_variant, 'genotypes': hom_only_lookup_variant['genotypes'], + **lookup_variant, 'genotypes': hom_only_lookup_variant['genotypes'], }, 'variant_lookup_results__1-439-AC-A__37__hom', genome_version='37', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], @@ -1271,7 +1260,7 @@ def test_variant_lookup(self, mock_liftover): url = self._assert_expected_lookup( 'M-4429-G-A', MITO_VARIANT1, 'variant_lookup_results__M-4429-G-A__38', project_guids=['R0001_1kg'], family_guids=['F000002_2'], - individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733',], + individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) url = url.replace('38', '37') diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 2b9466fc42..4c2c756899 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1132,56 +1132,6 @@ }, } -LOOKUP_GENOTYPE = {k: v for k, v in PROJECT_2_VARIANT1['genotypes']['I000015_na20885'].items() if k != 'individualGuid'} -VARIANT_LOOKUP_VARIANT = { - **VARIANT1, - 'liftedFamilyGuids': ['F000014_14'], - 'familyGenotypes': { - VARIANT1['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for gs in VARIANT1_BOTH_SAMPLE_TYPES['genotypes'].values() for g in gs - ], key=lambda x: (x['sampleType'] == 'WES', x['sampleId']), reverse=True), - 'F000011_11': [{**LOOKUP_GENOTYPE, 'sampleType': 'WES'}, LOOKUP_GENOTYPE], - 'F000014_14': [{ - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', - 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531, 'filters': [], - }], - } -} -SV_LOOKUP_VARIANT = { - **SV_VARIANT4, - 'familyGenotypes': { - SV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in SV_VARIANT4['genotypes'].values()], - }, -} -GCNV_LOOKUP_VARIANT = { - **GCNV_VARIANT4, - 'familyGenotypes': { - GCNV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in GCNV_VARIANT4['genotypes'].values()], - 'F000002_2_x': [ - {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} - for individual_guid, g in GCNV_VARIANT4['genotypes'].items() if individual_guid != 'I000005_hg00732' - ], - }, -} -GCNV_LOOKUP_VARIANT_3 = { - **MULTI_PROJECT_GCNV_VARIANT3, - 'familyGenotypes': { - **{family_guid: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MULTI_PROJECT_GCNV_VARIANT3['genotypes'].values() - if g['familyGuid'] == family_guid] for family_guid in MULTI_PROJECT_GCNV_VARIANT3['familyGuids'] - }, - 'F000002_2_x': [ - {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} - for individual_guid, g in GCNV_VARIANT3['genotypes'].items() if individual_guid != 'I000005_hg00732' - ], - }, -} -for k in {'familyGuids', 'genotypes'}: - VARIANT_LOOKUP_VARIANT.pop(k) - SV_LOOKUP_VARIANT.pop(k) - GCNV_LOOKUP_VARIANT.pop(k) - GCNV_LOOKUP_VARIANT_3.pop(k) - PROJECT_4_COMP_HET_VARIANT = { 'key': 22, 'variantId': '1-9310123-T-C', From b96dea9340bfd72d1f999ac057b40b4bbb135c2f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 09:56:09 -0400 Subject: [PATCH 137/252] update lookup test; --- clickhouse_search/search_tests.py | 40 +++++++++++-------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9a8c8935ef..20695fbbc5 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1129,32 +1129,19 @@ def test_variant_lookup(self, mock_liftover): ) self.login_manager() - individual_guid_map = [ # TODO clean up - ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), - ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), - ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), - ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), - ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), - ] - expected_variant.update({ - 'lookupFamilyGuids': ['F000002_2', 'F000011_11', 'F000014_14'], - 'liftedFamilyGuids': ['F000014_14'], - 'genotypes': { - individual_guid: [ - {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in - expected_variant['genotypes'][anon_individual_guid] - ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { - **expected_variant['genotypes'][anon_individual_guid], **genotype, - 'individualGuid': individual_guid, - } for individual_guid, anon_individual_guid, genotype in individual_guid_map - }, - }) + individual_guid_map = { + 'I000006_hg00733': 'I0_F0_1-10439-AC-A', + 'I000005_hg00732': 'I1_F0_1-10439-AC-A', + 'I000004_hg00731': 'I2_F0_1-10439-AC-A', + 'I000015_na20885': 'I0_F1_1-10439-AC-A', + 'I000018_na21234': 'I0_F2_1-10439-AC-A', + } expected_individuals = { individual_guid: { **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, **{k: v for k, v in expected_individuals[anon_individual_guid].items() if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, - } for individual_guid, anon_individual_guid, _ in individual_guid_map + } for individual_guid, anon_individual_guid in individual_guid_map.items() } expected_individuals.update( {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) @@ -1184,8 +1171,9 @@ def test_variant_lookup(self, mock_liftover): **lookup_variant, 'genotypes': { **lookup_variant['genotypes'], - **{guid: next(gt for gt in expected_variant['genotypes'][guid] if gt['sampleType'] == 'WGS') - for guid in ['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733']}, + **{guid: VARIANT1_BOTH_SAMPLE_TYPES['genotypes'][guid][1] for guid in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', + ]}, }, } del hom_only_lookup_variant['liftedFamilyGuids'] @@ -1193,7 +1181,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-10439-AC-A', hom_only_lookup_variant, f'{cache_key}__hom', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], - individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], ) # TODO test affected_only @@ -1244,7 +1232,7 @@ def test_variant_lookup(self, mock_liftover): self._assert_expected_lookup( '1-439-AC-A', lookup_variant, 'variant_lookup_results__1-439-AC-A__37', genome_version='37', project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], - individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], ) mock_liftover.assert_called_with('hg19', 'hg38') mock_convert_coordinate.assert_called_with('chr1', 439) @@ -1254,7 +1242,7 @@ def test_variant_lookup(self, mock_liftover): **lookup_variant, 'genotypes': hom_only_lookup_variant['genotypes'], }, 'variant_lookup_results__1-439-AC-A__37__hom', genome_version='37', hom_only=True, project_guids=['R0001_1kg', 'R0003_test'], family_guids=['F000002_2', 'F000011_11'], - individual_guids=[guid for guid, _, _ in individual_guid_map if guid != 'I000018_na21234'], + individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], ) url = self._assert_expected_lookup( From 86eb8c9753c2f1d6a6c5e8112e5794f21e4d54cc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 10:42:56 -0400 Subject: [PATCH 138/252] add lookup test coverage --- .../fixtures/clickhouse_search.json | 2 +- clickhouse_search/search.py | 2 +- clickhouse_search/search_tests.py | 41 ++++++++++--------- clickhouse_search/test_utils.py | 8 ++-- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index de16fe36db..82508b5e00 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -893,7 +893,7 @@ "filters": [], "sign": 1, "calls": [ - ["NA19675", 1, 0, 0.16, 45] + ["NA19678", 1, 0, 0.16, 45] ] } }, { diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 99041a3b1e..fcd9428f7e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -730,7 +730,7 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, dataset_type, def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only): lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) - if not lifted_entry_cls: + if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return lifted_id = (variant['liftedOverChrom'], variant['liftedOverPos'], *variant_id[2:]) lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 20695fbbc5..54563531c0 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1145,27 +1145,11 @@ def test_variant_lookup(self, mock_liftover): } expected_individuals.update( {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) - url = self._assert_expected_lookup( + self._assert_expected_lookup( '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], family_guids=['F000002_2', 'F000011_11', 'F000014_14'], ) - # TODO test variant with search context! - # expected_body.update({ - # 'mmeSubmissionsByGuid': {'MS000018_P0004517': mock.ANY}, - # 'rnaSeqData': {'I000019_na21987': {'outliers': {}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}}, - # 'savedVariantsByGuid': { - # 'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0004_non': mock.ANY, - # }, - # 'variantTagsByGuid': { - # 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726970_2103343353_r0004_tes': mock.ANY, - # 'VT1726985_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0005_tes': mock.ANY, - # }, - # 'variantNotesByGuid': { - # 'VN0714935_2103343353_r0390_100': mock.ANY, - # 'VN0714937_2103343353_r0390_100': mock.ANY, - # }, - # }) hom_only_lookup_variant = { **lookup_variant, @@ -1184,9 +1168,28 @@ def test_variant_lookup(self, mock_liftover): individual_guids=[guid for guid in individual_guid_map.keys() if guid != 'I000018_na21234'], ) - # TODO test affected_only + url = self._assert_expected_lookup( + '21-3343353-GAGA-G', FAMILY_1_VARIANT, 'variant_lookup_results__21-3343353-GAGA-G__38', project_guids=['R0001_1kg'], + family_guids=['F000001_1'], individual_guids=['I000001_na19675', 'I000002_na19678', 'I000003_na19679'], + familyNotesByGuid={'FAN000001_1': mock.ANY, 'FAN000001_2': mock.ANY, 'FAN000001_3': mock.ANY}, + genesById={'ENSG00000227232': mock.ANY, 'ENSG00000268903': mock.ANY}, + igvSamplesByGuid={'S000145_na19675': mock.ANY}, + mmeSubmissionsByGuid={'MS000001_na19675': mock.ANY}, + phenotypeGeneScores={'I000001_na19675': mock.ANY, 'I000002_na19678': mock.ANY}, + rnaSeqData={'I000001_na19675': {'outliers': {'ENSG00000268903': mock.ANY}, 'spliceOutliers': {'ENSG00000268903': mock.ANY}}}, + savedVariantsByGuid={'SV0000001_2103343353_r0390_100': mock.ANY }, + variantTagsByGuid={'VT1708633_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0390_100': mock.ANY}, + variantFunctionalDataByGuid={ + 'VFD0000023_1248367227_r0390_10': mock.ANY, 'VFD0000024_1248367227_r0390_10': mock.ANY, + 'VFD0000025_1248367227_r0390_10': mock.ANY, 'VFD0000026_1248367227_r0390_10': mock.ANY, + }, + ) + + response = self.client.get(url + '&affectedOnly=true') + self.assertEqual(response.status_code, 404) + self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) - url = url.replace('1-10439-AC-A', '1-91511686-TCA-G') + url = url.replace('21-3343353-GAGA-G', '1-91511686-TCA-G') response = self.client.get(url) self.assertEqual(response.status_code, 404) self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 4c2c756899..25e753ca38 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -397,9 +397,9 @@ 'xpos': 21003343353, 'rsid': None, 'familyGuids': ['F000001_1'], - 'genotypes': {'I000001_na19675': { - 'ab': 0.16, 'dp': 45, 'familyGuid': 'F000001_1', 'filters': [], 'gq': 0, 'individualGuid': 'I000001_na19675', - 'numAlt': 1, 'sampleId': 'NA19675', 'sampleType': 'WES', + 'genotypes': {'I000002_na19678': { + 'ab': 0.16, 'dp': 45, 'familyGuid': 'F000001_1', 'filters': [], 'gq': 0, 'individualGuid': 'I000002_na19678', + 'numAlt': 1, 'sampleId': 'NA19678', 'sampleType': 'WES', }}, 'clinvar': None, 'hgmd': None, @@ -1454,7 +1454,7 @@ def format_cached_variant(variant): '', '', '', '', ''], ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '', '0', '0', '0', '', '', '', '', '', '', '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', - 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19675', '1', '', '0', '0.16', '', '', + 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19678', '1', '', '0', '0.16', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', From b019db39c507f60c4e2c8450e66d0ef8463c447a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 11:22:37 -0400 Subject: [PATCH 139/252] cache test --- clickhouse_search/search_tests.py | 73 +++++++++++++------------------ 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 54563531c0..950d1e0d13 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,3 +1,4 @@ +import re from collections import defaultdict from datetime import timedelta from django.contrib.auth.models import User @@ -125,7 +126,6 @@ def setUp(self): # TODO remove self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) self.user = User.objects.get(username='test_user') - self.search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': {'callset': {'ac': 1000}}}) self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) self.results_model.families.set(self.families) @@ -134,6 +134,9 @@ def setUp(self): def set_cache(self, cache_key, cached): self.MOCK_CACHE[cache_key] = json.dumps(cached) + self.mock_redis.keys.side_effect = lambda pattern: [ + key for key in list(self.MOCK_CACHE.keys()) if re.match(pattern, key) + ] def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' @@ -170,7 +173,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, response_search=None, project_families=None, additional_response=None, export_data=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -191,9 +194,10 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun expected_response['omimIntervals'] = mock.ANY self.assertDictEqual(response.json(), expected_response) - cache_key = f'search_results__VRS{search_hash:07d}__{sort}' - cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) - self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) + if not skip_cache_check: + cache_key = f'search_results__VRS{search_hash:07d}__{sort}' + cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) + self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) if gene_counts: self._assert_expected_gene_counts(search_hash, gene_counts) @@ -663,11 +667,11 @@ def test_inheritance_filter(self): def test_exclude_previous_search_results(self): self.mock_results_guid.return_value = 'VRS00079516' - VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') - self.set_cache('search_results__abc1234__gnomad', {'all_results': [ + vsr = VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') + cache_key = f'search_results__{vsr.guid}__gnomad' + self.set_cache(cache_key, {'all_results': [ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], ]}) - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] exclude = {'previousSearch': True, 'previousSearchHash': 'abc1234'} self._assert_expected_search( @@ -680,10 +684,9 @@ def test_exclude_previous_search_results(self): ], check_login=self.check_collaborator_login, ) - self.set_cache('search_results__abc1234__gnomad', {'all_results': [ + self.set_cache(cache_key, {'all_results': [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, ]}) - self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, inheritance_mode='recessive', cached_variant_fields=[ @@ -2365,44 +2368,28 @@ def test_search_context(self): self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) def test_cached_query_variants(self): - # TODO - cache_key_prefix = f'search_results__{self.results_model.guid}' + search_hash = 987 + cache_key_prefix = 'search_results__VRS0000987' cached_variants = [VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1, VARIANT2] cache_result = self._format_cached_variants(cached_variants) - self.set_cache(cache_result) - - variants, total = query_variants(self.results_model, user=self.user) - self.assertEqual(total, 5) - self.assertListEqual(self._encode_variants(variants), cached_variants) - self.mock_redis.get.assert_called_with(f'{cache_key_prefix}__xpos') - self.mock_redis.set.assert_not_called() + self.set_cache(f'{cache_key_prefix}__xpos', cache_result) - variants, _ = query_variants(self.results_model, user=self.user, num_results=2, page=2) - self.assertListEqual(variants, [GCNV_VARIANT1, MITO_VARIANT1]) + self._assert_expected_search( + cached_variants, search_hash=search_hash, skip_cache_check=True, check_login=self.check_collaborator_login, + ) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, - }) + self._assert_expected_search( + cached_variants, search_hash=search_hash, skip_cache_check=True, + results_page=[GCNV_VARIANT1, MITO_VARIANT1], query_params={'page': 2, 'per_page': 2}, gene_counts={ + 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, + 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, + }, + ) - self.mock_redis.get.side_effect = [None, json.dumps(cache_result)] - self.mock_redis.keys.return_value = [f'{cache_key_prefix}__xpos', f'{cache_key_prefix}__gnomad'] - - variants, total = query_variants(self.results_model, user=self.user, sort='cadd') - self.assertEqual(total, 5) - sorted_variants = [VARIANT2, VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1] - self.assertListEqual(self._encode_variants(variants), sorted_variants) - self.mock_redis.get.assert_has_calls([ - mock.call(f'{cache_key_prefix}__cadd'), - mock.call(f'{cache_key_prefix}__xpos'), - ]) - self.mock_redis.keys.assert_called_with(pattern=f'{cache_key_prefix}__*') - self.assert_cached_results( - {'all_results': [format_cached_variant(v) for v in sorted_variants], 'total_results': 5}, - sort='cadd', + self._assert_expected_search( + [VARIANT2, VARIANT1, SV_VARIANT1, GCNV_VARIANT1, MITO_VARIANT1], search_hash=search_hash, sort='cadd', ) class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): From 8204115145cb581e9b4f5c5080cf9ee6864690ac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 11:24:47 -0400 Subject: [PATCH 140/252] clean up --- clickhouse_search/search_tests.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 950d1e0d13..bbab5181a4 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1,7 +1,6 @@ import re from collections import defaultdict from datetime import timedelta -from django.contrib.auth.models import User from django.core.management import call_command from django.db import connections from django.urls.base import reverse @@ -123,13 +122,6 @@ def setUp(self): self.mock_results_guid.return_value = random.randint(1000, 10000) # nosec self.addCleanup(patcher.stop) - # TODO remove - self.families = Family.objects.filter(guid__in=['F000003_3', 'F000002_2', 'F000005_5']) - self.user = User.objects.get(username='test_user') - self.search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}, 'freqs': {'callset': {'ac': 1000}}}) - self.results_model = VariantSearchResults.objects.create(variant_search=self.search_model) - self.results_model.families.set(self.families) - super().setUp() def set_cache(self, cache_key, cached): @@ -138,8 +130,7 @@ def set_cache(self, cache_key, cached): key for key in list(self.MOCK_CACHE.keys()) if re.match(pattern, key) ] - def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): - cache_key = cache_key or f'search_results__{self.results_model.guid}__{sort}' + def assert_cached_results(self, expected_results, cache_key): self.mock_redis.set.assert_any_call(cache_key, mock.ANY) call_index = next(i for i, call in enumerate(self.mock_redis.set.call_args_list) if call.args[0] == cache_key) self.assertEqual(json.loads(self.mock_redis.set.call_args_list[call_index].args[1]), expected_results) @@ -197,7 +188,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun if not skip_cache_check: cache_key = f'search_results__VRS{search_hash:07d}__{sort}' cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) - self.assert_cached_results(cached_variants, sort=sort, cache_key=cache_key) + self.assert_cached_results(cached_variants, cache_key) if gene_counts: self._assert_expected_gene_counts(search_hash, gene_counts) @@ -1337,7 +1328,7 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version for family_guid, gts in family_genotypes.items() }, }) - self.assert_cached_results(parsed_cached_variants, cache_key=cache_key) + self.assert_cached_results(parsed_cached_variants, cache_key) return url def test_get_single_variant(self): From 6c1dd30d6967cbfad32c3e9d1c84a99f8028a4a2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 12:46:00 -0400 Subject: [PATCH 141/252] cleaner helper function for all variant query --- clickhouse_search/search.py | 4 +- seqr/utils/search/utils.py | 91 +++++++++------------------ seqr/views/apis/variant_search_api.py | 5 +- 3 files changed, 34 insertions(+), 66 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fcd9428f7e..7e7e7dca4a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -31,7 +31,7 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version,page=1, num_results=100, sort=None, **kwargs): +def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version, sort=None, **kwargs): inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) @@ -97,8 +97,6 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno logger.info(f'Total results: {cache_results["total_results"]}', user) - return format_clickhouse_results(cache_results['all_results'][(page-1)*num_results:page*num_results], genome_version) - def get_search_queryset(genome_version, dataset_type, sample_data, **search_kwargs): entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] variants_cls = VARIANTS_CLASS_MAP[genome_version][dataset_type] diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index e031bfca8a..6a53d8784a 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -130,64 +130,46 @@ def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) -def _get_any_sort_cached_results(search_model): - cache_key = _get_search_cache_key(search_model, sort='*') - return safe_redis_get_wildcard_json(cache_key) - - -def _validate_export_variant_count(total_variants): +def export_variants(search_model, user): + previous_search_results, genome_version = _query_variants(search_model, user) + total_variants = previous_search_results['total_results'] if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + return format_clickhouse_results(previous_search_results['all_results'], genome_version) -def _get_previous_search_results(search_model, sort, page, num_results, load_all, genome_version): - cache_key = _get_search_cache_key(search_model, sort=sort) - previous_search_results = safe_redis_get_json(cache_key) or {} +def _get_previous_search_results(search_model, sort): + previous_search_results = None + if sort: + cache_key = _get_search_cache_key(search_model, sort=sort) + previous_search_results = safe_redis_get_json(cache_key) or {} if not previous_search_results: - unsorted_results = _get_any_sort_cached_results(search_model) - if unsorted_results: + wildcard_cache_key = _get_search_cache_key(search_model, sort='*') + previous_search_results = safe_redis_get_wildcard_json(wildcard_cache_key) + if previous_search_results and sort: previous_search_results = get_clickhouse_cache_results( - unsorted_results['all_results'], sort, family_guid=search_model.families.first().guid, + previous_search_results['all_results'], sort, family_guid=search_model.families.first().guid, ) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - - total_results = previous_search_results.get('total_results') - if load_all: - num_results = total_results or MAX_EXPORT_VARIANTS - _validate_export_variant_count(num_results) - start_index = (page - 1) * num_results - end_index = page * num_results - if total_results is not None: - end_index = min(end_index, total_results) - - cached_page = None - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) - - return previous_search_results, cached_page, num_results + return previous_search_results -def query_variants(search_model, sort=XPOS_SORT_KEY, load_all=False, user=None, page=1, num_results=100): - genome_version = _get_search_genome_version(search_model) - previous_search_results, cached_page, num_results = _get_previous_search_results( - search_model, sort, page, num_results, load_all, genome_version, - ) - if cached_page is not None: - return cached_page, previous_search_results.get('total_results') +def query_variants(search_model, sort, page, num_results, user): + previous_search_results, genome_version = _query_variants(search_model, user, sort=sort) - variants, total_results = _query_variants( - search_model, user, previous_search_results, genome_version, sort=sort, page=page, num_results=num_results, - ) + all_results = previous_search_results.get('all_results') or [] + results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results], genome_version) - if load_all: - _validate_export_variant_count(total_results) + return results_page, previous_search_results.get('total_results') - return variants, total_results +def _query_variants(search_model, user, sort=None, **kwargs): + genome_version = _get_search_genome_version(search_model) + previous_search_results = _get_previous_search_results(search_model, sort) or {} + if previous_search_results: + return previous_search_results, genome_version -def _query_variants(search_model, user, previous_search_results, genome_version, sort=None, num_results=100, **kwargs): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() @@ -200,7 +182,6 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if locus and exclude_locations: raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - variant_ids = None parsed_search = {**search} genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: @@ -216,9 +197,6 @@ def _query_variants(search_model, user, previous_search_results, genome_version, raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) - if variant_ids: - num_results = len(variant_ids) - exclude.pop('rawItems', None) if exclude.get('clinvar') and (search.get('pathogenicity') or {}).get('clinvar'): duplicates = set(search['pathogenicity']['clinvar']).intersection(exclude['clinvar']) @@ -228,7 +206,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, exclude.pop('previousSearch', None) exclude_previous_hash = exclude.pop('previousSearchHash', None) if exclude_previous_hash: - parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user, genome_version)) + parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user)) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): @@ -249,23 +227,20 @@ def _query_variants(search_model, user, previous_search_results, genome_version, _validate_search(parsed_search, samples, previous_search_results) - variant_results = get_clickhouse_variants( + get_clickhouse_variants( samples, parsed_search, user, previous_search_results, genome_version, - sort=sort, num_results=num_results, **kwargs, + sort=sort, **kwargs, ) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - return variant_results, previous_search_results.get('total_results') + return previous_search_results, genome_version -def _get_clickhouse_exclude_keys(search_hash, user, genome_version): +def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - cached_results = _get_any_sort_cached_results(previous_search_model) - if not cached_results: - cached_results = {} - _query_variants(previous_search_model, user, cached_results, genome_version) + cached_results, genome_version = _query_variants(previous_search_model, user) results = cached_results['all_results'] exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) @@ -289,11 +264,7 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - previous_search_results = _get_any_sort_cached_results(search_model) or {} - if len(previous_search_results.get('all_results', [])) != previous_search_results.get('total_results'): - genome_version = _get_search_genome_version(search_model) - _query_variants(search_model, user, previous_search_results, genome_version) - + previous_search_results, genome_version = _query_variants(search_model, user) flat_variants = [ v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) ] diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index fd86ce0e35..266fc6e3a6 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -14,9 +14,8 @@ from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.gene_utils import get_gene from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - variant_lookup, parse_variant_id + variant_lookup, parse_variant_id, export_variants from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException from seqr.utils.xpos_utils import get_xpos @@ -280,7 +279,7 @@ def export_variants_handler(request, search_hash): families = results_model.families.all() family_ids_by_guid = {family.guid: family.family_id for family in families} - variants, _ = query_variants(results_model, page=1, load_all=True, user=request.user) + variants = export_variants(results_model, request.user) variants = _flatten_variants(variants) saved_variants, variants_by_id = _get_saved_variant_models(variants, families) From fc47e9046cfd9694adced85ba53027546b83dee4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 13:01:12 -0400 Subject: [PATCH 142/252] clean uo es configs --- clickhouse_search/test_utils.py | 6 ++--- seqr/views/apis/variant_search_api.py | 35 ++++++--------------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 25e753ca38..821d8162f9 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1430,7 +1430,7 @@ def format_cached_variant(variant): 'rs112766696', '', '', 'Likely_pathogenic', '', '2', '', '', '', '', '', 'HG00731', '1', '', '99', '0.5', 'HG00732', '0', '', '0', '0', 'HG00733', '0', '', '20', '0', '', '', '', '', ''], ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', '0.00288997', '0.246152', - '25.9', '0.197', '2.001', '0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', + '25.9', '0.197', '2.001', '0', '0.1', '0.05', 'polymorphism', '0', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1', 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0', '', '', '', '', ''], ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', @@ -1438,7 +1438,7 @@ def format_cached_variant(variant): '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857'], ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '', '0.00026519', '0', '0', '29.9', - '0.526', '9.491', '0', '', '', '', '', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', + '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', '', '', '', '', ''], ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', @@ -1459,7 +1459,7 @@ def format_cached_variant(variant): ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '', '', '', '', + ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', '', 'ENST00000361381.2:c.838A>G', 'ENSP00000354961.2:p.Thr280Ala', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], ['M', '14783', 'T', 'C', 'ENSG00000198727', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', '', diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 266fc6e3a6..b2d0bff97c 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -173,19 +173,6 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo return response_json -PREDICTION_MAP = { - 'D': 'damaging', - 'T': 'tolerated', -} - - -POLYPHEN_MAP = { - 'D': 'probably_damaging', - 'P': 'possibly_damaging', - 'B': 'benign', -} - - MUTTASTR_MAP = { 'A': 'disease_causing', 'D': 'disease_causing', @@ -194,14 +181,8 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo } -def _get_prediction_val(prediction): - try: - return float(prediction or '') - except ValueError: - return PREDICTION_MAP.get(prediction[0]) if prediction else None - - def _get_variant_main_transcript_field_val(parsed_variant): + # TODO return next( (t for t in parsed_variant['transcripts'] if t['transcriptId'] == parsed_variant['mainTranscriptId']), {} ).get('value') @@ -214,8 +195,8 @@ def _get_variant_main_transcript_field_val(parsed_variant): {'header': 'alt'}, {'header': 'gene', 'value_path': '{transcripts: transcripts.*[].{value: geneSymbol || geneId, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, {'header': 'worst_consequence', 'value_path': '{transcripts: transcripts.*[].{value: majorConsequence, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'callset_freq', 'value_path': 'populations.callset.af || populations.seqr.ac'}, - {'header': 'exac_freq', 'value_path': 'populations.exac.af'}, + {'header': 'callset_freq', 'value_path': 'populations.seqr.ac'}, #TODO rename + {'header': 'exac_freq', 'value_path': 'populations.exac.af'}, # TODO remove {'header': 'gnomad_genomes_freq', 'value_path': 'populations.gnomad_genomes.af'}, {'header': 'gnomad_exomes_freq', 'value_path': 'populations.gnomad_exomes.af'}, {'header': 'topmed_freq', 'value_path': 'populations.topmed.af'}, @@ -223,14 +204,14 @@ def _get_variant_main_transcript_field_val(parsed_variant): {'header': 'revel', 'value_path': 'predictions.revel'}, {'header': 'eigen', 'value_path': 'predictions.eigen'}, {'header': 'splice_ai', 'value_path': 'predictions.splice_ai'}, - {'header': 'polyphen', 'value_path': 'predictions.polyphen', 'process': _get_prediction_val}, - {'header': 'sift', 'value_path': 'predictions.sift', 'process': _get_prediction_val}, - {'header': 'muttaster', 'value_path': 'predictions.mut_taster', 'process': _get_prediction_val}, - {'header': 'fathmm', 'value_path': 'predictions.fathmm', 'process': _get_prediction_val}, + {'header': 'polyphen', 'value_path': 'predictions.polyphen'}, + {'header': 'sift', 'value_path': 'predictions.sift'}, + {'header': 'muttaster', 'value_path': 'predictions.mut_taster', 'process': MUTTASTR_MAP.get}, + {'header': 'fathmm', 'value_path': 'predictions.fathmm'}, {'header': 'rsid', 'value_path': 'rsid'}, {'header': 'hgvsc', 'value_path': '{transcripts: transcripts.*[].{value: hgvsc, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, {'header': 'hgvsp', 'value_path': '{transcripts: transcripts.*[].{value: hgvsp, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'clinvar_clinical_significance', 'value_path': 'clinvar.clinicalSignificance || clinvar.pathogenicity'}, + {'header': 'clinvar_clinical_significance', 'value_path': 'clinvar.pathogenicity'}, {'header': 'clinvar_gold_stars', 'value_path': 'clinvar.goldStars'}, ] From 6015b11aab65c0993cc28bfb86df55c445b2b578 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:11:14 -0400 Subject: [PATCH 143/252] cleaner format export logic --- clickhouse_search/search.py | 19 +++++++++++++++++++ seqr/utils/search/utils.py | 4 ++-- seqr/views/apis/variant_search_api.py | 16 ++++------------ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 7e7e7dca4a..bb21ad2b6e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -331,6 +331,25 @@ def get_clickhouse_cache_results(results, sort, family_guid): return {'all_results': sorted_results, 'total_results': total_results} +def format_clickhouse_export_results(results, genome_version): + formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] + keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} + detail_qs = _get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) + details_by_key = { + detail['key']: detail for detail in detail_qs.values( + 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), + **detail_qs.split_variant_id_annotations(), + ) + } + for result in formatted_results: + if 'transcripts' in result: + result['mainTranscript'] = next((gene_transcripts[0] for gene_transcripts in result['transcripts'].values()), {}) + else: + result.update(details_by_key.get(result['key'], {})) + + return formatted_results + + def format_clickhouse_results(results, genome_version): keys_with_no_details = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 6a53d8784a..cfa978538d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -3,7 +3,7 @@ from datetime import timedelta from pyliftover.liftover import LiftOver -from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project, VariantSearchResults @@ -135,7 +135,7 @@ def export_variants(search_model, user): total_variants = previous_search_results['total_results'] if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') - return format_clickhouse_results(previous_search_results['all_results'], genome_version) + return format_clickhouse_export_results(previous_search_results['all_results'], genome_version) def _get_previous_search_results(search_model, sort): diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index b2d0bff97c..394576ef63 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -181,20 +181,13 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo } -def _get_variant_main_transcript_field_val(parsed_variant): - # TODO - return next( - (t for t in parsed_variant['transcripts'] if t['transcriptId'] == parsed_variant['mainTranscriptId']), {} - ).get('value') - - VARIANT_EXPORT_DATA = [ {'header': 'chrom'}, {'header': 'pos'}, {'header': 'ref'}, {'header': 'alt'}, - {'header': 'gene', 'value_path': '{transcripts: transcripts.*[].{value: geneSymbol || geneId, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'worst_consequence', 'value_path': '{transcripts: transcripts.*[].{value: majorConsequence, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, + {'header': 'gene', 'value_path': 'mainTranscript.geneId'}, + {'header': 'worst_consequence', 'value_path': 'mainTranscript.majorConsequence'}, {'header': 'callset_freq', 'value_path': 'populations.seqr.ac'}, #TODO rename {'header': 'exac_freq', 'value_path': 'populations.exac.af'}, # TODO remove {'header': 'gnomad_genomes_freq', 'value_path': 'populations.gnomad_genomes.af'}, @@ -209,8 +202,8 @@ def _get_variant_main_transcript_field_val(parsed_variant): {'header': 'muttaster', 'value_path': 'predictions.mut_taster', 'process': MUTTASTR_MAP.get}, {'header': 'fathmm', 'value_path': 'predictions.fathmm'}, {'header': 'rsid', 'value_path': 'rsid'}, - {'header': 'hgvsc', 'value_path': '{transcripts: transcripts.*[].{value: hgvsc, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, - {'header': 'hgvsp', 'value_path': '{transcripts: transcripts.*[].{value: hgvsp, transcriptId: transcriptId}, mainTranscriptId: mainTranscriptId}', 'process': _get_variant_main_transcript_field_val}, + {'header': 'hgvsc', 'value_path': 'mainTranscript.hgvsc'}, + {'header': 'hgvsp', 'value_path': 'mainTranscript.hgvsp'}, {'header': 'clinvar_clinical_significance', 'value_path': 'clinvar.pathogenicity'}, {'header': 'clinvar_gold_stars', 'value_path': 'clinvar.goldStars'}, ] @@ -261,7 +254,6 @@ def export_variants_handler(request, search_hash): family_ids_by_guid = {family.guid: family.family_id for family in families} variants = export_variants(results_model, request.user) - variants = _flatten_variants(variants) saved_variants, variants_by_id = _get_saved_variant_models(variants, families) json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=families[0].project.genome_version) From ace51cd9b9a42e04b45b00778667b10edde4cd55 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:14:54 -0400 Subject: [PATCH 144/252] clean up --- clickhouse_search/test_utils.py | 36 +++++++++++++-------------- seqr/views/apis/variant_search_api.py | 3 +-- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 821d8162f9..c7282aa111 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1420,67 +1420,67 @@ def format_cached_variant(variant): } EXPORT_DATA = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', + ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3', 'sample_4', 'num_alt_alleles_4', 'filters_4', 'gq_4', 'ab_4'], - ['1', '10439', 'AC', 'A', '', '', '8', '', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', + ['1', '10439', 'AC', 'A', '', '', '8', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', 'rs112766696', '', '', 'Likely_pathogenic', '', '2', '', '', '', '', '', 'HG00731', '1', '', '99', '0.5', 'HG00732', '0', '', '0', '0', 'HG00733', '0', '', '20', '0', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '', '0.00344493', '0.00288997', '0.246152', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.00344493', '0.00288997', '0.246152', '25.9', '0.197', '2.001', '0', '0.1', '0.05', 'polymorphism', '0', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1', 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0', '', '', '', '', ''], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '3', '', '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857'], - ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '', '0.00026519', '0', '0', '29.9', + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', '', '', '', '', ''], - ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', + ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], - ['16', '29809156', '', '', 'ENSG00000103495', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['16', '29809156', '', '', 'ENSG00000103495', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '1', '', '', '', '', '', '', '', ''], - ['17', '38717327', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['17', '38717327', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '2', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], - ['17', '38721781', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['17', '38721781', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '1', '', '', '', '', '', '', '', ''], - ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '', '0', '0', '0', '', '', '', '', '', '', + ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '0', '0', '0', '', '', '', '', '', '', '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19678', '1', '', '0', '0.16', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', + ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', + ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', '', 'ENST00000361381.2:c.838A>G', 'ENSP00000354961.2:p.Thr280Ala', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '14783', 'T', 'C', 'ENSG00000198727', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', '', + ['M', '14783', 'T', 'C', 'ENSG00000198727', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', 'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '', '', '', 'HG00731', '2', 'artifact_prone_site'], ] SPLIT_FAMILY_EXPORT_DATA = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'callset_freq', 'exac_freq', 'gnomad_genomes_freq', + ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', '', '', '', '', ''], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '3', '', '', 'HG00733', '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857', '', '', '', '', ''], - ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '', '0.00026519', '0', '0', '29.9', - '0.526', '9.491', '0', '', '', '', '', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', 'HG00731', '1', + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', + '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0'], ] diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 394576ef63..f2ea5eff0a 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -188,8 +188,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo {'header': 'alt'}, {'header': 'gene', 'value_path': 'mainTranscript.geneId'}, {'header': 'worst_consequence', 'value_path': 'mainTranscript.majorConsequence'}, - {'header': 'callset_freq', 'value_path': 'populations.seqr.ac'}, #TODO rename - {'header': 'exac_freq', 'value_path': 'populations.exac.af'}, # TODO remove + {'header': 'seqr_ac', 'value_path': 'populations.seqr.ac'}, {'header': 'gnomad_genomes_freq', 'value_path': 'populations.gnomad_genomes.af'}, {'header': 'gnomad_exomes_freq', 'value_path': 'populations.gnomad_exomes.af'}, {'header': 'topmed_freq', 'value_path': 'populations.topmed.af'}, From 4ebc95f709dd524706807025db972178072da52d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:34:38 -0400 Subject: [PATCH 145/252] export geen symbol --- clickhouse_search/search.py | 11 ++++++++- clickhouse_search/test_utils.py | 34 +++++++++++++-------------- seqr/views/apis/variant_search_api.py | 3 ++- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index bb21ad2b6e..b999e91926 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -14,7 +14,7 @@ from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP -from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP +from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization, Individual from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ @@ -341,11 +341,20 @@ def format_clickhouse_export_results(results, genome_version): **detail_qs.split_variant_id_annotations(), ) } + + gene_ids = set() for result in formatted_results: if 'transcripts' in result: result['mainTranscript'] = next((gene_transcripts[0] for gene_transcripts in result['transcripts'].values()), {}) else: result.update(details_by_key.get(result['key'], {})) + if result.get('mainTranscript'): + gene_ids.add(result['mainTranscript']['geneId']) + + gene_id_map = dict(GeneInfo.objects.filter(gene_id__in=gene_ids).values_list('gene_id', 'gene_symbol')) + for result in formatted_results: + if result.get('mainTranscript'): + result['geneSymbol'] = gene_id_map.get(result['mainTranscript']['geneId']) return formatted_results diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index c7282aa111..91f9ced017 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1420,66 +1420,66 @@ def format_cached_variant(variant): } EXPORT_DATA = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', + ['chrom', 'pos', 'ref', 'alt', 'gene', 'gene_symbol', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', 'family_id_2', 'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3', 'sample_4', 'num_alt_alleles_4', 'filters_4', 'gq_4', 'ab_4'], - ['1', '10439', 'AC', 'A', '', '', '8', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', + ['1', '10439', 'AC', 'A', '', '', '', '8', '0.03444932', '0', '0.07842', '4.668', '', '', '', '', '', '', '', 'rs112766696', '', '', 'Likely_pathogenic', '', '2', '', '', '', '', '', 'HG00731', '1', '', '99', '0.5', 'HG00732', '0', '', '0', '0', 'HG00733', '0', '', '20', '0', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.00344493', '0.00288997', '0.246152', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'MTHFR', 'missense_variant', '10', '0.00344493', '0.00288997', '0.246152', '25.9', '0.197', '2.001', '0', '0.1', '0.05', 'polymorphism', '0', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1', 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0', '', '', '', '', ''], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '3', '', '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857'], - ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'CDC7', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', '', '', '', '', ''], - ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', + ['14', '22438910', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], - ['16', '29809156', '', '', 'ENSG00000103495', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['16', '29809156', '', '', 'ENSG00000103495', '', 'COPY_GAIN', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '1', '', '', '', '', '', '', '', ''], - ['17', '38717327', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['17', '38717327', '', '', 'ENSG00000275023', 'CDC9', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '2', '', '', '', 'HG00732', '1', '', '', '', 'HG00733', '0', '', '', '', '', '', '', '', ''], - ['17', '38721781', '', '', 'ENSG00000275023', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + ['17', '38721781', '', '', 'ENSG00000275023', 'CDC9', 'LOF', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', 'HG00732', '0', '', '', '', 'HG00733', '1', '', '', '', '', '', '', '', ''], - ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'missense_variant', '1', '0', '0', '0', '', '', '', '', '', '', + ['21', '3343353', 'GAGA', 'G', 'ENSG00000227232', 'WASH7P', 'missense_variant', '1', '0', '0', '0', '', '', '', '', '', '', '', '', '', 'ENST00000257261.10:c.131C>T', 'ENSP00000257261.6:p.Thr44Ile', '', '', '1', 'Tier 1 - Novel gene and phenotype (None)|Review (None)', '', '', '', '', 'NA19678', '1', '', '0', '0.16', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '4429', 'G', 'A', 'ENSG00000210112', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', + ['M', '4429', 'G', 'A', 'ENSG00000210112', '', 'non_coding_transcript_exon_variant', '0', '', '', '', '', '', '', '', '', '', '', '', 'rs1603219456', 'ENST00000387377.1:n.28G>A', '', 'Uncertain_significance', '1', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '11597', 'A', 'G', 'ENSG00000198886', 'missense_variant', '0', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', + ['M', '11597', 'A', 'G', 'ENSG00000198886', '', 'missense_variant', '0', '', '', '', '', '', '', '', '', '0', 'polymorphism', '', '', 'ENST00000361381.2:c.838A>G', 'ENSP00000354961.2:p.Thr280Ala', '', '', '2', '', '', '', '', '', 'HG00731', '1', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['M', '14783', 'T', 'C', 'ENSG00000198727', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', + ['M', '14783', 'T', 'C', 'ENSG00000198727', '', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', 'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '', '', '', 'HG00731', '2', 'artifact_prone_site'], ] SPLIT_FAMILY_EXPORT_DATA = [ - ['chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', + ['chrom', 'pos', 'ref', 'alt', 'gene', 'gene_symbol', 'worst_consequence', 'seqr_ac', 'gnomad_genomes_freq', 'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen', 'splice_ai', 'polyphen', 'sift', 'muttaster', 'fathmm', 'rsid', 'hgvsc', 'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars', 'family_id_1', 'tags_1', 'notes_1', 'sample_1', 'num_alt_alleles_1', 'filters_1', 'gq_1', 'ab_1', 'sample_2', 'num_alt_alleles_2', 'filters_2', 'gq_2', 'ab_2', 'sample_3', 'num_alt_alleles_3', 'filters_3', 'gq_3', 'ab_3'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', '1', '', '99', '1', 'HG00732', '0', '', '99', '0.45946', '', '', '', '', ''], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'CDC7', 'intron_variant', '7', '0.00380411', '0', '0.362681', '28.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '3', '', '', 'HG00733', '1', '', '99', '0.40741', 'NA20870', '1', '', '99', '0.67857', '', '', '', '', ''], - ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', + ['1', '91511686', 'T', 'G', 'ENSG00000097046', 'CDC7', 'splice_donor_variant', '5', '0.00026519', '0', '0', '29.9', '0.526', '9.491', '0', '0', '0', 'polymorphism', '0', '', 'ENST00000466716.5:c.-264+1G>A', '', '', '', '2', '', '', 'HG00731', '1', 'VQSRTrancheSNP99.95to100.00', '58', '0.27241', 'HG00732', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0', 'HG00733', '0', 'VQSRTrancheSNP99.95to100.00', '30', '0'], diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index f2ea5eff0a..c76e1cc0a4 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -187,6 +187,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo {'header': 'ref'}, {'header': 'alt'}, {'header': 'gene', 'value_path': 'mainTranscript.geneId'}, + {'header': 'gene_symbol', 'value_path': 'geneSymbol'}, {'header': 'worst_consequence', 'value_path': 'mainTranscript.majorConsequence'}, {'header': 'seqr_ac', 'value_path': 'populations.seqr.ac'}, {'header': 'gnomad_genomes_freq', 'value_path': 'populations.gnomad_genomes.af'}, @@ -324,7 +325,7 @@ def export_variants_handler(request, search_hash): def _get_field_value(value, config, variant=None): field_value = jmespath.search(config.get('value_path', config['header']), value) - if config.get('variant_value_path') and not field_value: + if config.get('variant_value_path') and not field_value: # TODO clean up this special case field_value = jmespath.search(config['variant_value_path'], variant) if config.get('process'): field_value = config['process'](field_value) From 7ab31aee2862003a5e3147820893b9f3cb9d67d0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:36:32 -0400 Subject: [PATCH 146/252] clean up --- seqr/views/apis/variant_search_api.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index c76e1cc0a4..84a34fb493 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -223,7 +223,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo VARIANT_SAMPLE_DATA = [ {'header': 'sample', 'value_path': 'sampleId'}, {'header': 'num_alt_alleles', 'value_path': 'numAlt'}, - {'header': 'filters', 'process': lambda val: ';'.join(val or []), 'variant_value_path': '[genotypeFilters][*]'}, + {'header': 'filters', 'process': lambda val: ';'.join(val or [])}, {'header': 'gq'}, {'header': 'ab'}, ] @@ -308,7 +308,7 @@ def export_variants_handler(request, search_hash): genotypes = [genotype for _, genotype in sorted(variant['genotypes'].items())] for genotype in genotypes: - row += [_get_field_value(genotype, config, variant=variant) for config in VARIANT_SAMPLE_DATA] + row += [_get_field_value(genotype, config) for config in VARIANT_SAMPLE_DATA] row += ['' for i in range(len(VARIANT_SAMPLE_DATA) * (max_samples_per_variant - len(genotypes)))] rows.append(row) @@ -323,10 +323,8 @@ def export_variants_handler(request, search_hash): return export_table('search_results_{}'.format(search_hash), header, rows, file_format, titlecase_header=False) -def _get_field_value(value, config, variant=None): +def _get_field_value(value, config): field_value = jmespath.search(config.get('value_path', config['header']), value) - if config.get('variant_value_path') and not field_value: # TODO clean up this special case - field_value = jmespath.search(config['variant_value_path'], variant) if config.get('process'): field_value = config['process'](field_value) return field_value From b8140685bfbfa05b6fdb403d18d239ea382566d3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:37:43 -0400 Subject: [PATCH 147/252] codacy --- seqr/utils/search/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index cfa978538d..7caffdac6f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -240,7 +240,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - cached_results, genome_version = _query_variants(previous_search_model, user) + cached_results, _ = _query_variants(previous_search_model, user) results = cached_results['all_results'] exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) @@ -264,7 +264,7 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - previous_search_results, genome_version = _query_variants(search_model, user) + previous_search_results, _ = _query_variants(search_model, user) flat_variants = [ v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) ] From 1211c3774b17179eb9a187503d6d8bebbaf7d837 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:44:02 -0400 Subject: [PATCH 148/252] codacy --- seqr/utils/search/utils.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 7caffdac6f..a84ca41391 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -170,6 +170,17 @@ def _query_variants(search_model, user, sort=None, **kwargs): if previous_search_results: return previous_search_results, genome_version + parsed_search, samples = _parse_search(search_model, genome_version, user, sort) + _validate_search(parsed_search, samples, previous_search_results) + get_clickhouse_variants(samples, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) + + cache_key = _get_search_cache_key(search_model, sort=sort) + safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) + + return previous_search_results, genome_version + + +def _parse_search(search_model, genome_version, user, sort): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() @@ -225,17 +236,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): if parsed_search.get('inheritance'): samples = _parse_inheritance(parsed_search, samples) - _validate_search(parsed_search, samples, previous_search_results) - - get_clickhouse_variants( - samples, parsed_search, user, previous_search_results, genome_version, - sort=sort, **kwargs, - ) - - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - - return previous_search_results, genome_version + return parsed_search, samples def _get_clickhouse_exclude_keys(search_hash, user): From b6bbbd79e4f652b8206c9dbbc52a7ae91e1428df Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:54:41 -0400 Subject: [PATCH 149/252] codacy complexity --- seqr/utils/search/utils.py | 47 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index a84ca41391..09ffac8e08 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -170,7 +170,27 @@ def _query_variants(search_model, user, sort=None, **kwargs): if previous_search_results: return previous_search_results, genome_version - parsed_search, samples = _parse_search(search_model, genome_version, user, sort) + search = deepcopy(search_model.variant_search.search) + families = search_model.families.all() + _validate_sort(sort, families) + + parsed_search = _parse_search(search, genome_version, user) + + dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) + parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) + search_dataset_type = None + if dataset_type and dataset_type != ALL_DATA_TYPES: + if secondary_dataset_type is None or secondary_dataset_type == dataset_type: + search_dataset_type = lookup_dataset_type or dataset_type + elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: + search_dataset_type = DATASET_TYPE_NO_MITO + + samples = _get_families_search_data( + families, dataset_type=search_dataset_type, include_no_access_projects=bool(search.get('no_access_project_genome_version')), + ) + if parsed_search.get('inheritance'): + samples = _parse_inheritance(parsed_search, samples) + _validate_search(parsed_search, samples, previous_search_results) get_clickhouse_variants(samples, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) @@ -180,13 +200,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): return previous_search_results, genome_version -def _parse_search(search_model, genome_version, user, sort): - search = deepcopy(search_model.variant_search.search) - - families = search_model.families.all() - _validate_sort(sort, families) - - no_access_project_genome_version = search.get('no_access_project_genome_version') +def _parse_search(search, genome_version, user): locus = search.pop('locus', None) or {} exclude = search.get('exclude', None) or {} exclude_locations = bool(exclude.get('rawItems')) @@ -197,7 +211,7 @@ def _parse_search(search_model, genome_version, user, sort): genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) - if no_access_project_genome_version and len(genes or []) != 1: + if search.get('no_access_project_genome_version') and len(genes or []) != 1: raise InvalidSearchException('Including external projects is only available when searching for a single gene') if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: raise InvalidSearchException('Too many genes/intervals') @@ -223,20 +237,7 @@ def _parse_search(search_model, genome_version, user, sort): if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) - parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) - search_dataset_type = None - if dataset_type and dataset_type != ALL_DATA_TYPES: - if secondary_dataset_type is None or secondary_dataset_type == dataset_type: - search_dataset_type = lookup_dataset_type or dataset_type - elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: - search_dataset_type = DATASET_TYPE_NO_MITO - - samples = _get_families_search_data(families, dataset_type=search_dataset_type, include_no_access_projects=bool(no_access_project_genome_version)) - if parsed_search.get('inheritance'): - samples = _parse_inheritance(parsed_search, samples) - - return parsed_search, samples + return parsed_search def _get_clickhouse_exclude_keys(search_hash, user): From 2d5a93687e9ac619e852b053ba3128b2eb4a3fdc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 13 Mar 2026 14:59:28 -0400 Subject: [PATCH 150/252] readability --- seqr/utils/search/utils.py | 40 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 09ffac8e08..be4d3034f2 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -175,23 +175,10 @@ def _query_variants(search_model, user, sort=None, **kwargs): _validate_sort(sort, families) parsed_search = _parse_search(search, genome_version, user) - - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) - parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) - search_dataset_type = None - if dataset_type and dataset_type != ALL_DATA_TYPES: - if secondary_dataset_type is None or secondary_dataset_type == dataset_type: - search_dataset_type = lookup_dataset_type or dataset_type - elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: - search_dataset_type = DATASET_TYPE_NO_MITO - - samples = _get_families_search_data( - families, dataset_type=search_dataset_type, include_no_access_projects=bool(search.get('no_access_project_genome_version')), - ) - if parsed_search.get('inheritance'): - samples = _parse_inheritance(parsed_search, samples) - + search_dataset_type = _parse_dataset_type(parsed_search, genome_version) + samples = _get_search_samples(parsed_search, families, search_dataset_type) _validate_search(parsed_search, samples, previous_search_results) + get_clickhouse_variants(samples, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) cache_key = _get_search_cache_key(search_model, sort=sort) @@ -201,6 +188,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): def _parse_search(search, genome_version, user): + no_access_project_genome_version = search.get('no_access_project_genome_version') locus = search.pop('locus', None) or {} exclude = search.get('exclude', None) or {} exclude_locations = bool(exclude.get('rawItems')) @@ -211,7 +199,7 @@ def _parse_search(search, genome_version, user): genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) - if search.get('no_access_project_genome_version') and len(genes or []) != 1: + if no_access_project_genome_version and len(genes or []) != 1: raise InvalidSearchException('Including external projects is only available when searching for a single gene') if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: raise InvalidSearchException('Too many genes/intervals') @@ -239,6 +227,24 @@ def _parse_search(search, genome_version, user): return parsed_search +def _parse_dataset_type(parsed_search, genome_version): + dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) + parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) + search_dataset_type = None + if dataset_type and dataset_type != ALL_DATA_TYPES: + if secondary_dataset_type is None or secondary_dataset_type == dataset_type: + search_dataset_type = lookup_dataset_type or dataset_type + elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: + search_dataset_type = DATASET_TYPE_NO_MITO + return search_dataset_type + + +def _get_search_samples(parsed_search, families, search_dataset_type): + samples = _get_families_search_data(families, dataset_type=search_dataset_type, include_no_access_projects=bool(parsed_search.get('no_access_project_genome_version'))) + if parsed_search.get('inheritance'): + samples = _parse_inheritance(parsed_search, samples) + return samples + def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) From 90037c88a51a2b18553a94c94112b0884cbdd52a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Mar 2026 05:40:49 +0000 Subject: [PATCH 151/252] Bump pyjwt from 2.8.0 to 2.12.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.8.0 to 2.12.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.8.0...2.12.0) --- updated-dependencies: - dependency-name: pyjwt dependency-version: 2.12.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53b74e6f0f..248127c912 100644 --- a/requirements.txt +++ b/requirements.txt @@ -123,7 +123,7 @@ pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 # via cffi -pyjwt==2.8.0 +pyjwt==2.12.0 # via social-auth-core pyliftover==0.4 # via -r requirements.in From 2f4025bce1f15e62b83560a0f37d2a2db0d61c4a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 11:24:56 -0400 Subject: [PATCH 152/252] flag missing coverage; --- clickhouse_search/managers.py | 2 ++ clickhouse_search/models/reference_data_models.py | 1 + clickhouse_search/search.py | 1 + seqr/utils/logging_utils.py | 1 + seqr/utils/middleware.py | 1 + seqr/utils/vcf_utils.py | 1 + seqr/views/apis/data_manager_api.py | 1 + seqr/views/apis/data_manager_api_tests.py | 1 + seqr/views/apis/family_api.py | 1 + seqr/views/apis/individual_api.py | 5 +++++ seqr/views/apis/project_api.py | 1 + seqr/views/apis/variant_search_api.py | 1 + seqr/views/utils/anvil_metadata_utils.py | 1 + seqr/views/utils/individual_utils.py | 1 + seqr/views/utils/orm_to_json_utils.py | 3 +++ seqr/views/utils/pedigree_info_utils.py | 2 ++ seqr/views/utils/permissions_utils.py | 1 + seqr/views/utils/project_context_utils.py | 1 + seqr/views/utils/terra_api_utils_tests.py | 1 + 19 files changed, 27 insertions(+) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8eae82707b..4438b42376 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -764,6 +764,7 @@ def annotation_values(self): @classmethod def _clinvar_path_q(cls, pathogenicity): + # TODO return None @staticmethod @@ -1312,6 +1313,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu if annotate_num_families: entries = entries.annotate(numFamilies=Count('family_guid')) elif skip_entry_fields: + # TODO entries = entries.distinct('key') else: gt_field, gt_expression = self.genotype_expression(sample_data) diff --git a/clickhouse_search/models/reference_data_models.py b/clickhouse_search/models/reference_data_models.py index 5a7b26a041..2432c56910 100644 --- a/clickhouse_search/models/reference_data_models.py +++ b/clickhouse_search/models/reference_data_models.py @@ -507,6 +507,7 @@ class HgmdMvMeta(RefreshableMaterializedViewMeta): HGMD_INFO_STRUCTURE = 'CHROM String, POS UInt32, ID String, REF String, ALT String, QUAL String, FILTER String, INFO String' def _hgmd_source(url): + # TODO if not url: return f"null('{HGMD_INFO_STRUCTURE}')" return f"gcs(pipeline_data_access, url='{url}', format='TSV', structure='{HGMD_INFO_STRUCTURE}')" diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 99041a3b1e..0493aa848f 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -859,5 +859,6 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): } def _clickhouse_dataset_type(dataset_type, sample_type): if dataset_type == Sample.DATASET_TYPE_SV_CALLS: + # TODO dataset_type = SV_DATASET_TYPES[sample_type] return dataset_type \ No newline at end of file diff --git a/seqr/utils/logging_utils.py b/seqr/utils/logging_utils.py index be7a7d27a3..c03fa8b6a7 100644 --- a/seqr/utils/logging_utils.py +++ b/seqr/utils/logging_utils.py @@ -22,6 +22,7 @@ def formatMessage(self, record): if getattr(record, 'user', None) and record.user.is_authenticated: log_json['user'] = record.user.email elif getattr(record, 'user_email', None): + # TODO log_json['user'] = record.user_email if hasattr(record, 'db_update'): diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 0b77e1b1f0..e0b41db242 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -57,6 +57,7 @@ def _get_exception_status_code(exception): try: return status(exception) except Exception: + # TODO return 500 def _get_core_exception_json(exception): diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index 8112f9ffaf..6043f69f4d 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -79,6 +79,7 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None vcf_filename = _validate_valid_vcf_name(data_path, user, allowed_exts) if vcf_filename is None: + # TODO return None byte_range = None if vcf_filename.endswith('.vcf') else (0, BLOCK_SIZE) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index e9386f6848..3fd78fce89 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -94,6 +94,7 @@ def update_rna_seq(request): def _get_sample_file_path(file_dir, sample_guid): + # TODO return os.path.join(file_dir, f'{sample_guid}.json.gz') diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 1e46dc6e0a..9ba2a3e649 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -510,6 +510,7 @@ def _has_expected_file_loading_logs(self, file, file_path, user, info=None, warn for log in reversed(additional_logs): expected_logs.insert(additional_logs_offset, log) else: + # TODO expected_logs += additional_logs self.assert_json_logs(user, expected_logs) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 538774d430..12b26bf0a6 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -166,6 +166,7 @@ def edit_families_handler(request, project_guid): family_guids = [f.get('familyGuid') for f in modified_families] family_models = {f.guid: f for f in Family.objects.filter(project=project, guid__in=family_guids)} if len(family_models) != len(family_guids): + # TODO missing_guids = set(family_guids) - set(family_models.keys()) return create_json_response({'error': 'Invalid family guids: {}'.format(', '.join(missing_guids))}, status=400) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index df12dce8ee..1469aa196a 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -380,6 +380,7 @@ def _gene_value(val): def _gene_list_value(val): if isinstance(val, list): + # TODO return val seperator_escaped_val = ''.join(m.replace(',', ';') if not m.startswith('(') else m for m in re.split('(\([^)]+\))', val)) return [_gene_value(o) for o in seperator_escaped_val.split(';')] @@ -410,6 +411,7 @@ def _gene_list_value(val): } def _get_year(val): + # TODO return datetime.strptime(val, '%Y-%m-%d').year def _nested_val(nested_key): @@ -417,6 +419,7 @@ def _nested_val(nested_key): def _get_phenotips_features(observed): def get_observed_features(features): + # TODO return [{'id': feature['id']} for feature in features if feature['observed'] == observed] return get_observed_features @@ -454,6 +457,7 @@ def get_observed_features(features): } def _parse_phenotips_record(row): + # TODO record = {} for k, formatters in PHENOTIPS_JSON_FIELD_MAP.items(): val = row.get(k) @@ -634,6 +638,7 @@ def _get_record_individual(record, individual_lookup): if not individual: individual = individual_lookup['{}_{}'.format(family_id, individual_id)].get(family_id) else: + # TODO individual = next((i for i in individuals.values()), None) return individual, individual_id diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 2593c1cc56..4323f21efc 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -140,6 +140,7 @@ def update_project_handler(request, project_guid): @pm_required def update_project_workspace(request, project_guid): if not is_anvil_authenticated(request.user): + # TODO raise PermissionDenied() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index fd1f9bdb38..ceb196034c 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -529,6 +529,7 @@ def _get_saved_variant_models(variants, families): for family_guid in variant.get('familyGuids', []): variant_ids_by_family[family_guid].add(variant['variantId']) if variant.get('liftedOverGenomeVersion') == GENOME_VERSION_GRCh37 and hg37_family_guids: + # TODO variant_hg37_families = [family_guid for family_guid in variant['familyGuids'] if family_guid in hg37_family_guids] if variant_hg37_families: lifted_xpos = get_xpos(variant['liftedOverChrom'], variant['liftedOverPos']) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 01eb449849..7fb67525a9 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -471,6 +471,7 @@ def _get_variant_main_transcript(variant_model): main_transcript = next((t for t in transcripts if t['transcriptId'] == main_transcript_id), None) if main_transcript: if 'geneId' not in main_transcript: + # TODO main_transcript['geneId'] = gene_id return main_transcript elif len(variant.get('transcripts', {})) == 1: diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b9e46d1a21..7d3a86c7a9 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -141,6 +141,7 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated # Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID if 'father' in record or 'mother' in record: + # TODO parent_updates.append({ 'individual': individual, 'mother': record.pop('mother', None), diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index aea73090ae..838d43e872 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -242,6 +242,7 @@ def _get_json_for_families(families, user=None, add_individual_guids_field=False 'pedigreeImage': NullIf(Concat(Value(MEDIA_URL), 'pedigree_image', output_field=CharField()), Value(MEDIA_URL)), } if additional_values: + # TODO family_additional_values.update(additional_values) if add_individual_guids_field: family_additional_values.update(INDIVIDUAL_GUIDS_VALUES) @@ -287,6 +288,7 @@ def _get_json_for_individuals(individuals, user=None, project_guid=None, family_ {'fields': ('family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, ] if family_fields: + # TODO for field in family_fields: nested_fields.append({'fields': ('family', field), 'key': _to_camel_case(field)}) @@ -343,6 +345,7 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid {'fields': ('individual', 'family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, ]} else: + # TODO additional_kwargs = {'additional_model_fields': ['individual_id']} return {'guid_key': 'sampleGuid', **additional_kwargs, **kwargs} diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index fd122b4fb2..c0e22a738e 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -49,6 +49,7 @@ def parse_pedigree_table(parsed_file, filename, user, project): if not user_is_pm(user): raise ValueError('Unsupported file format') if not project: + # TODO raise ValueError('Project argument required for parsing sample manifest') header, rows = _parse_merged_pedigree_sample_manifest_rows(parsed_file[1:]) else: @@ -306,6 +307,7 @@ def validate_fam_file_records(project, records, errors=None, clear_invalid_value sex=Individual.SEX_LOOKUP[r[JsonConstants.SEX_COLUMN]] ) if clear_invalid_values: + # TODO r[JsonConstants.PROBAND_RELATIONSHIP] = None warnings.append(f'Skipped {message}') else: diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index 9617f8734b..db208f36b2 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -161,6 +161,7 @@ def _map_anvil_seqr_permission(anvil_permission): def anvil_has_perm(user, permission_level, project): if not project_has_anvil(project): + # TODO return False return has_workspace_perm(user, permission_level, project.workspace_namespace, project.workspace_name) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 5e6851f26f..22b5c41f64 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -148,6 +148,7 @@ def families_discovery_tags(families, genome_version, project=None): def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): + # TODO return next(( gene_id for gene_id, gene_transcripts in (transcripts or {}).items() if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts) diff --git a/seqr/views/utils/terra_api_utils_tests.py b/seqr/views/utils/terra_api_utils_tests.py index 2e95ebb74b..02075d3fba 100644 --- a/seqr/views/utils/terra_api_utils_tests.py +++ b/seqr/views/utils/terra_api_utils_tests.py @@ -98,6 +98,7 @@ def _check_exceptions(self, path, func, args, kwargs=None, responses_body=None): _ = func(*args, **kwargs) self.assertEqual(str(ec.exception), message.format(path=path)) if isinstance(exception, TerraAPIException): + # TODO self.assertEqual(ec.exception.status_code, status) self.assert_no_logs() From ba6b14cfd78398074f25f36d8a3658aa6947dfb9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 11:35:39 -0400 Subject: [PATCH 153/252] properly test hgmd sort --- clickhouse_search/search_tests.py | 10 ++++++---- seqr/views/apis/variant_search_api.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index bbab5181a4..f628d178a2 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -164,7 +164,7 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { @@ -186,7 +186,7 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun self.assertDictEqual(response.json(), expected_response) if not skip_cache_check: - cache_key = f'search_results__VRS{search_hash:07d}__{sort}' + cache_key = f'search_results__VRS{search_hash:07d}__{cache_sort or sort}' cached_variants = self._format_cached_variants(expected_results, cached_variant_fields=cached_variant_fields) self.assert_cached_results(cached_variants, cache_key) @@ -668,7 +668,7 @@ def test_exclude_previous_search_results(self): self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - exclude=exclude, cached_variant_fields=[ + exclude=exclude, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, @@ -1907,9 +1907,10 @@ def test_sort(self): [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], sort='pathogenicity', annotations=None, ) + self.login_analyst_user() self._assert_expected_search( [VARIANT1, MITO_VARIANT3, VARIANT2, MITO_VARIANT1, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT2], - sort='pathogenicity_hgmd', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + sort='pathogenicity', cache_sort='pathogenicity_hgmd', project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) sorted_variants = [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT2, VARIANT3, MITO_VARIANT3, VARIANT1, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4] @@ -1977,6 +1978,7 @@ def test_sort(self): [GCNV_VARIANT1, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='size', locus=None, ) + self.login_manager() self._assert_expected_search( [SV_VARIANT4, SV_VARIANT1, SV_VARIANT3, SV_VARIANT2], sort='size', project_families=SV_PROJECT_FAMILIES, ) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index fd86ce0e35..68c0d6901b 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -114,6 +114,7 @@ def _get_or_create_results_model(search_hash, search_context, user): search_dict = search_context.get('search', {}) if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): + # TODO test search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] if search_context.get('includeNoAccessProjects'): search_dict['no_access_project_genome_version'] = all_project_genome_version From 0e71a8cb4ed4d9ce722d6d32bcb210bdb0d1c7b0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 11:41:52 -0400 Subject: [PATCH 154/252] properly test body for exclude previous --- clickhouse_search/search_tests.py | 8 +++++--- seqr/views/apis/variant_search_api.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f628d178a2..626e792698 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -664,11 +664,13 @@ def test_exclude_previous_search_results(self): VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], ]}) - exclude = {'previousSearch': True, 'previousSearchHash': 'abc1234'} + request_body = {'projectFamilies': DEFAULT_PROJECT_FAMILIES, 'previousSearchHash': 'abc1234'} + exclude = {'previousSearch': True} + response_search = {'exclude': {**exclude, 'previousSearchHash': 'abc1234'}} self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - exclude=exclude, cached_variant_fields=[ + request_body=request_body, response_search=response_search, exclude=exclude, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, {}, @@ -680,7 +682,7 @@ def test_exclude_previous_search_results(self): ]}) self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, - inheritance_mode='recessive', cached_variant_fields=[ + request_body=request_body, response_search=response_search, inheritance_mode='recessive', cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], ], ) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 68c0d6901b..fd86ce0e35 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -114,7 +114,6 @@ def _get_or_create_results_model(search_hash, search_context, user): search_dict = search_context.get('search', {}) if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): - # TODO test search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] if search_context.get('includeNoAccessProjects'): search_dict['no_access_project_genome_version'] = all_project_genome_version From a0021a046d76e9a44ad608263e4a02401c5c9e17 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 11:52:40 -0400 Subject: [PATCH 155/252] clean up --- clickhouse_search/managers.py | 20 ++++++------------- .../models/reference_data_models.py | 6 ------ clickhouse_search/search.py | 2 +- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 4438b42376..31ce7895c9 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -39,10 +39,6 @@ def _pathogenicity_tuple(model, field_prefix, **kwargs): output_field=NamedTupleField(list(fields.values()), null_if_empty=True, null_empty_arrays=True, **kwargs), ) - @classmethod - def _clinvar_path_q(cls, pathogenicity): - return cls._clinvar_filter_q(pathogenicity, allowed_filters=CLINVAR_PATH_SIGNIFICANCES) - @classmethod def _clinvar_filter_q(cls, pathogenicity, allowed_filters=None): clinvar_filters = (pathogenicity or {}).get(CLINVAR_KEY) @@ -762,11 +758,6 @@ def annotation_values(self): annotations['transcripts'] = annotations.pop(self.model.sorted_gene_consequences.field.db_column) return annotations - @classmethod - def _clinvar_path_q(cls, pathogenicity): - # TODO - return None - @staticmethod def _genotype_override_expression(index, col, cn_index): expressions = [f'x.{index}', f'sample_{col}'] @@ -1020,6 +1011,10 @@ def genome_version(self): def callset_filter_field(self): return 'callset' + @classmethod + def _clinvar_path_q(cls, pathogenicity): + return cls._clinvar_filter_q(pathogenicity, allowed_filters=CLINVAR_PATH_SIGNIFICANCES) + def search(self, sample_data, exclude_keys=None, **kwargs): entries = self.filter_locus(**kwargs) @@ -1302,7 +1297,7 @@ def annotation_fields(cls, entries): fields.append('seqrPop') return fields - def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, annotate_num_families=False, override_annotations=None, **kwargs): + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, **kwargs): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -1310,11 +1305,8 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu if multi_sample_type_families or sample_data is None or sample_data['num_families'] > 1: entries = entries.values(*fields) - if annotate_num_families: + if skip_entry_fields: entries = entries.annotate(numFamilies=Count('family_guid')) - elif skip_entry_fields: - # TODO - entries = entries.distinct('key') else: gt_field, gt_expression = self.genotype_expression(sample_data) entries = entries.annotate( diff --git a/clickhouse_search/models/reference_data_models.py b/clickhouse_search/models/reference_data_models.py index 2432c56910..901fbafb61 100644 --- a/clickhouse_search/models/reference_data_models.py +++ b/clickhouse_search/models/reference_data_models.py @@ -506,12 +506,6 @@ class HgmdMvMeta(RefreshableMaterializedViewMeta): HGMD_INFO_STRUCTURE = 'CHROM String, POS UInt32, ID String, REF String, ALT String, QUAL String, FILTER String, INFO String' -def _hgmd_source(url): - # TODO - if not url: - return f"null('{HGMD_INFO_STRUCTURE}')" - return f"gcs(pipeline_data_access, url='{url}', format='TSV', structure='{HGMD_INFO_STRUCTURE}')" - class HgmdAllMv(RefreshableMaterializedView): variant_id = models.StringField(db_column='variantId', primary_key=True) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 0493aa848f..4eab2377e6 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -88,7 +88,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno if search.get('no_access_project_genome_version'): logger.info('Looking up variants in projects with no user access', user) results += _get_search_results( - genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, **search, sample_data=None, skip_entry_fields=True, annotate_num_families=True, + genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, **search, sample_data=None, skip_entry_fields=True, exclude_projects=sample_data_by_dataset_type.get(Sample.DATASET_TYPE_VARIANT_CALLS, {}).get('project_guids'), ) From 8a4eeded628da660fb337222a7b459ff50345ccf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 11:58:16 -0400 Subject: [PATCH 156/252] test delete gcnv data --- clickhouse_search/search.py | 14 ++------------ clickhouse_search/search_tests.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 4eab2377e6..62fd5a6109 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -842,7 +842,8 @@ def _main_transcript(selected_transcript_id, sorted_transcripts): def delete_clickhouse_project(project, dataset_type, sample_type=None): - dataset_type = _clickhouse_dataset_type(dataset_type, sample_type) + if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: + dataset_type = 'GCNV' table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) @@ -851,14 +852,3 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh() ENTRY_CLASS_MAP[project.genome_version][dataset_type].gt_stats.rel.related_model.reload() return f'Deleted all {dataset_type} search data for project {project.name}' - - -SV_DATASET_TYPES = { - Sample.SAMPLE_TYPE_WGS: Sample.DATASET_TYPE_SV_CALLS, - Sample.SAMPLE_TYPE_WES: 'GCNV', -} -def _clickhouse_dataset_type(dataset_type, sample_type): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - # TODO - dataset_type = SV_DATASET_TYPES[sample_type] - return dataset_type \ No newline at end of file diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ab4bd6df3c..114508f268 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1939,8 +1939,9 @@ def test_trigger_delete_project(self): self.check_data_manager_login(url) Project.objects.filter(guid='R0001_1kg').update(genome_version='38') + body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} response = self.client.post( - url, content_type='application/json', data=json.dumps({'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'}) + url, content_type='application/json', data=json.dumps(body) ) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { @@ -1968,4 +1969,16 @@ def test_trigger_delete_project(self): self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0) self.assertEqual(project_samples.count(), 4) + body['datasetType'] = 'SV' + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': [ + 'Deactivated search for 3 individuals', + 'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de', + ], + }) + self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0) + self.assertEqual(project_samples.count(), 1) + From 00d7d9341ada082582ca9297fe405c94861719c6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 12:03:50 -0400 Subject: [PATCH 157/252] clean up --- seqr/utils/logging_utils.py | 1 - seqr/utils/middleware.py | 11 ++++------- seqr/utils/search/utils.py | 4 ---- seqr/utils/vcf_utils.py | 1 - 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/seqr/utils/logging_utils.py b/seqr/utils/logging_utils.py index c03fa8b6a7..be7a7d27a3 100644 --- a/seqr/utils/logging_utils.py +++ b/seqr/utils/logging_utils.py @@ -22,7 +22,6 @@ def formatMessage(self, record): if getattr(record, 'user', None) and record.user.is_authenticated: log_json['user'] = record.user.email elif getattr(record, 'user_email', None): - # TODO log_json['user'] = record.user_email if hasattr(record, 'db_update'): diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index e0b41db242..4f694435c9 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -11,7 +11,7 @@ import json import traceback -from seqr.utils.search.utils import SEARCH_EXCEPTION_ERROR_MAP +from seqr.utils.search.utils import InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.terra_api_utils import TerraAPIException @@ -33,12 +33,12 @@ def __init__(self, errors, warnings=None): ObjectDoesNotExist: 404, Http404: 404, ErrorsWarningsException: 400, + InvalidSearchException: 400, AuthException: 401, HTTPError: lambda e: int(e.response.status_code), TerraAPIException: lambda e: e.status_code, AnymailError: lambda e: getattr(e, 'status_code', None) or 400, } -EXCEPTION_ERROR_MAP.update(SEARCH_EXCEPTION_ERROR_MAP) EXCEPTION_JSON_MAP = { ErrorsWarningsException: lambda e: {'errors': e.errors, 'warnings': e.warnings} @@ -54,11 +54,8 @@ def _get_exception_status_code(exception): if isinstance(status, int): return status - try: - return status(exception) - except Exception: - # TODO - return 500 + return status(exception) + def _get_core_exception_json(exception): exception_json_formatter = next((f for exc, f in EXCEPTION_JSON_MAP.items() if isinstance(exception, exc)), None) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index e031bfca8a..921a9a2bd6 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -20,10 +20,6 @@ class InvalidSearchException(Exception): pass -SEARCH_EXCEPTION_ERROR_MAP = { - InvalidSearchException: 400, -} - DATASET_TYPES_LOOKUP = { data_types[0]: data_types for data_types in [ [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS], diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index 6043f69f4d..8112f9ffaf 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -79,7 +79,6 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None vcf_filename = _validate_valid_vcf_name(data_path, user, allowed_exts) if vcf_filename is None: - # TODO return None byte_range = None if vcf_filename.endswith('.vcf') else (0, BLOCK_SIZE) From 7855f5287bc0280c0632fa2ac5d90ca533afd8d6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 12:13:25 -0400 Subject: [PATCH 158/252] api cleanup --- seqr/views/apis/data_manager_api.py | 5 ----- seqr/views/apis/data_manager_api_tests.py | 8 ++------ seqr/views/apis/family_api.py | 1 - seqr/views/apis/family_api_tests.py | 6 ++++++ seqr/views/apis/individual_api.py | 5 ----- seqr/views/apis/project_api.py | 1 - 6 files changed, 8 insertions(+), 18 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 3fd78fce89..821bff7af3 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -93,11 +93,6 @@ def update_rna_seq(request): return create_json_response(response_json, status=status) -def _get_sample_file_path(file_dir, sample_guid): - # TODO - return os.path.join(file_dir, f'{sample_guid}.json.gz') - - def _notify_phenotype_prioritization_loaded(project, tool, num_samples): send_project_notification( project, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 9ba2a3e649..3b688ab47e 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -506,12 +506,8 @@ def _has_expected_file_loading_logs(self, file, file_path, user, info=None, warn (warn_log, {'severity': 'WARNING'}) for warn_log in warnings or [] ] if additional_logs: - if additional_logs_offset: - for log in reversed(additional_logs): - expected_logs.insert(additional_logs_offset, log) - else: - # TODO - expected_logs += additional_logs + for log in reversed(additional_logs): + expected_logs.insert(additional_logs_offset, log) self.assert_json_logs(user, expected_logs) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 12b26bf0a6..538774d430 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -166,7 +166,6 @@ def edit_families_handler(request, project_guid): family_guids = [f.get('familyGuid') for f in modified_families] family_models = {f.guid: f for f in Family.objects.filter(project=project, guid__in=family_guids)} if len(family_models) != len(family_guids): - # TODO missing_guids = set(family_guids) - set(family_models.keys()) return create_json_response({'error': 'Invalid family guids: {}'.format(', '.join(missing_guids))}, status=400) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 4d2ad4eac0..67fc881917 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -250,6 +250,12 @@ def test_edit_families_handler(self, mock_pm_group): self.assertEqual(response.status_code, 400) self.assertEqual(response.reason_phrase, "'families' not specified") + response = self.client.post(url, content_type='application/json', data=json.dumps({'families': [ + {'familyGuid': '2', 'familyId': '2', 'description': 'Test description 1'} + ]})) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['error'], 'Invalid family guids: 2') + response = self.client.post(url, content_type='application/json', data=json.dumps({'families': [ {'familyGuid': FAMILY_GUID, 'familyId': '2', 'description': 'Test description 1'} ]})) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 1469aa196a..df12dce8ee 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -380,7 +380,6 @@ def _gene_value(val): def _gene_list_value(val): if isinstance(val, list): - # TODO return val seperator_escaped_val = ''.join(m.replace(',', ';') if not m.startswith('(') else m for m in re.split('(\([^)]+\))', val)) return [_gene_value(o) for o in seperator_escaped_val.split(';')] @@ -411,7 +410,6 @@ def _gene_list_value(val): } def _get_year(val): - # TODO return datetime.strptime(val, '%Y-%m-%d').year def _nested_val(nested_key): @@ -419,7 +417,6 @@ def _nested_val(nested_key): def _get_phenotips_features(observed): def get_observed_features(features): - # TODO return [{'id': feature['id']} for feature in features if feature['observed'] == observed] return get_observed_features @@ -457,7 +454,6 @@ def get_observed_features(features): } def _parse_phenotips_record(row): - # TODO record = {} for k, formatters in PHENOTIPS_JSON_FIELD_MAP.items(): val = row.get(k) @@ -638,7 +634,6 @@ def _get_record_individual(record, individual_lookup): if not individual: individual = individual_lookup['{}_{}'.format(family_id, individual_id)].get(family_id) else: - # TODO individual = next((i for i in individuals.values()), None) return individual, individual_id diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 4323f21efc..2593c1cc56 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -140,7 +140,6 @@ def update_project_handler(request, project_guid): @pm_required def update_project_workspace(request, project_guid): if not is_anvil_authenticated(request.user): - # TODO raise PermissionDenied() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) From e9037372450b49fb1cdeed0a9fc4264751be8035 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 12:41:41 -0400 Subject: [PATCH 159/252] remove handling for multi genome build searches --- seqr/views/apis/variant_search_api.py | 38 ++++++++------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 84a34fb493..6646369f62 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -29,7 +29,7 @@ from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ user_is_analyst, login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission from seqr.views.utils.project_context_utils import get_projects_child_entities -from seqr.views.utils.variant_utils import get_variant_key, get_variants_response +from seqr.views.utils.variant_utils import get_variants_response from seqr.views.utils.vlm_utils import vlm_lookup @@ -152,7 +152,8 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo return {'searchedVariants': variants} flat_variants = _flatten_variants(variants) - saved_variants, variants_by_id = _get_saved_variant_models(flat_variants, families) + variants_by_id = {v['variantId']: v for v in flat_variants} + saved_variants = _get_saved_variant_models(flat_variants) response_json = get_variants_response( request, saved_variants, response_variants=flat_variants, add_all_context=add_all_context, @@ -161,7 +162,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo for saved_variant in response_json['savedVariantsByGuid'].values(): family_guids = saved_variant['familyGuids'] - searched_variant = variants_by_id.get(get_variant_key(**saved_variant)) + searched_variant = variants_by_id.get(saved_variant['variantId']) if not searched_variant: # This can occur when an hg38 family has a saved variant that did not successfully lift from hg37 continue @@ -255,17 +256,12 @@ def export_variants_handler(request, search_hash): variants = export_variants(results_model, request.user) - saved_variants, variants_by_id = _get_saved_variant_models(variants, families) - json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=families[0].project.genome_version) + saved_variants = _get_saved_variant_models(variants) + json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants) saved_variants_by_variant_family = {} for saved_variant in json_saved_variants['savedVariantsByGuid'].values(): - variant_key = get_variant_key(**saved_variant) - searched_variant = variants_by_id.get(variant_key) - if searched_variant: - # Handles lifted over variant matching - variant_key = get_variant_key(**searched_variant) - saved_variants_by_variant_family[variant_key] = { + saved_variants_by_variant_family[saved_variant['variantId']] = { family_guid: saved_variant['variantGuid'] for family_guid in saved_variant['familyGuids'] } @@ -295,7 +291,7 @@ def export_variants_handler(request, search_hash): for variant in variants: row = [_get_field_value(variant, config) for config in VARIANT_EXPORT_DATA] - family_saved_variants = saved_variants_by_variant_family.get(get_variant_key(**variant), {}) + family_saved_variants = saved_variants_by_variant_family.get(variant['variantId'], {}) for family_guid in variant['familyGuids']: variant_guid = family_saved_variants.get(family_guid, '') family_tags = { @@ -488,24 +484,12 @@ def _get_saved_searches(user): return {'savedSearchesByGuid': {search['savedSearchGuid']: search for search in saved_searches}} -def _get_saved_variant_models(variants, families): - hg37_family_guids = families.filter(project__genome_version=GENOME_VERSION_GRCh37).values_list('guid', flat=True) if families else [] - +def _get_saved_variant_models(variants): variant_qs = [] - variants_by_id = {} variant_ids_by_family = defaultdict(set) for variant in variants: - variants_by_id[get_variant_key(**variant)] = variant for family_guid in variant.get('familyGuids', []): variant_ids_by_family[family_guid].add(variant['variantId']) - if variant.get('liftedOverGenomeVersion') == GENOME_VERSION_GRCh37 and hg37_family_guids: - variant_hg37_families = [family_guid for family_guid in variant['familyGuids'] if family_guid in hg37_family_guids] - if variant_hg37_families: - lifted_xpos = get_xpos(variant['liftedOverChrom'], variant['liftedOverPos']) - variant_qs.append(Q(xpos=lifted_xpos, ref=variant['ref'], alt=variant['alt'], family__guid__in=variant_hg37_families)) - variants_by_id[get_variant_key( - xpos=lifted_xpos, ref=variant['ref'], alt=variant['alt'], genomeVersion=variant['liftedOverGenomeVersion'] - )] = variant for family_guid, variant_ids in variant_ids_by_family.items(): variant_qs.append(Q(variant_id__in=variant_ids, family__guid=family_guid)) @@ -518,7 +502,7 @@ def _get_saved_variant_models(variants, families): else: saved_variants = SavedVariant.objects.none() - return saved_variants, variants_by_id + return saved_variants def _flatten_variants(variants): flattened_variants = [] @@ -549,7 +533,7 @@ def variant_lookup_handler(request): for variant in variants: variant['familyGuids'] = list(families.values_list('guid', flat=True)) - saved_variants, _ = _get_saved_variant_models(variants, None) if families else (None, None) + saved_variants = _get_saved_variant_models(variants) if families else None response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, From 390a93ed6fe3add03de82768156bfe12f794753e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 13:04:04 -0400 Subject: [PATCH 160/252] clean up --- seqr/views/apis/family_api.py | 2 +- seqr/views/apis/variant_search_api.py | 1 - seqr/views/utils/anvil_metadata_utils.py | 1 - seqr/views/utils/individual_utils.py | 1 - seqr/views/utils/orm_to_json_utils.py | 33 ++++++++---------------- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 538774d430..f0152e154e 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -47,7 +47,7 @@ def family_page_data(request, family_guid): sample_models = Sample.objects.filter(individual__family=family) samples = get_json_for_samples( - sample_models, project_guid=project.guid, family_guid=family_guid, skip_nested=True, is_analyst=is_analyst + sample_models, project_guid=project.guid, family_guid=family_guid, is_analyst=is_analyst ) response = { 'samplesByGuid': {s['sampleGuid']: s for s in samples} diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ceb196034c..fd1f9bdb38 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -529,7 +529,6 @@ def _get_saved_variant_models(variants, families): for family_guid in variant.get('familyGuids', []): variant_ids_by_family[family_guid].add(variant['variantId']) if variant.get('liftedOverGenomeVersion') == GENOME_VERSION_GRCh37 and hg37_family_guids: - # TODO variant_hg37_families = [family_guid for family_guid in variant['familyGuids'] if family_guid in hg37_family_guids] if variant_hg37_families: lifted_xpos = get_xpos(variant['liftedOverChrom'], variant['liftedOverPos']) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 7fb67525a9..01eb449849 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -471,7 +471,6 @@ def _get_variant_main_transcript(variant_model): main_transcript = next((t for t in transcripts if t['transcriptId'] == main_transcript_id), None) if main_transcript: if 'geneId' not in main_transcript: - # TODO main_transcript['geneId'] = gene_id return main_transcript elif len(variant.get('transcripts', {})) == 1: diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 7d3a86c7a9..b9e46d1a21 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -141,7 +141,6 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated # Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID if 'father' in record or 'mother' in record: - # TODO parent_updates.append({ 'individual': individual, 'mother': record.pop('mother', None), diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 838d43e872..f0cb1f652a 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -235,15 +235,12 @@ def _get_case_review_fields(model_cls, has_case_review_perm): def _get_json_for_families(families, user=None, add_individual_guids_field=False, project_guid=None, is_analyst=None, - has_case_review_perm=False, additional_values=None): + has_case_review_perm=False): family_additional_values = { **FAMILY_ADDITIONAL_VALUES, 'pedigreeImage': NullIf(Concat(Value(MEDIA_URL), 'pedigree_image', output_field=CharField()), Value(MEDIA_URL)), } - if additional_values: - # TODO - family_additional_values.update(additional_values) if add_individual_guids_field: family_additional_values.update(INDIVIDUAL_GUIDS_VALUES) @@ -268,8 +265,8 @@ def get_json_for_family_note(note): INDIVIDUAL_DISPLAY_NAME_EXPR = Coalesce(NullIf('display_name', Value('')), 'individual_id', output_field=CharField()) -def _get_json_for_individuals(individuals, user=None, project_guid=None, family_guid=None, add_sample_guids_field=False, - family_fields=None, add_hpo_details=False, is_analyst=None, has_case_review_perm=False): +def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False, + add_hpo_details=False, is_analyst=None, has_case_review_perm=False): """Returns a JSON representation for the given list of Individuals. Args: @@ -284,13 +281,9 @@ def _get_json_for_individuals(individuals, user=None, project_guid=None, family_ additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm) nested_fields = [ - {'fields': ('family', 'guid'), 'value': family_guid}, + {'fields': ('family', 'guid')}, {'fields': ('family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, ] - if family_fields: - # TODO - for field in family_fields: - nested_fields.append({'fields': ('family', field), 'key': _to_camel_case(field)}) if add_hpo_details: additional_model_fields += [ @@ -337,17 +330,13 @@ def add_individual_hpo_details(parsed_individuals): feature.update({'category': hpo.category_id, 'label': hpo.name}) -def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid=None, skip_nested=False, **kwargs): - if project_guid or not skip_nested: - additional_kwargs = {'nested_fields': [ - {'fields': ('individual', 'guid'), 'value': individual_guid}, - {'fields': ('individual', 'family', 'guid'), 'key': 'familyGuid', 'value': family_guid}, - {'fields': ('individual', 'family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, - ]} - else: - # TODO - additional_kwargs = {'additional_model_fields': ['individual_id']} - return {'guid_key': 'sampleGuid', **additional_kwargs, **kwargs} +def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid=None, **kwargs): + nested_fields = [ + {'fields': ('individual', 'guid'), 'value': individual_guid}, + {'fields': ('individual', 'family', 'guid'), 'key': 'familyGuid', 'value': family_guid}, + {'fields': ('individual', 'family', 'project', 'guid'), 'key': 'projectGuid', 'value': project_guid}, + ] + return {'guid_key': 'sampleGuid', 'nested_fields': nested_fields, **kwargs} def get_json_for_samples(samples, **kwargs): From 3aa89f94454be938de3ea6d250adc3d6520b26f5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 13:33:44 -0400 Subject: [PATCH 161/252] deprecate no key multi gene discovery variants --- seqr/views/utils/pedigree_info_utils.py | 4 ---- seqr/views/utils/permissions_utils.py | 8 +------- seqr/views/utils/project_context_utils.py | 17 +---------------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index c0e22a738e..83378fabc2 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -48,9 +48,6 @@ def parse_pedigree_table(parsed_file, filename, user, project): if is_merged_pedigree_sample_manifest: if not user_is_pm(user): raise ValueError('Unsupported file format') - if not project: - # TODO - raise ValueError('Project argument required for parsing sample manifest') header, rows = _parse_merged_pedigree_sample_manifest_rows(parsed_file[1:]) else: header = None @@ -307,7 +304,6 @@ def validate_fam_file_records(project, records, errors=None, clear_invalid_value sex=Individual.SEX_LOOKUP[r[JsonConstants.SEX_COLUMN]] ) if clear_invalid_values: - # TODO r[JsonConstants.PROBAND_RELATIONSHIP] = None warnings.append(f'Skipped {message}') else: diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index db208f36b2..5ec013e8f9 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -159,12 +159,6 @@ def _map_anvil_seqr_permission(anvil_permission): return CAN_VIEW if access_level == 'READER' else None -def anvil_has_perm(user, permission_level, project): - if not project_has_anvil(project): - # TODO - return False - return has_workspace_perm(user, permission_level, project.workspace_namespace, project.workspace_name) - def has_workspace_perm(user, permission_level, namespace, name, can_share=False, meta_fields=None): kwargs = {'meta_fields': meta_fields } if meta_fields else {} @@ -212,7 +206,7 @@ def has_project_permissions(project, user, can_edit=False): def _user_project_permission(user, permission_level, project): if anvil_enabled(): - return anvil_has_perm(user, permission_level, project) + return has_workspace_perm(user, permission_level, project.workspace_namespace, project.workspace_name) return user.has_perm(permission_level, project) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 22b5c41f64..adce3b5d5f 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -147,23 +147,8 @@ def families_discovery_tags(families, genome_version, project=None): } -def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): - # TODO - return next(( - gene_id for gene_id, gene_transcripts in (transcripts or {}).items() - if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts) - ), None) - - def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): - family_discovery_genes += [ - (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts)) - for family_guid, selected_main_transcript_id, transcripts in discovery_variants.filter(key__isnull=True).values_list( - 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts', - ) - ] - - tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( + tags_by_dataset_type = discovery_variants.values('dataset_type').annotate( keys=ArrayAgg('key', distinct=True), variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')), ) From d9f73b4aec5b82fdbdc09168aba17a8eac28b750 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 13:41:16 -0400 Subject: [PATCH 162/252] clean up --- seqr/views/utils/terra_api_utils_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/views/utils/terra_api_utils_tests.py b/seqr/views/utils/terra_api_utils_tests.py index 02075d3fba..8268bd06ac 100644 --- a/seqr/views/utils/terra_api_utils_tests.py +++ b/seqr/views/utils/terra_api_utils_tests.py @@ -97,8 +97,7 @@ def _check_exceptions(self, path, func, args, kwargs=None, responses_body=None): with self.assertRaises(exception) as ec: _ = func(*args, **kwargs) self.assertEqual(str(ec.exception), message.format(path=path)) - if isinstance(exception, TerraAPIException): - # TODO + if status == 401: self.assertEqual(ec.exception.status_code, status) self.assert_no_logs() From d1d074319741f89c69f75949c1bb5d60f83b6f41 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 14:00:25 -0400 Subject: [PATCH 163/252] fixture clean up --- .../tests/set_saved_variant_key_tests.py | 180 +++++++++++++++++- seqr/fixtures/1kg_project.json | 142 +------------- seqr/fixtures/report_variants.json | 38 +--- seqr/views/utils/test_utils.py | 3 - 4 files changed, 183 insertions(+), 180 deletions(-) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 9084105a3d..76e4cfe78a 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -8,12 +8,188 @@ b'chr\tstart\tend\tname\tsample\tsample_fix\tsvtype\tGT\tCN\tNP\tQA\tQS\tQSE\tQSS\tploidy\tstrand\tvariant_name\tID\trmsstd\tdefragmented\tvaf\tvac\tlt100_raw_calls\tlt10_highQS_rare_calls\tPASS_SAMPLE\tPASS_FREQ\tPASS_QS\tHIGH_QUALITY\tgenes_any_overlap\tgenes_any_overlap_exonsPerGene\tgenes_any_overlap_totalExons\tgenes_strict_overlap\tgenes_strict_overlap_exonsPerGene\tgenes_strict_overlap_totalExons\tgenes_CG\tgenes_LOF\tgenes_any_overlap_Ensemble_ID\tidentical\tpartial_ovl\tany_ovl\tno_ovl\tsource', b'chr10\t122599967\t122600191\tCASE_7_Y_cnv_57308\tC1933_010-001_v1_Exome_GCP\t010-001_v1_Exome_C1933\tDEL\t1\t1\t1\t7\t7\t7\t72\t\t-\tprefix_70191\t326536\t0\tFALSE\t0.05066805\t1077\tTRUE\tTRUE\tTRUE\tFALSE\tFALSE\tFALSE\tDMBT1\t1\t1\tNA\t0\t0\tNA\tDMBT1\tENSG00000187908.19\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117;prefix_19107\tFALSE\tround2', ] - +# +# SAVED_VARIANT_JSON = { +# 'SV0000002_1248367227_r0390_100': { +# "clinvar": { +# "pathogenicity": "Uncertain_significance", +# "alleleId": 12345, +# "assertions": null, +# "conditions": null, +# "conflictingPathogenicities": null, +# "submitters": null, +# "goldStars": null +# }, +# "liftedOverGenomeVersion": "38", +# "familyGuids": [ +# "F000001_1" +# ], +# "liftedOverPos": "", +# "populations": { +# "callset": { +# "ac": null, +# "an": null, +# "af": null +# }, +# "g1k": { +# "ac": null, +# "an": null, +# "af": 0.0 +# }, +# "gnomad_genomes": { +# "hemi": null, +# "ac": null, +# "an": null, +# "hom": null, +# "af": 0.00012925741614425127 +# }, +# "gnomad_exomes": { +# "hemi": null, +# "ac": null, +# "an": null, +# "hom": null, +# "af": 6.505916317651364e-05 +# }, +# "exac": { +# "hemi": null, +# "ac": null, +# "an": null, +# "hom": null, +# "af": 0.0006726888333653661 +# }, +# "topmed": { +# "ac": null, +# "an": null, +# "af": null +# } +# }, +# "genomeVersion": "37", +# "pos": 248367227, +# "predictions": { +# "eigen": null, +# "revel": null, +# "sift": null, +# "cadd": "27.2", +# "metasvm": "", +# "mpc": null, +# "splice_ai": null, +# "phastcons_100_vert": null, +# "mut_taster": null, +# "fathmm": null, +# "polyphen": null, +# "dann": null, +# "primate_ai": null, +# "gerp_rs": null +# }, +# "hgmd": { +# "accession": null, +# "class": null +# }, +# "rsid": null, +# "liftedOverChrom": "", +# "transcripts": { +# "ENSG00000135953": [ +# {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000135953"} +# ], +# "ENSG00000240361": [] +# }, +# "chrom": "1", +# "genotypes": { +# "I000004_hg00731": { +# "numAlt": 2 +# }, +# "I000005_hg00732": { +# "numAlt": 1 +# } +# }, +# "CAID": "CA1501729" +# }, +# 'SV0000006_1248367227_r0003_tes': { +# "clinvar": { +# "pathogenicity": "Uncertain_significance", +# "alleleId": 12345, +# "assertions": null, +# "conditions": null, +# "conflictingPathogenicities": null, +# "submitters": null, +# "goldStars": null +# }, +# "liftedOverGenomeVersion": "38", "liftedOverPos": "", +# "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, +# "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.00012925741614425127}, +# "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 6.505916317651364e-05}, +# "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0006726888333653661}, +# "topmed": {"ac": null, "an": null, "af": null}}, +# "genomeVersion": "37", "pos": 248367227, "predictions": { +# "eigen": null, "revel": null, "sift": null, "cadd": "27.2", "metasvm": "", "mpc": null, +# "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, +# "dann": null, "primate_ai": null, "gerp_rs": null}, +# "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", +# "mainTranscriptId": "ENST00000505820", "transcripts": { +# "ENSG00000135953": [ +# {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000228198"} +# ], +# "ENSG00000240361": [ +# {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", +# "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, +# "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", +# "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000240361", +# "aminoAcids": "V/M", "cdnaPosition": "3955"} +# ] +# }, "chrom": "1", "genotypes": { +# "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, +# "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} +# }, +# "CAID": "CA1501729" +# }, +# 'SV0000006_1248367227_r0004_non': { +# "liftedOverGenomeVersion": "37", "liftedOverPos": "", "genomeVersion": "38", "pos": 248367227, +# "transcripts": {}, "chrom": "1", "genotypes": { +# "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} +# }, +# "CAID": "CA1501729" +# }, +# 'SV0000009_25000014783_r0004_no': { +# "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, +# "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, +# "rsid": "rs193302982", "familyGuids": ["F000002_2"], "genotypes": { +# "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} +# }, "populations": {"seqr": {"af": 0.019480518996715546, "ac": 3, "an": 154}, "seqr_heteroplasmy": {"af": 0.006493506487458944, "ac": 1, "an": 154}, +# "gnomad_mito": {"af": 0.05534649267792702, "ac": 3118, "an": 56336}, "gnomad_mito_heteroplasmy": {"af": 5.3251918870955706e-05, "ac": 3, "an": 56336, "max_hl": 1.0}, +# "helix": {"af": 0.04884607344865799, "ac": 9573, "an": 195983}, "helix_heteroplasmy": {"af": 9.184470400214195e-05, "ac": 18, "an": 195983, "max_hl": 0.962689995765686}}, +# "predictions": {"apogee": null, "haplogroup_defining": "Y", "hmtvar": null, "mitotip": null, "mut_taster": null, "sift": null, "mlc": 0.7514}, +# "commonLowHeteroplasmy": true, "mitomapPathogenic": true, "clinvar": null, "transcripts": { +# "ENSG00000198727": [{"aminoAcids": "L", "canonical": 1, "codons": "Tta/Cta", "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "transcriptId": "ENST00000361789", "isLofNagnag": null, "transcriptRank": 0, "biotype": "protein_coding", "lofFilters": null, "majorConsequence": "synonymous_variant", "consequenceTerms": ["synonymous_variant"]}]}, +# "mainTranscriptId": "ENST00000361789", "selectedMainTranscriptId": null +# }, +# 'SV0000013_prefix_19107_DEL_r00': { +# "liftedOverGenomeVersion": null, +# "pos": 249045487, +# "end": 249045898, +# "xpos": 1249045487, +# "predictions": {"strvctvre": 0.374}, +# "alt": null, +# "numExon": 2, +# "genotypeFilters": [], +# "ref": null, +# "genotypes": { +# "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} +# }, +# "liftedOverPos": null, +# "liftedOverChrom": null, +# "svType": "DEL", +# "variantId": "suffix_19107_DEL", +# "chrom": "1", +# "endChrom": "1", +# "genomeVersion": "37", +# "populations": {"sv_callset": {}}, +# "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} +# } +# } class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] - SKIP_RESET_VARIANT_JSON = True MOCK_GCNV_DATA = MOCK_GCNV_DATA @classmethod diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 8eab981a31..2d93844b42 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1922,100 +1922,7 @@ } }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": { - "clinvar": { - "pathogenicity": "Uncertain_significance", - "alleleId": 12345, - "assertions": null, - "conditions": null, - "conflictingPathogenicities": null, - "submitters": null, - "goldStars": null - }, - "liftedOverGenomeVersion": "38", - "familyGuids": [ - "F000001_1" - ], - "liftedOverPos": "", - "populations": { - "callset": { - "ac": null, - "an": null, - "af": null - }, - "g1k": { - "ac": null, - "an": null, - "af": 0.0 - }, - "gnomad_genomes": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 0.00012925741614425127 - }, - "gnomad_exomes": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 6.505916317651364e-05 - }, - "exac": { - "hemi": null, - "ac": null, - "an": null, - "hom": null, - "af": 0.0006726888333653661 - }, - "topmed": { - "ac": null, - "an": null, - "af": null - } - }, - "genomeVersion": "37", - "pos": 248367227, - "predictions": { - "eigen": null, - "revel": null, - "sift": null, - "cadd": "27.2", - "metasvm": "", - "mpc": null, - "splice_ai": null, - "phastcons_100_vert": null, - "mut_taster": null, - "fathmm": null, - "polyphen": null, - "dann": null, - "primate_ai": null, - "gerp_rs": null - }, - "hgmd": { - "accession": null, - "class": null - }, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000135953"} - ], - "ENSG00000240361": [] - }, - "chrom": "1", - "genotypes": { - "I000004_hg00731": { - "numAlt": 2 - }, - "I000005_hg00732": { - "numAlt": 1 - } - }, - "CAID": "CA1501729" - }, + "saved_variant_json": {}, "selected_main_transcript_id": "ENST00000371839", "family": 2 } @@ -2166,44 +2073,7 @@ "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": { - "clinvar": { - "pathogenicity": "Uncertain_significance", - "alleleId": 12345, - "assertions": null, - "conditions": null, - "conflictingPathogenicities": null, - "submitters": null, - "goldStars": null - }, - "liftedOverGenomeVersion": "38", "liftedOverPos": "", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, - "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.00012925741614425127}, - "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 6.505916317651364e-05}, - "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0006726888333653661}, - "topmed": {"ac": null, "an": null, "af": null}}, - "genomeVersion": "37", "pos": 248367227, "predictions": { - "eigen": null, "revel": null, "sift": null, "cadd": "27.2", "metasvm": "", "mpc": null, - "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, - "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", - "mainTranscriptId": "ENST00000505820", "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000228198"} - ], - "ENSG00000240361": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", - "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, - "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", - "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000240361", - "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, "chrom": "1", "genotypes": { - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, - "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} - }, - "CAID": "CA1501729" - }, + "saved_variant_json": {}, "family": 12 } }, @@ -2264,13 +2134,7 @@ "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", "liftedOverPos": "", "genomeVersion": "38", "pos": 248367227, - "transcripts": {}, "chrom": "1", "genotypes": { - "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} - }, - "CAID": "CA1501729" - }, + "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 0d64249d1c..02a9f039fe 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -149,19 +149,7 @@ "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} }, "gene_ids": ["ENSG00000198727"], - "saved_variant_json": { - "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, - "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, - "rsid": "rs193302982", "familyGuids": ["F000002_2"], "genotypes": { - "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} - }, "populations": {"seqr": {"af": 0.019480518996715546, "ac": 3, "an": 154}, "seqr_heteroplasmy": {"af": 0.006493506487458944, "ac": 1, "an": 154}, - "gnomad_mito": {"af": 0.05534649267792702, "ac": 3118, "an": 56336}, "gnomad_mito_heteroplasmy": {"af": 5.3251918870955706e-05, "ac": 3, "an": 56336, "max_hl": 1.0}, - "helix": {"af": 0.04884607344865799, "ac": 9573, "an": 195983}, "helix_heteroplasmy": {"af": 9.184470400214195e-05, "ac": 18, "an": 195983, "max_hl": 0.962689995765686}}, - "predictions": {"apogee": null, "haplogroup_defining": "Y", "hmtvar": null, "mitotip": null, "mut_taster": null, "sift": null, "mlc": 0.7514}, - "commonLowHeteroplasmy": true, "mitomapPathogenic": true, "clinvar": null, "transcripts": { - "ENSG00000198727": [{"aminoAcids": "L", "canonical": 1, "codons": "Tta/Cta", "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "transcriptId": "ENST00000361789", "isLofNagnag": null, "transcriptRank": 0, "biotype": "protein_coding", "lofFilters": null, "majorConsequence": "synonymous_variant", "consequenceTerms": ["synonymous_variant"]}]}, - "mainTranscriptId": "ENST00000361789", "selectedMainTranscriptId": null - }, + "saved_variant_json": {}, "family": 14 } }, { @@ -183,29 +171,7 @@ "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], - "saved_variant_json": { - "liftedOverGenomeVersion": null, - "pos": 249045487, - "end": 249045898, - "xpos": 1249045487, - "predictions": {"strvctvre": 0.374}, - "alt": null, - "numExon": 2, - "genotypeFilters": [], - "ref": null, - "genotypes": { - "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} - }, - "liftedOverPos": null, - "liftedOverChrom": null, - "svType": "DEL", - "variantId": "suffix_19107_DEL", - "chrom": "1", - "endChrom": "1", - "genomeVersion": "37", - "populations": {"sv_callset": {}}, - "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} - }, + "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 4af77065fb..985c7485c7 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -593,9 +593,6 @@ def set_up_test(self): self.mock_get_group_members.side_effect = get_group_members_side_effect self.addCleanup(patcher.stop) super().set_up_test() - if self.CLICKHOUSE_HOSTNAME and not self.SKIP_RESET_VARIANT_JSON: - # TODO update fixture data to match - SavedVariant.objects.filter(key__isnull=False).update(saved_variant_json={}) @classmethod def add_additional_user_groups(cls): From 59bfb89ed97903d41e51bce6498763b2e7ea3453 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 14:09:29 -0400 Subject: [PATCH 164/252] fix set key tests --- .../tests/set_saved_variant_key_tests.py | 186 +----------------- 1 file changed, 8 insertions(+), 178 deletions(-) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 76e4cfe78a..ba61b38d94 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -8,184 +8,7 @@ b'chr\tstart\tend\tname\tsample\tsample_fix\tsvtype\tGT\tCN\tNP\tQA\tQS\tQSE\tQSS\tploidy\tstrand\tvariant_name\tID\trmsstd\tdefragmented\tvaf\tvac\tlt100_raw_calls\tlt10_highQS_rare_calls\tPASS_SAMPLE\tPASS_FREQ\tPASS_QS\tHIGH_QUALITY\tgenes_any_overlap\tgenes_any_overlap_exonsPerGene\tgenes_any_overlap_totalExons\tgenes_strict_overlap\tgenes_strict_overlap_exonsPerGene\tgenes_strict_overlap_totalExons\tgenes_CG\tgenes_LOF\tgenes_any_overlap_Ensemble_ID\tidentical\tpartial_ovl\tany_ovl\tno_ovl\tsource', b'chr10\t122599967\t122600191\tCASE_7_Y_cnv_57308\tC1933_010-001_v1_Exome_GCP\t010-001_v1_Exome_C1933\tDEL\t1\t1\t1\t7\t7\t7\t72\t\t-\tprefix_70191\t326536\t0\tFALSE\t0.05066805\t1077\tTRUE\tTRUE\tTRUE\tFALSE\tFALSE\tFALSE\tDMBT1\t1\t1\tNA\t0\t0\tNA\tDMBT1\tENSG00000187908.19\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117;prefix_19107\tFALSE\tround2', ] -# -# SAVED_VARIANT_JSON = { -# 'SV0000002_1248367227_r0390_100': { -# "clinvar": { -# "pathogenicity": "Uncertain_significance", -# "alleleId": 12345, -# "assertions": null, -# "conditions": null, -# "conflictingPathogenicities": null, -# "submitters": null, -# "goldStars": null -# }, -# "liftedOverGenomeVersion": "38", -# "familyGuids": [ -# "F000001_1" -# ], -# "liftedOverPos": "", -# "populations": { -# "callset": { -# "ac": null, -# "an": null, -# "af": null -# }, -# "g1k": { -# "ac": null, -# "an": null, -# "af": 0.0 -# }, -# "gnomad_genomes": { -# "hemi": null, -# "ac": null, -# "an": null, -# "hom": null, -# "af": 0.00012925741614425127 -# }, -# "gnomad_exomes": { -# "hemi": null, -# "ac": null, -# "an": null, -# "hom": null, -# "af": 6.505916317651364e-05 -# }, -# "exac": { -# "hemi": null, -# "ac": null, -# "an": null, -# "hom": null, -# "af": 0.0006726888333653661 -# }, -# "topmed": { -# "ac": null, -# "an": null, -# "af": null -# } -# }, -# "genomeVersion": "37", -# "pos": 248367227, -# "predictions": { -# "eigen": null, -# "revel": null, -# "sift": null, -# "cadd": "27.2", -# "metasvm": "", -# "mpc": null, -# "splice_ai": null, -# "phastcons_100_vert": null, -# "mut_taster": null, -# "fathmm": null, -# "polyphen": null, -# "dann": null, -# "primate_ai": null, -# "gerp_rs": null -# }, -# "hgmd": { -# "accession": null, -# "class": null -# }, -# "rsid": null, -# "liftedOverChrom": "", -# "transcripts": { -# "ENSG00000135953": [ -# {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000135953"} -# ], -# "ENSG00000240361": [] -# }, -# "chrom": "1", -# "genotypes": { -# "I000004_hg00731": { -# "numAlt": 2 -# }, -# "I000005_hg00732": { -# "numAlt": 1 -# } -# }, -# "CAID": "CA1501729" -# }, -# 'SV0000006_1248367227_r0003_tes': { -# "clinvar": { -# "pathogenicity": "Uncertain_significance", -# "alleleId": 12345, -# "assertions": null, -# "conditions": null, -# "conflictingPathogenicities": null, -# "submitters": null, -# "goldStars": null -# }, -# "liftedOverGenomeVersion": "38", "liftedOverPos": "", -# "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, -# "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.00012925741614425127}, -# "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 6.505916317651364e-05}, -# "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0006726888333653661}, -# "topmed": {"ac": null, "an": null, "af": null}}, -# "genomeVersion": "37", "pos": 248367227, "predictions": { -# "eigen": null, "revel": null, "sift": null, "cadd": "27.2", "metasvm": "", "mpc": null, -# "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, -# "dann": null, "primate_ai": null, "gerp_rs": null}, -# "hgmd": {"accession": null, "class": null}, "rsid": null, "liftedOverChrom": "", -# "mainTranscriptId": "ENST00000505820", "transcripts": { -# "ENSG00000135953": [ -# {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000228198"} -# ], -# "ENSG00000240361": [ -# {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", -# "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, -# "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", -# "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000240361", -# "aminoAcids": "V/M", "cdnaPosition": "3955"} -# ] -# }, "chrom": "1", "genotypes": { -# "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, -# "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} -# }, -# "CAID": "CA1501729" -# }, -# 'SV0000006_1248367227_r0004_non': { -# "liftedOverGenomeVersion": "37", "liftedOverPos": "", "genomeVersion": "38", "pos": 248367227, -# "transcripts": {}, "chrom": "1", "genotypes": { -# "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} -# }, -# "CAID": "CA1501729" -# }, -# 'SV0000009_25000014783_r0004_no': { -# "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, -# "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, -# "rsid": "rs193302982", "familyGuids": ["F000002_2"], "genotypes": { -# "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} -# }, "populations": {"seqr": {"af": 0.019480518996715546, "ac": 3, "an": 154}, "seqr_heteroplasmy": {"af": 0.006493506487458944, "ac": 1, "an": 154}, -# "gnomad_mito": {"af": 0.05534649267792702, "ac": 3118, "an": 56336}, "gnomad_mito_heteroplasmy": {"af": 5.3251918870955706e-05, "ac": 3, "an": 56336, "max_hl": 1.0}, -# "helix": {"af": 0.04884607344865799, "ac": 9573, "an": 195983}, "helix_heteroplasmy": {"af": 9.184470400214195e-05, "ac": 18, "an": 195983, "max_hl": 0.962689995765686}}, -# "predictions": {"apogee": null, "haplogroup_defining": "Y", "hmtvar": null, "mitotip": null, "mut_taster": null, "sift": null, "mlc": 0.7514}, -# "commonLowHeteroplasmy": true, "mitomapPathogenic": true, "clinvar": null, "transcripts": { -# "ENSG00000198727": [{"aminoAcids": "L", "canonical": 1, "codons": "Tta/Cta", "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "transcriptId": "ENST00000361789", "isLofNagnag": null, "transcriptRank": 0, "biotype": "protein_coding", "lofFilters": null, "majorConsequence": "synonymous_variant", "consequenceTerms": ["synonymous_variant"]}]}, -# "mainTranscriptId": "ENST00000361789", "selectedMainTranscriptId": null -# }, -# 'SV0000013_prefix_19107_DEL_r00': { -# "liftedOverGenomeVersion": null, -# "pos": 249045487, -# "end": 249045898, -# "xpos": 1249045487, -# "predictions": {"strvctvre": 0.374}, -# "alt": null, -# "numExon": 2, -# "genotypeFilters": [], -# "ref": null, -# "genotypes": { -# "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} -# }, -# "liftedOverPos": null, -# "liftedOverChrom": null, -# "svType": "DEL", -# "variantId": "suffix_19107_DEL", -# "chrom": "1", -# "endChrom": "1", -# "genomeVersion": "37", -# "populations": {"sv_callset": {}}, -# "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} -# } -# } + class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] @@ -196,6 +19,13 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): def setUpTestData(cls): Project.objects.filter(id=3).update(genome_version='38') Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) + for sv in SavedVariant.objects.filter(key__isnull=False): + sv.saved_variant_json = { + 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, + } + if sv.guid == 'SV0000009_25000014783_r0004_no': + sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} + sv.save() SavedVariant.objects.update(key=None) @mock.patch('clickhouse_search.management.commands.set_saved_variant_key.BATCH_SIZE', 2) From 9da73b960f798f8d0d3ca38eec3dc7193da2efbe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 14:45:05 -0400 Subject: [PATCH 165/252] remove inaccurate docstring --- seqr/views/utils/orm_to_json_utils.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index f0cb1f652a..44a0d4c803 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -267,18 +267,6 @@ def get_json_for_family_note(note): def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False, add_hpo_details=False, is_analyst=None, has_case_review_perm=False): - """Returns a JSON representation for the given list of Individuals. - - Args: - individuals (array): array of django models for the individual. - user (object): Django User object for determining whether to include restricted/internal-only fields - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - family_guid (boolean): An optional field to use as the familyGuid instead of querying the DB - add_sample_guids_field (boolean): A flag to indicate weather sample ids should be added - Returns: - array: array of json objects - """ - additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm) nested_fields = [ {'fields': ('family', 'guid')}, From d31a3e7fbbe1a54cef0e414e42ce74ef52a82909 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 15:35:07 -0400 Subject: [PATCH 166/252] restore gene lookup page; --- .../components/GeneVariantLookup.jsx | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 ui/pages/SummaryData/components/GeneVariantLookup.jsx diff --git a/ui/pages/SummaryData/components/GeneVariantLookup.jsx b/ui/pages/SummaryData/components/GeneVariantLookup.jsx new file mode 100644 index 0000000000..19b316a0ec --- /dev/null +++ b/ui/pages/SummaryData/components/GeneVariantLookup.jsx @@ -0,0 +1,129 @@ +import React from 'react' +import { connect } from 'react-redux' +import PropTypes from 'prop-types' +import { Grid, Header } from 'semantic-ui-react' + +import { validators } from 'shared/components/form/FormHelpers' +import FormWrapper from 'shared/components/form/FormWrapper' +import { AlignedCheckboxGroup } from 'shared/components/form/Inputs' +import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' +import { Variant } from 'shared/components/panel/variants/Variants' +import { + GENE_SEARCH_FREQUENCIES, + GENOME_VERSION_FIELD, + GROUPED_VEP_CONSEQUENCES, + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +} from 'shared/utils/constants' +import { snakecaseToTitlecase } from 'shared/utils/stringUtils' +import { loadGeneVariantLookup } from '../reducers' +import { getGeneVariantLookupLoading, getGeneVariantLookupResults } from '../selectors' + +const validateAnnotations = (value, { annotations }) => ( + value || Object.values(annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' +) + +const CONSEQUENCE_FILEDS = [ + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +].map((group, i) => ({ + name: `annotations.${group}`, + component: AlignedCheckboxGroup, + groupLabel: snakecaseToTitlecase(group), + options: GROUPED_VEP_CONSEQUENCES[group], + format: value => value || [], + inline: true, + validate: i === 0 ? validateAnnotations : undefined, +})) + +const FIELDS = [ + { validate: validators.required, ...GENOME_VERSION_FIELD }, + { + name: 'geneId', + label: 'Gene', + control: AwesomeBarFormInput, + categories: ['genes'], + fluid: true, + placeholder: 'Search for gene', + validate: validators.required, + }, + ...CONSEQUENCE_FILEDS, +] + +const INITIAL_VALUES = { freqs: GENE_SEARCH_FREQUENCIES } + +const GeneVariantLookup = ({ loading, results, onSubmit }) => ( + + + +
+ Lookup up all rare variants is seqr in a given gene, regardless of whether or not they are in your + projects. +
+ Variants are only returned if they have a gnomAD Allele Frequency < 3% + and have a seqr global Allele Count < 3000. + + )} + /> + + + + + + + + + + {!loading && results && ( + + + Found   + {results.length} +   variants + + + )} + {!loading && results?.map(variant => ( + + ))} + +) + +GeneVariantLookup.propTypes = { + loading: PropTypes.bool, + results: PropTypes.arrayOf(PropTypes.object), + onSubmit: PropTypes.func, +} + +const mapStateToProps = state => ({ + loading: getGeneVariantLookupLoading(state), + results: getGeneVariantLookupResults(state), +}) + +const mapDispatchToProps = { + onSubmit: loadGeneVariantLookup, +} + +export default connect(mapStateToProps, mapDispatchToProps)(GeneVariantLookup) From e186172512466a11dba5cba580eac1ba458b153f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 15:51:48 -0400 Subject: [PATCH 167/252] top level variant lookup --- seqr/urls.py | 1 + ui/app.jsx | 2 ++ .../components => Search}/VariantLookup.jsx | 5 ++- ui/pages/Search/reducers.js | 5 +++ ui/pages/Search/selectors.js | 32 ++++++++++++++++- ui/pages/SummaryData/SummaryData.jsx | 2 -- ui/pages/SummaryData/reducers.js | 4 --- ui/pages/SummaryData/selectors.js | 34 ------------------- ui/shared/components/page/Header.jsx | 1 + .../components/panel/variants/Annotations.jsx | 2 +- 10 files changed, 43 insertions(+), 45 deletions(-) rename ui/pages/{SummaryData/components => Search}/VariantLookup.jsx (98%) diff --git a/seqr/urls.py b/seqr/urls.py index 8204a9f8f4..5e00389df9 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -158,6 +158,7 @@ 'gene_info/.*', 'gene_lists/.*', 'variant_search/.*', + 'variant_lookup/.*', 'report/.*', 'data_management/.*', 'summary_data/.*', diff --git a/ui/app.jsx b/ui/app.jsx index 619f7e0a63..de187a0b60 100644 --- a/ui/app.jsx +++ b/ui/app.jsx @@ -10,6 +10,7 @@ import Dashboard from 'pages/Dashboard/Dashboard' import Project from 'pages/Project/Project' import LoadWorkspaceData, { WorkspaceAccessError } from 'pages/AnvilWorkspace/LoadWorkspaceData' import VariantSearch from 'pages/Search/VariantSearch' +import VariantLookup from 'pages/Search/VariantLookup' import DataManagement from 'pages/DataManagement/DataManagement' import Report from 'pages/Report/Report' import SummaryData from 'pages/SummaryData/SummaryData' @@ -50,6 +51,7 @@ ReactDOM.render( + diff --git a/ui/pages/SummaryData/components/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx similarity index 98% rename from ui/pages/SummaryData/components/VariantLookup.jsx rename to ui/pages/Search/VariantLookup.jsx index 961f236a2a..218f01b9b1 100644 --- a/ui/pages/SummaryData/components/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -16,8 +16,8 @@ import FamilyVariantTags from 'shared/components/panel/variants/FamilyVariantTag import Variants, { Variant, StyledVariantRow } from 'shared/components/panel/variants/Variants' import { FamilyVariantIndividuals } from 'shared/components/panel/variants/VariantIndividuals' import { GENOME_VERSION_FIELD, GENOME_VERSION_37, GENOME_VERSION_38 } from 'shared/utils/constants' -import { sendVlmContactEmail } from '../reducers' -import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail } from '../selectors' +import { sendVlmContactEmail } from './reducers' +import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail } from './selectors' const FIELDS = [ { @@ -205,7 +205,6 @@ const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled -
diff --git a/ui/pages/Search/reducers.js b/ui/pages/Search/reducers.js index 8d3e38a32b..8c8197d9e6 100644 --- a/ui/pages/Search/reducers.js +++ b/ui/pages/Search/reducers.js @@ -219,6 +219,11 @@ export const loadGeneBreakdown = searchHash => (dispatch, getState) => { } } +// TODO update url path +export const sendVlmContactEmail = values => () => new HttpRequestHelper( + '/api/summary_data/send_vlm_email', +).post(values) + // reducers export const reducers = { diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 0b646d900d..6fea91ec0d 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -14,8 +14,9 @@ import { getSearchesByHash, getProjectDatasetTypes, getActiveDatasetsByFamily, + getSortedIndividualsByFamily, } from 'redux/selectors' -import { FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' +import { getVariantMainGeneId, getVariantSummary, FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' import { compareObjects } from 'shared/utils/sortUtils' import { compHetGene } from 'shared/components/panel/variants/VariantUtils' @@ -278,3 +279,32 @@ export const getSearchGeneBreakdownValues = createSelector( }), })), ) + +export const getVlmFamiliesByContactEmail = createSelector( + getSortedIndividualsByFamily, + (state, ownProps) => ownProps.variant, + (individualsByFamily, variant) => (variant.lookupFamilyGuids || []).reduce((acc, familyGuid) => { + const individual = individualsByFamily[familyGuid]?.[0] + const contactEmail = individual?.projectGuid ? 'internal' : (individual?.vlmContactEmail || 'disabled') + return { ...acc, [contactEmail]: [...(acc[contactEmail] || []), familyGuid] } + }, {}), +) + +export const getVlmDefaultContactEmails = createSelector( + getVlmFamiliesByContactEmail, + getGenesById, + getUser, + (state, ownProps) => ownProps.variant, + (familiesByContactEmail, genesById, user, variant) => { + const gene = genesById[getVariantMainGeneId(variant)]?.geneSymbol + const subject = `${gene || variant.variantId} variant match in seqr` + const defaultEmailContent = `harboring ${getVariantSummary(variant)} in ${gene || 'no genes'} (${window.location.href}). + \n\nWe have identified the variant in a case with [replace with phenotype]. + \n\n[List your specific questions for the researcher here.] + \n\nWe appreciate your assistance and look forward to hearing more from you.\n\nBest wishes,\n${user.displayName}` + return Object.entries(familiesByContactEmail).reduce((acc, [to, familyGuids]) => ({ + ...acc, + [to]: { to, subject, body: `Dear researcher,\n\nWe are interested in learning more about your ${familyGuids.length} cases in seqr ${defaultEmailContent}` }, + }), {}) + }, +) diff --git a/ui/pages/SummaryData/SummaryData.jsx b/ui/pages/SummaryData/SummaryData.jsx index e683abcb51..01bc14758d 100644 --- a/ui/pages/SummaryData/SummaryData.jsx +++ b/ui/pages/SummaryData/SummaryData.jsx @@ -16,7 +16,6 @@ import LocusLists from './components/LocusLists' import ExternalAnalysis from './components/ExternalAnalysis' import Hpo from './components/Hpo' import IndividualMetadata from './components/IndividualMetadata' -import VariantLookup from './components/VariantLookup' const GenePage = ({ match }) => ( match.params.geneId ? : @@ -27,7 +26,6 @@ GenePage.propTypes = { } const SUMMARY_DATA_PAGES = [ - { path: 'variant_lookup', component: VariantLookup }, { path: 'gene_info', params: '/:geneId?', component: GenePage }, { path: 'gene_lists', component: LocusLists }, { path: 'saved_variants', component: SavedVariants }, diff --git a/ui/pages/SummaryData/reducers.js b/ui/pages/SummaryData/reducers.js index a3111e8086..761f79b87e 100644 --- a/ui/pages/SummaryData/reducers.js +++ b/ui/pages/SummaryData/reducers.js @@ -78,10 +78,6 @@ export const updateExternalAnalysis = values => dispatch => new HttpRequestHelpe }, ).post(values) -export const sendVlmContactEmail = values => () => new HttpRequestHelper( - '/api/summary_data/send_vlm_email', -).post(values) - export const reducers = { successStoryLoading: loadingReducer(REQUEST_SUCCESS_STORY, RECEIVE_SUCCESS_STORY), successStoryRows: createSingleValueReducer(RECEIVE_SUCCESS_STORY, []), diff --git a/ui/pages/SummaryData/selectors.js b/ui/pages/SummaryData/selectors.js index 236dd98249..9676010112 100644 --- a/ui/pages/SummaryData/selectors.js +++ b/ui/pages/SummaryData/selectors.js @@ -1,8 +1,3 @@ -import { createSelector } from 'reselect' - -import { getSortedIndividualsByFamily, getGenesById, getUser } from 'redux/selectors' -import { getVariantMainGeneId, getVariantSummary } from 'shared/utils/constants' - export const getSuccessStoryLoading = state => state.successStoryLoading.isLoading export const getSuccessStoryLoadingError = state => state.successStoryLoading.errorMessage export const getSuccessStoryRows = state => state.successStoryRows @@ -11,32 +6,3 @@ export const getMmeLoadingError = state => state.mmeLoading.errorMessage export const getMmeMetrics = state => state.mmeMetrics export const getMmeSubmissions = state => state.mmeSubmissions export const getExternalAnalysisUploadStats = state => state.externalAnalysisUploadStats - -export const getVlmFamiliesByContactEmail = createSelector( - getSortedIndividualsByFamily, - (state, ownProps) => ownProps.variant, - (individualsByFamily, variant) => (variant.lookupFamilyGuids || []).reduce((acc, familyGuid) => { - const individual = individualsByFamily[familyGuid]?.[0] - const contactEmail = individual?.projectGuid ? 'internal' : (individual?.vlmContactEmail || 'disabled') - return { ...acc, [contactEmail]: [...(acc[contactEmail] || []), familyGuid] } - }, {}), -) - -export const getVlmDefaultContactEmails = createSelector( - getVlmFamiliesByContactEmail, - getGenesById, - getUser, - (state, ownProps) => ownProps.variant, - (familiesByContactEmail, genesById, user, variant) => { - const gene = genesById[getVariantMainGeneId(variant)]?.geneSymbol - const subject = `${gene || variant.variantId} variant match in seqr` - const defaultEmailContent = `harboring ${getVariantSummary(variant)} in ${gene || 'no genes'} (${window.location.href}). - \n\nWe have identified the variant in a case with [replace with phenotype]. - \n\n[List your specific questions for the researcher here.] - \n\nWe appreciate your assistance and look forward to hearing more from you.\n\nBest wishes,\n${user.displayName}` - return Object.entries(familiesByContactEmail).reduce((acc, [to, familyGuids]) => ({ - ...acc, - [to]: { to, subject, body: `Dear researcher,\n\nWe are interested in learning more about your ${familyGuids.length} cases in seqr ${defaultEmailContent}` }, - }), {}) - }, -) diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index 1afbc17906..9a06d787ea 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -26,6 +26,7 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => {
seqr
 {Object.keys(user).length ? [ , + , , (user.isAnalyst || user.isPm) ? : null, (user.isDataManager || user.isPm) ? : null, diff --git a/ui/shared/components/panel/variants/Annotations.jsx b/ui/shared/components/panel/variants/Annotations.jsx index 6d0a58613a..fbb99e52fd 100644 --- a/ui/shared/components/panel/variants/Annotations.jsx +++ b/ui/shared/components/panel/variants/Annotations.jsx @@ -352,7 +352,7 @@ const variantSearchLinks = (variant, mainTranscript, genesById, user) => { const seqrSearchLink = ( Variant Lookup From 5130a0f04c7d7df35a73f1562217bae6b22cc353 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 16:09:42 -0400 Subject: [PATCH 168/252] add gene lookup page --- ui/pages/Search/VariantLookup.jsx | 21 ++++++++++++++++++- .../components/GeneVariantLookup.jsx | 10 ++++----- ui/shared/components/page/Header.jsx | 1 + ui/shared/components/page/PageHeader.jsx | 1 + 4 files changed, 27 insertions(+), 6 deletions(-) rename ui/pages/{SummaryData => Search}/components/GeneVariantLookup.jsx (92%) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 218f01b9b1..590d98c39a 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -1,6 +1,7 @@ import React from 'react' import { connect } from 'react-redux' import PropTypes from 'prop-types' +import { Route, Switch } from 'react-router-dom' import { Grid, Header, Label, Table } from 'semantic-ui-react' import { RECEIVE_DATA } from 'redux/utils/reducerUtils' @@ -14,8 +15,10 @@ import { BaseSemanticInput, InlineToggle } from 'shared/components/form/Inputs' import FamilyReads from 'shared/components/panel/family/FamilyReads' import FamilyVariantTags from 'shared/components/panel/variants/FamilyVariantTags' import Variants, { Variant, StyledVariantRow } from 'shared/components/panel/variants/Variants' +import { Error404 } from 'shared/components/page/Errors' import { FamilyVariantIndividuals } from 'shared/components/panel/variants/VariantIndividuals' import { GENOME_VERSION_FIELD, GENOME_VERSION_37, GENOME_VERSION_38 } from 'shared/utils/constants' +import GeneVariantLookup from './components/GeneVariantLookup' import { sendVlmContactEmail } from './reducers' import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail } from './selectors' @@ -205,6 +208,8 @@ const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled +
+
@@ -273,4 +278,18 @@ WrappedVariantLookup.propTypes = { vlmEnabled: PropTypes.bool, } -export default connect(mapGlobalStateToProps, mapDispatchToProps)(WrappedVariantLookup) +const ConnectedVariantLookup = connect(mapGlobalStateToProps, mapDispatchToProps)(WrappedVariantLookup) + +const VariantLookupRoute = ({ match }) => ( + + + + + +) + +VariantLookupRoute.propTypes = { + match: PropTypes.object, +} + +export default VariantLookupRoute diff --git a/ui/pages/SummaryData/components/GeneVariantLookup.jsx b/ui/pages/Search/components/GeneVariantLookup.jsx similarity index 92% rename from ui/pages/SummaryData/components/GeneVariantLookup.jsx rename to ui/pages/Search/components/GeneVariantLookup.jsx index 19b316a0ec..c381fd34eb 100644 --- a/ui/pages/SummaryData/components/GeneVariantLookup.jsx +++ b/ui/pages/Search/components/GeneVariantLookup.jsx @@ -21,8 +21,8 @@ import { VEP_GROUP_EXTENDED_SPLICE_SITE, } from 'shared/utils/constants' import { snakecaseToTitlecase } from 'shared/utils/stringUtils' -import { loadGeneVariantLookup } from '../reducers' -import { getGeneVariantLookupLoading, getGeneVariantLookupResults } from '../selectors' +// import { loadGeneVariantLookup } from '../reducers' +// import { getGeneVariantLookupLoading, getGeneVariantLookupResults } from '../selectors' const validateAnnotations = (value, { annotations }) => ( value || Object.values(annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' @@ -118,12 +118,12 @@ GeneVariantLookup.propTypes = { } const mapStateToProps = state => ({ - loading: getGeneVariantLookupLoading(state), - results: getGeneVariantLookupResults(state), + // loading: getGeneVariantLookupLoading(state), + // results: getGeneVariantLookupResults(state), }) const mapDispatchToProps = { - onSubmit: loadGeneVariantLookup, + // onSubmit: loadGeneVariantLookup, } export default connect(mapStateToProps, mapDispatchToProps)(GeneVariantLookup) diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index 9a06d787ea..793161ccb1 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -27,6 +27,7 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { {Object.keys(user).length ? [ , , + , , (user.isAnalyst || user.isPm) ? : null, (user.isDataManager || user.isPm) ? : null, diff --git a/ui/shared/components/page/PageHeader.jsx b/ui/shared/components/page/PageHeader.jsx index 269b2ebd14..5cb7d96fd5 100644 --- a/ui/shared/components/page/PageHeader.jsx +++ b/ui/shared/components/page/PageHeader.jsx @@ -33,6 +33,7 @@ export const GenePageHeader = connect(mapGeneStateToProps)(BaseGenePageHeader) const NO_HEADER_PAGES = [ '/dashboard', '/create_project_from_workspace', '/workspace', '/users', '/login', '/accept_policies', ...PUBLIC_PAGES, + '/variant_lookup', ] const NO_HEADER_PAGE_TITLES = { From 7b59e33b3607b045961c39ad86d384f32dba43e7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 17:31:46 -0400 Subject: [PATCH 169/252] handle both pages --- ui/app.jsx | 2 +- ui/pages/Search/VariantLookup.jsx | 226 ++++++++++++------ .../Search/components/GeneVariantLookup.jsx | 129 ---------- 3 files changed, 157 insertions(+), 200 deletions(-) delete mode 100644 ui/pages/Search/components/GeneVariantLookup.jsx diff --git a/ui/app.jsx b/ui/app.jsx index de187a0b60..533fd4c5b7 100644 --- a/ui/app.jsx +++ b/ui/app.jsx @@ -51,7 +51,7 @@ ReactDOM.render( - + diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 590d98c39a..88d51af8f2 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -1,28 +1,41 @@ import React from 'react' import { connect } from 'react-redux' import PropTypes from 'prop-types' -import { Route, Switch } from 'react-router-dom' import { Grid, Header, Label, Table } from 'semantic-ui-react' import { RECEIVE_DATA } from 'redux/utils/reducerUtils' +import { navigateSavedHashedSearch } from 'redux/rootReducer' import { getVlmEnabled } from 'redux/selectors' import { QueryParamsEditor } from 'shared/components/QueryParamEditor' import StateDataLoader from 'shared/components/StateDataLoader' import SendEmailButton from 'shared/components/buttons/SendEmailButton' import FormWrapper from 'shared/components/form/FormWrapper' -import { helpLabel } from 'shared/components/form/FormHelpers' -import { BaseSemanticInput, InlineToggle } from 'shared/components/form/Inputs' +import { helpLabel, validators } from 'shared/components/form/FormHelpers' +import { BaseSemanticInput, InlineToggle, AlignedCheckboxGroup } from 'shared/components/form/Inputs' +import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' import FamilyReads from 'shared/components/panel/family/FamilyReads' import FamilyVariantTags from 'shared/components/panel/variants/FamilyVariantTags' import Variants, { Variant, StyledVariantRow } from 'shared/components/panel/variants/Variants' -import { Error404 } from 'shared/components/page/Errors' import { FamilyVariantIndividuals } from 'shared/components/panel/variants/VariantIndividuals' -import { GENOME_VERSION_FIELD, GENOME_VERSION_37, GENOME_VERSION_38 } from 'shared/utils/constants' -import GeneVariantLookup from './components/GeneVariantLookup' +import { + GENOME_VERSION_FIELD, + GENOME_VERSION_37, + GENOME_VERSION_38, + GENE_SEARCH_FREQUENCIES, + GROUPED_VEP_CONSEQUENCES, + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +} from 'shared/utils/constants' +import { snakecaseToTitlecase } from 'shared/utils/stringUtils' import { sendVlmContactEmail } from './reducers' -import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail } from './selectors' +import { getVlmDefaultContactEmails, getVlmFamiliesByContactEmail, getSearchedVariantsIsLoading } from './selectors' -const FIELDS = [ +const LOOKUP_FIELDS = [ { name: 'variantId', label: helpLabel('Variant ID', ( @@ -59,6 +72,44 @@ const FIELDS = [ }, ] +const validateAnnotations = (value, { annotations }) => ( + value || Object.values(annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' +) + +const CONSEQUENCE_FILEDS = [ + VEP_GROUP_NONSENSE, + VEP_GROUP_ESSENTIAL_SPLICE_SITE, + VEP_GROUP_FRAMESHIFT, + VEP_GROUP_MISSENSE, + VEP_GROUP_INFRAME, + VEP_GROUP_SYNONYMOUS, + VEP_GROUP_EXTENDED_SPLICE_SITE, +].map((group, i) => ({ + name: `annotations.${group}`, + component: AlignedCheckboxGroup, + groupLabel: snakecaseToTitlecase(group), + options: GROUPED_VEP_CONSEQUENCES[group], + format: value => value || [], + inline: true, + validate: i === 0 ? validateAnnotations : undefined, +})) + +const GENE_LOOKUP_FIELDS = [ + { validate: validators.required, ...GENOME_VERSION_FIELD }, + { + name: 'geneId', + label: 'Gene', + control: AwesomeBarFormInput, + categories: ['genes'], + fluid: true, + placeholder: 'Search for gene', + validate: validators.required, + }, + ...CONSEQUENCE_FILEDS, +] + +const INITIAL_GENE_LOOKUP_VALUES = { freqs: GENE_SEARCH_FREQUENCIES } + const VlmDisplay = ({ vlmMatches }) => ( {Object.entries(vlmMatches || {}).map( @@ -181,12 +232,12 @@ BaseLookupVariant.propTypes = { vlmDefaultContactEmails: PropTypes.object, } -const mapStateToProps = (state, ownProps) => ({ +const mapVariantStateToProps = (state, ownProps) => ({ familiesByContactEmail: getVlmFamiliesByContactEmail(state, ownProps), vlmDefaultContactEmails: getVlmDefaultContactEmails(state, ownProps), }) -const LookupVariant = connect(mapStateToProps)(BaseLookupVariant) +const LookupVariant = connect(mapVariantStateToProps)(BaseLookupVariant) const VariantDisplay = ({ variants }) => ( (variants || [])[0]?.lookupFamilyGuids ? : @@ -196,48 +247,32 @@ VariantDisplay.propTypes = { variants: PropTypes.arrayOf(PropTypes.object), } -const onSubmit = updateQueryParams => (data) => { - updateQueryParams(data) - return Promise.resolve() -} - const passThroughResponse = response => response -const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled }) => ( - - - - -
-
- +const BaseVariantLookupResults = ({ queryParams, receiveData, vlmEnabled }) => ([ + ...(vlmEnabled ? [( + + +
+ + + - - {vlmEnabled && ( - - -
- - - - - - )} - {vlmEnabled && ( - - -
- - - - )} - + ), ( + + +
+ + + + )] : []), ( + - -) + ), +]) -VariantLookup.propTypes = { +BaseVariantLookupResults.propTypes = { receiveData: PropTypes.func, - updateQueryParams: PropTypes.func, queryParams: PropTypes.object, vlmEnabled: PropTypes.bool, } -const mapGlobalStateToProps = state => ({ +const mapResultsStateToProps = state => ({ vlmEnabled: getVlmEnabled(state), }) -const mapDispatchToProps = dispatch => ({ +const mapResultsDispatchToProps = dispatch => ({ receiveData: (updatesById) => { dispatch({ type: RECEIVE_DATA, updatesById }) return updatesById }, }) -const WrappedVariantLookup = ({ vlmEnabled, ...props }) => ( - - - -) +const VariantLookupResults = connect(mapResultsStateToProps, mapResultsDispatchToProps)(BaseVariantLookupResults) -WrappedVariantLookup.propTypes = { - vlmEnabled: PropTypes.bool, +const LOOKUP_HEADER = { content: 'Lookup Variant' } +const GENE_LOOKUP_HEADER = { + content: 'Lookup Variants in Gene', + subheader: ( + + Lookup up all rare variants is seqr in a given gene, regardless of whether or not they are in your + projects. +
+ Variants are only returned if they have a gnomAD Allele Frequency < 3% + and have a seqr global Allele Count < 3000. + + ), } -const ConnectedVariantLookup = connect(mapGlobalStateToProps, mapDispatchToProps)(WrappedVariantLookup) - -const VariantLookupRoute = ({ match }) => ( - - - - - +const VariantLookupRoute = ({ match, formProps, onSubmit, ...props }) => ( + + + +
+
+ + + + + + + + + + {match.params.gene ? JSON.stringify(props) : } + ) VariantLookupRoute.propTypes = { match: PropTypes.object, + formProps: PropTypes.object, + onSubmit: PropTypes.func, } -export default VariantLookupRoute +const mapStateToProps = (state, ownProps) => ({ + formProps: ownProps.match.params.gene ? { + showErrorPanel: true, + loading: getSearchedVariantsIsLoading(state), + initialValues: INITIAL_GENE_LOOKUP_VALUES, + fields: GENE_LOOKUP_FIELDS, + } : { + fields: LOOKUP_FIELDS, + initialValues: ownProps.queryParams, + }, +}) + +const onQueryParamSubmit = (updateQueryParams, data) => { + updateQueryParams(data) + return Promise.resolve() +} + +const mapDispatchToProps = (dispatch, ownProps) => ({ + onSubmit: data => (ownProps.match.params.gene ? + dispatch(navigateSavedHashedSearch(data)) : + onQueryParamSubmit(ownProps.updateQueryParams, data) + ), +}) + +const ConnectedVariantLookup = connect(mapStateToProps, mapDispatchToProps)(VariantLookupRoute) + +export default props => ( + + + +) diff --git a/ui/pages/Search/components/GeneVariantLookup.jsx b/ui/pages/Search/components/GeneVariantLookup.jsx deleted file mode 100644 index c381fd34eb..0000000000 --- a/ui/pages/Search/components/GeneVariantLookup.jsx +++ /dev/null @@ -1,129 +0,0 @@ -import React from 'react' -import { connect } from 'react-redux' -import PropTypes from 'prop-types' -import { Grid, Header } from 'semantic-ui-react' - -import { validators } from 'shared/components/form/FormHelpers' -import FormWrapper from 'shared/components/form/FormWrapper' -import { AlignedCheckboxGroup } from 'shared/components/form/Inputs' -import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' -import { Variant } from 'shared/components/panel/variants/Variants' -import { - GENE_SEARCH_FREQUENCIES, - GENOME_VERSION_FIELD, - GROUPED_VEP_CONSEQUENCES, - VEP_GROUP_NONSENSE, - VEP_GROUP_ESSENTIAL_SPLICE_SITE, - VEP_GROUP_FRAMESHIFT, - VEP_GROUP_MISSENSE, - VEP_GROUP_INFRAME, - VEP_GROUP_SYNONYMOUS, - VEP_GROUP_EXTENDED_SPLICE_SITE, -} from 'shared/utils/constants' -import { snakecaseToTitlecase } from 'shared/utils/stringUtils' -// import { loadGeneVariantLookup } from '../reducers' -// import { getGeneVariantLookupLoading, getGeneVariantLookupResults } from '../selectors' - -const validateAnnotations = (value, { annotations }) => ( - value || Object.values(annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' -) - -const CONSEQUENCE_FILEDS = [ - VEP_GROUP_NONSENSE, - VEP_GROUP_ESSENTIAL_SPLICE_SITE, - VEP_GROUP_FRAMESHIFT, - VEP_GROUP_MISSENSE, - VEP_GROUP_INFRAME, - VEP_GROUP_SYNONYMOUS, - VEP_GROUP_EXTENDED_SPLICE_SITE, -].map((group, i) => ({ - name: `annotations.${group}`, - component: AlignedCheckboxGroup, - groupLabel: snakecaseToTitlecase(group), - options: GROUPED_VEP_CONSEQUENCES[group], - format: value => value || [], - inline: true, - validate: i === 0 ? validateAnnotations : undefined, -})) - -const FIELDS = [ - { validate: validators.required, ...GENOME_VERSION_FIELD }, - { - name: 'geneId', - label: 'Gene', - control: AwesomeBarFormInput, - categories: ['genes'], - fluid: true, - placeholder: 'Search for gene', - validate: validators.required, - }, - ...CONSEQUENCE_FILEDS, -] - -const INITIAL_VALUES = { freqs: GENE_SEARCH_FREQUENCIES } - -const GeneVariantLookup = ({ loading, results, onSubmit }) => ( - - - -
- Lookup up all rare variants is seqr in a given gene, regardless of whether or not they are in your - projects. -
- Variants are only returned if they have a gnomAD Allele Frequency < 3% - and have a seqr global Allele Count < 3000. - - )} - /> - - - - - - - - - - {!loading && results && ( - - - Found   - {results.length} -   variants - - - )} - {!loading && results?.map(variant => ( - - ))} - -) - -GeneVariantLookup.propTypes = { - loading: PropTypes.bool, - results: PropTypes.arrayOf(PropTypes.object), - onSubmit: PropTypes.func, -} - -const mapStateToProps = state => ({ - // loading: getGeneVariantLookupLoading(state), - // results: getGeneVariantLookupResults(state), -}) - -const mapDispatchToProps = { - // onSubmit: loadGeneVariantLookup, -} - -export default connect(mapStateToProps, mapDispatchToProps)(GeneVariantLookup) From 5c7db3eaf875641ec7db98eec8f633e5aae24ac7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 17:42:01 -0400 Subject: [PATCH 170/252] format gene search data properly --- ui/pages/Search/VariantLookup.jsx | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 88d51af8f2..1763d26779 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -72,8 +72,8 @@ const LOOKUP_FIELDS = [ }, ] -const validateAnnotations = (value, { annotations }) => ( - value || Object.values(annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' +const validateAnnotations = (value, { search }) => ( + value || Object.values(search.annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' ) const CONSEQUENCE_FILEDS = [ @@ -85,7 +85,7 @@ const CONSEQUENCE_FILEDS = [ VEP_GROUP_SYNONYMOUS, VEP_GROUP_EXTENDED_SPLICE_SITE, ].map((group, i) => ({ - name: `annotations.${group}`, + name: `search.annotations.${group}`, component: AlignedCheckboxGroup, groupLabel: snakecaseToTitlecase(group), options: GROUPED_VEP_CONSEQUENCES[group], @@ -95,9 +95,9 @@ const CONSEQUENCE_FILEDS = [ })) const GENE_LOOKUP_FIELDS = [ - { validate: validators.required, ...GENOME_VERSION_FIELD }, + { validate: validators.required, ...GENOME_VERSION_FIELD, name: 'allGenomeProjectFamilies' }, { - name: 'geneId', + name: 'search.locus.rawItems', label: 'Gene', control: AwesomeBarFormInput, categories: ['genes'], @@ -108,7 +108,10 @@ const GENE_LOOKUP_FIELDS = [ ...CONSEQUENCE_FILEDS, ] -const INITIAL_GENE_LOOKUP_VALUES = { freqs: GENE_SEARCH_FREQUENCIES } +const INITIAL_GENE_LOOKUP_VALUES = { + includeNoAccessProjects: true, + search: { freqs: GENE_SEARCH_FREQUENCIES }, +} const VlmDisplay = ({ vlmMatches }) => (
From 476f412b8b17831bf1d952b33e537d7d91f77598 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 17:51:46 -0400 Subject: [PATCH 171/252] better helper message --- ui/pages/Search/VariantLookup.jsx | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 1763d26779..372421bdd0 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -1,7 +1,7 @@ import React from 'react' import { connect } from 'react-redux' import PropTypes from 'prop-types' -import { Grid, Header, Label, Table } from 'semantic-ui-react' +import { Grid, Header, Icon, Label, Popup, Table } from 'semantic-ui-react' import { RECEIVE_DATA } from 'redux/utils/reducerUtils' import { navigateSavedHashedSearch } from 'redux/rootReducer' @@ -317,6 +317,22 @@ const GENE_LOOKUP_HEADER = {
Variants are only returned if they have a gnomAD Allele Frequency < 3% and have a seqr global Allele Count < 3000. +
+ For more granular control over the returned variants, we recommend using the main Search page instead of this + lookup   + } + content={ + 'On the Search page, select your desired genome build in the "Include All Projects" section, select the ' + + '"Include External Projects" checkbox next to it, and enter your gene of interest in the "Location" search ' + + 'box along with your other desired search filters.' + } + hoverable + /> +
+ +
+ ), } From 77adc5535d52eeec911f555f18862fedd2ebc75d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 17:59:52 -0400 Subject: [PATCH 172/252] clean up --- ui/pages/Search/VariantLookup.jsx | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 372421bdd0..66c0d62644 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -250,6 +250,11 @@ VariantDisplay.propTypes = { variants: PropTypes.arrayOf(PropTypes.object), } +const onQueryParamSubmit = (updateQueryParams, data) => { + updateQueryParams(data) + return Promise.resolve() +} + const passThroughResponse = response => response const BaseVariantLookupResults = ({ queryParams, receiveData, vlmEnabled }) => ([ @@ -337,7 +342,7 @@ const GENE_LOOKUP_HEADER = { ), } -const VariantLookupRoute = ({ match, formProps, onSubmit, ...props }) => ( +const BaseVariantLookup = ({ match, formProps, onSubmit, ...props }) => ( @@ -357,11 +362,11 @@ const VariantLookupRoute = ({ match, formProps, onSubmit, ...props }) => ( - {match.params.gene ? JSON.stringify(props) : } + {!match.params.gene && } ) -VariantLookupRoute.propTypes = { +BaseVariantLookup.propTypes = { match: PropTypes.object, formProps: PropTypes.object, onSubmit: PropTypes.func, @@ -379,11 +384,6 @@ const mapStateToProps = (state, ownProps) => ({ }, }) -const onQueryParamSubmit = (updateQueryParams, data) => { - updateQueryParams(data) - return Promise.resolve() -} - const mapDispatchToProps = (dispatch, ownProps) => ({ onSubmit: data => (ownProps.match.params.gene ? dispatch(navigateSavedHashedSearch(data)) : @@ -391,10 +391,10 @@ const mapDispatchToProps = (dispatch, ownProps) => ({ ), }) -const ConnectedVariantLookup = connect(mapStateToProps, mapDispatchToProps)(VariantLookupRoute) +const VariantLookup = connect(mapStateToProps, mapDispatchToProps)(BaseVariantLookup) export default props => ( - + ) From 13675da47f2b899ccc5158c78a5d304508d05d39 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 18:07:17 -0400 Subject: [PATCH 173/252] unnest send email route --- matchmaker/views/matchmaker_api.py | 26 ++++++++++++- matchmaker/views/matchmaker_api_tests.py | 47 ++++++++++++++++++++++- seqr/urls.py | 6 ++- seqr/views/apis/summary_data_api.py | 29 ++------------ seqr/views/apis/summary_data_api_tests.py | 46 +--------------------- ui/pages/Search/reducers.js | 3 +- 6 files changed, 79 insertions(+), 78 deletions(-) diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 40056b7e64..a1791115fd 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -11,7 +11,7 @@ get_submission_json_for_external_match, parse_mme_features, get_submission_gene_variants, get_mme_matches, \ get_gene_ids_for_feature, validate_patient_data, get_hpo_terms_by_id, MME_DISCLAIMER from seqr.models import Individual, SavedVariant -from seqr.utils.communication_utils import safe_post_to_slack +from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_chrom_pos @@ -24,7 +24,7 @@ has_project_permissions, login_and_policies_required, get_project_and_check_permissions from settings import BASE_URL, MME_ACCEPT_HEADER, MME_NODES, MME_DEFAULT_CONTACT_EMAIL, \ - MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL + MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL logger = SeqrLogger(__name__) @@ -559,3 +559,25 @@ def _generate_notification_for_seqr_match(submission, results): from_email=MME_DEFAULT_CONTACT_EMAIL, ) email_message.send() + + +@login_and_policies_required +def send_vlm_email(request): + request_json = json.loads(request.body) + email_message = EmailMessage( + subject=request_json['subject'], + body=request_json['body'], + bcc=[s.strip() for s in request_json['to'].split(',')], + cc=[request.user.email], + reply_to=[request.user.email], + to=[VLM_SEND_EMAIL], + from_email=VLM_SEND_EMAIL, + ) + set_email_message_stream(email_message, 'vlm') + + try: + email_message.send() + except Exception as e: + logger.error(f'VLM Email Error: {e}', request.user, detail=request_json) + + return create_json_response({'success': True}) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index 845e85411e..0bbe9e53d7 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -10,7 +10,7 @@ from matchmaker.matchmaker_utils import MME_DISCLAIMER from matchmaker.views.matchmaker_api import get_individual_mme_matches, search_individual_mme_matches, \ update_mme_submission, delete_mme_submission, update_mme_result_status, send_mme_contact_email, \ - get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, \ + get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, send_vlm_email, \ update_mme_contact_note, update_mme_project_contact from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_FIELDS @@ -871,3 +871,48 @@ def test_update_mme_project_contact(self): {'name': 'Baylor UDN Clinical Site', 'email': 'UDNCC@hms.harvard.edu'}, {'name': '', 'email': 'matchmaker@phenomecentral.org'}, ]) + + + @mock.patch('matchmaker.views.matchmaker_api.EmailMessage') + def test_send_vlm_email(self, mock_email): + url = reverse(send_vlm_email) + self.check_require_login(url) + + self.reset_logs() + body = { + 'to': 'test@test.com , other_test@gmail.com', + 'body': 'some email content', + 'subject': 'some email subject' + } + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self._assert_expected_vlm_email(response, mock_email) + + self.reset_logs() + mock_email.return_value.send.side_effect = Exception('Send failed') + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self._assert_expected_vlm_email(response, mock_email, additional_logs=[ + ('VLM Email Error: Send failed', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + 'detail': body, + }), + ]) + + def _assert_expected_vlm_email(self, response, mock_email, additional_logs=None): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'success': True}) + + mock_email.assert_called_with( + subject='some email subject', + body='some email content', + bcc=['test@test.com', 'other_test@gmail.com'], + cc=['test_user_no_access@test.com'], + reply_to=['test_user_no_access@test.com'], + to=['vlm-noreply@broadinstitute.org'], + from_email='vlm-noreply@broadinstitute.org') + self.assertDictEqual(mock_email.return_value.esp_extra, {'MessageStream': 'vlm'}) + mock_email.return_value.send.assert_called() + + self.assert_json_logs(self.no_access_user, (additional_logs or []) + [ + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}) + ]) diff --git a/seqr/urls.py b/seqr/urls.py index 5e00389df9..d45e0cf0bb 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -90,6 +90,7 @@ update_mme_result_status, \ update_mme_contact_note, \ update_mme_project_contact, \ + send_vlm_email, \ send_mme_contact_email from seqr.views.apis.variant_search_api import \ @@ -130,7 +131,7 @@ gregor_export, \ seqr_stats from seqr.views.apis.summary_data_api import success_story, saved_variants_page, mme_details, hpo_summary_data, \ - bulk_update_family_external_analysis, individual_metadata, send_vlm_email + bulk_update_family_external_analysis, individual_metadata from seqr.views.apis.superuser_api import get_all_users from seqr.views.apis.awesomebar_api import awesomebar_autocomplete_handler @@ -346,7 +347,8 @@ 'summary_data/matchmaker': mme_details, 'summary_data/update_external_analysis': bulk_update_family_external_analysis, 'summary_data/individual_metadata/(?P[^/]+)': individual_metadata, - 'summary_data/send_vlm_email': send_vlm_email, + + 'send_vlm_email': send_vlm_email, 'create_project_from_workspace/(?P[^/]+)/(?P[^/]+)/grant_access': grant_workspace_access, 'create_project_from_workspace/(?P[^/]+)/(?P[^/]+)/validate_vcf': validate_anvil_vcf, diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 48ef125034..598d62c48b 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -1,7 +1,6 @@ from collections import defaultdict from datetime import datetime from django.core.exceptions import PermissionDenied -from django.core.mail.message import EmailMessage from django.contrib.auth.models import User from django.db.models import CharField, F, Value from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf @@ -11,10 +10,10 @@ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology -from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy, Sample +from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file -from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream +from seqr.utils.communication_utils import safe_post_to_slack from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException from seqr.views.utils.json_utils import create_json_response @@ -26,7 +25,7 @@ login_and_policies_required, get_project_and_check_permissions, get_internal_projects from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, DISCOVERY_ROW_TYPE from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, DISCOVERY_CATEGORY -from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL +from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL logger = SeqrLogger(__name__) @@ -335,25 +334,3 @@ def _get_airtable_collaborator_names(user, collaborator_ids): collaborator_id: collaborator_map.get(collaborator_id, {}).get('CollaboratorID') for collaborator_id in collaborator_ids } - - -@login_and_policies_required -def send_vlm_email(request): - request_json = json.loads(request.body) - email_message = EmailMessage( - subject=request_json['subject'], - body=request_json['body'], - bcc=[s.strip() for s in request_json['to'].split(',')], - cc=[request.user.email], - reply_to=[request.user.email], - to=[VLM_SEND_EMAIL], - from_email=VLM_SEND_EMAIL, - ) - set_email_message_stream(email_message, 'vlm') - - try: - email_message.send() - except Exception as e: - logger.error(f'VLM Email Error: {e}', request.user, detail=request_json) - - return create_json_response({'success': True}) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 32bc54acc2..067caeb781 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -6,7 +6,7 @@ import responses from seqr.views.apis.summary_data_api import mme_details, success_story, saved_variants_page, hpo_summary_data, \ - bulk_update_family_external_analysis, individual_metadata, send_vlm_email + bulk_update_family_external_analysis, individual_metadata from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest, PARSED_VARIANTS, SAVED_VARIANT_FIELDS from seqr.models import FamilyAnalysedBy, SavedVariant, VariantTag from settings import AIRTABLE_URL @@ -797,50 +797,6 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assert_expected_airtable_call( -1, "OR(RECORD_ID()='reca4hcBnbA2cnZf9')", ['CollaboratorID']) - @mock.patch('seqr.views.apis.summary_data_api.EmailMessage') - def test_send_vlm_email(self, mock_email): - url = reverse(send_vlm_email) - self.check_require_login(url) - - self.reset_logs() - body = { - 'to': 'test@test.com , other_test@gmail.com', - 'body': 'some email content', - 'subject': 'some email subject' - } - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_vlm_email(response, mock_email) - - self.reset_logs() - mock_email.return_value.send.side_effect = Exception('Send failed') - response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_vlm_email(response, mock_email, additional_logs=[ - ('VLM Email Error: Send failed', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - 'detail': body, - }), - ]) - - def _assert_expected_vlm_email(self, response, mock_email, additional_logs=None): - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), {'success': True}) - - mock_email.assert_called_with( - subject='some email subject', - body='some email content', - bcc=['test@test.com', 'other_test@gmail.com'], - cc=['test_user_no_access@test.com'], - reply_to=['test_user_no_access@test.com'], - to=['vlm-noreply@broadinstitute.org'], - from_email='vlm-noreply@broadinstitute.org') - self.assertDictEqual(mock_email.return_value.esp_extra, {'MessageStream': 'vlm'}) - mock_email.return_value.send.assert_called() - - self.assert_json_logs(self.no_access_user, (additional_logs or []) + [ - (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}) - ]) - # Tests for AnVIL access disabled class LocalSummaryDataAPITest(AuthenticationTestCase, SummaryDataAPITest): diff --git a/ui/pages/Search/reducers.js b/ui/pages/Search/reducers.js index 8c8197d9e6..67cbd5ea9a 100644 --- a/ui/pages/Search/reducers.js +++ b/ui/pages/Search/reducers.js @@ -219,9 +219,8 @@ export const loadGeneBreakdown = searchHash => (dispatch, getState) => { } } -// TODO update url path export const sendVlmContactEmail = values => () => new HttpRequestHelper( - '/api/summary_data/send_vlm_email', + '/api/send_vlm_email', ).post(values) // reducers From b677ec55e652a38c5a7387663ee4e7704cd372f6 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 16 Mar 2026 18:23:54 -0400 Subject: [PATCH 174/252] fix duplicate key Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ui/shared/components/page/Header.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index 793161ccb1..f56839d732 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -27,7 +27,7 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { {Object.keys(user).length ? [ , , - , + , , (user.isAnalyst || user.isPm) ? : null, (user.isDataManager || user.isPm) ? : null, From 77ec3380d80f23cb806da629c848a545ce455d26 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 18:24:08 -0400 Subject: [PATCH 175/252] pr feedback --- ui/pages/Search/VariantLookup.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 66c0d62644..ccbb4f6982 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -76,7 +76,7 @@ const validateAnnotations = (value, { search }) => ( value || Object.values(search.annotations || {}).some(val => val.length) ? undefined : 'At least one consequence filter is required' ) -const CONSEQUENCE_FILEDS = [ +const CONSEQUENCE_FIELDS = [ VEP_GROUP_NONSENSE, VEP_GROUP_ESSENTIAL_SPLICE_SITE, VEP_GROUP_FRAMESHIFT, @@ -105,7 +105,7 @@ const GENE_LOOKUP_FIELDS = [ placeholder: 'Search for gene', validate: validators.required, }, - ...CONSEQUENCE_FILEDS, + ...CONSEQUENCE_FIELDS, ] const INITIAL_GENE_LOOKUP_VALUES = { @@ -317,7 +317,7 @@ const GENE_LOOKUP_HEADER = { content: 'Lookup Variants in Gene', subheader: ( - Lookup up all rare variants is seqr in a given gene, regardless of whether or not they are in your + Lookup up all rare variants in seqr in a given gene, regardless of whether or not they are in your projects.
Variants are only returned if they have a gnomAD Allele Frequency < 3% From 4e013c6e1a60830a1f22baa99d5efd0a85b1b005 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 19:41:18 -0400 Subject: [PATCH 176/252] inline sample parsign and validation --- clickhouse_search/search_tests.py | 7 ++- seqr/utils/search/utils.py | 101 +++++++++++++----------------- 2 files changed, 49 insertions(+), 59 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 626e792698..baf8aee7be 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -218,6 +218,8 @@ def _assert_expected_export_data(self, search_hash, export_data): def _assert_expected_search_error(self, error, **kwargs): response, search_hash, _ = self._execute_search(**kwargs) + if response.status_code != 400: + import pdb; pdb.set_trace() self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': error}) return search_hash @@ -897,9 +899,10 @@ def test_invalid_search(self): self._assert_expected_search_error('Annotations must be specified to search for compound heterozygous variants', inheritance_mode='recessive') + annotations = {'frameshift': ['frameshift_variant']} self._assert_expected_search_error( 'Location must be specified to search for compound heterozygous variants across many families', - annotations={'frameshift': ['frameshift_variant']}, inheritance_mode='recessive', + annotations=annotations, inheritance_mode='recessive', ) self._assert_expected_search_error( @@ -912,6 +915,7 @@ def test_invalid_search(self): self._assert_expected_search_error( 'Inheritance based search is disabled in families with no data loaded for affected individuals', inheritance_mode='recessive', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000005_5']}], + annotations=annotations, ) no_sv_project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000003_3']}] @@ -928,6 +932,7 @@ def test_invalid_search(self): self._assert_expected_search_error( 'Inheritance based search is disabled in families with no data loaded for affected individuals', inheritance_mode='recessive', inheritance_filter={'affected': {'I000007_na20870': 'N'}}, project_families=no_sv_project_families, + annotations=annotations, ) self._assert_expected_search_error('Inheritance must be specified if custom affected status is set', inheritance_filter={'affected': {'I000007_na20870': 'N'}}) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index be4d3034f2..1870c9b650 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -52,22 +52,6 @@ def get_search_samples(projects, active_only=True): return _get_filtered_search_samples({'individual__family__project__in': projects}, active_only=active_only) -def _get_families_search_data(families, dataset_type, include_no_access_projects=False, sample_filter=None): - samples = _get_filtered_search_samples(sample_filter or {'individual__family__in': families}) - if len(samples) < 1: - if include_no_access_projects: - return samples - raise InvalidSearchException('No search data found for families {}'.format( - ', '.join([f.family_id for f in families]))) - - if dataset_type: - samples = samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dataset_type]) - if not samples: - raise InvalidSearchException(f'Unable to search against dataset type "{dataset_type}"') - - return samples - - def _get_search_genome_version(search_model): projects = Project.objects.filter(family__in=search_model.families.all()).values_list('genome_version', 'name').distinct() project_versions = defaultdict(set) @@ -89,7 +73,9 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_ids_dataset_type([parsed_variant_id]) - samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) + samples = _get_filtered_search_samples({'individual__family_id': family.id}) + if len(samples) < 1: + raise InvalidSearchException(f'No search data found for families {family.family_id}') variant = get_clickhouse_variant_by_id( variant_id, parsed_variant_id, samples, family.project.genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], ) @@ -176,8 +162,37 @@ def _query_variants(search_model, user, sort=None, **kwargs): parsed_search = _parse_search(search, genome_version, user) search_dataset_type = _parse_dataset_type(parsed_search, genome_version) - samples = _get_search_samples(parsed_search, families, search_dataset_type) - _validate_search(parsed_search, samples, previous_search_results) + + _validate_search(parsed_search, families) + + samples = _get_filtered_search_samples({'individual__family__in': families}) + if len(samples) < 1: + if parsed_search.get('no_access_project_genome_version'): + return samples + raise InvalidSearchException('No search data found for families {}'.format( + ', '.join([f.family_id for f in families]))) + if search_dataset_type: + samples = samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[search_dataset_type]) + if not samples: + raise InvalidSearchException(f'Unable to search against dataset type "{search_dataset_type}"') + if parsed_search.get('inheritance_mode') == X_LINKED_RECESSIVE: + samples = samples.exclude(dataset_type=Sample.DATASET_TYPE_MITO_CALLS) + if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): + samples = samples.select_related('individual') + if samples: + skipped_samples = _filter_inheritance_family_samples(samples, parsed_search['inheritance_filter']) + if skipped_samples: + samples = samples.exclude(id__in=[s.id for s in skipped_samples]) + + if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and parsed_search['secondary_dataset_type']: + invalid_type = next(( + dt for dt in [parsed_search['dataset_type'], parsed_search['secondary_dataset_type']] + if dt and dt != ALL_DATA_TYPES and samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dt]).count() < 1 + ), None) + if invalid_type: + raise InvalidSearchException( + f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' + ) get_clickhouse_variants(samples, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) @@ -225,6 +240,9 @@ def _parse_search(search, genome_version, user): if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} + if parsed_search.get('inheritance'): + _parse_inheritance(parsed_search) + return parsed_search def _parse_dataset_type(parsed_search, genome_version): @@ -239,13 +257,6 @@ def _parse_dataset_type(parsed_search, genome_version): return search_dataset_type -def _get_search_samples(parsed_search, families, search_dataset_type): - samples = _get_families_search_data(families, dataset_type=search_dataset_type, include_no_access_projects=bool(parsed_search.get('no_access_project_genome_version'))) - if parsed_search.get('inheritance'): - samples = _parse_inheritance(parsed_search, samples) - return samples - - def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) cached_results, _ = _query_variants(previous_search_model, user) @@ -374,7 +385,7 @@ def _annotation_dataset_type(annotations, chroms, pathogenicity=None, exclude_sv return ALL_DATA_TYPES -def _parse_inheritance(search, samples): +def _parse_inheritance(search): inheritance = search.pop('inheritance') inheritance_mode = inheritance.get('mode') inheritance_filter = inheritance.get('filter') or {} @@ -383,55 +394,29 @@ def _parse_inheritance(search, samples): inheritance_mode = None search.update({'inheritance_mode': inheritance_mode, 'inheritance_filter': inheritance_filter}) - if not (inheritance_mode or inheritance_filter): - return samples if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: raise InvalidSearchException('Inheritance must be specified if custom affected status is set') - if inheritance_mode == X_LINKED_RECESSIVE: - samples = samples.exclude(dataset_type=Sample.DATASET_TYPE_MITO_CALLS) - - samples = samples.select_related('individual') - if samples: - skipped_samples = _filter_inheritance_family_samples(samples, inheritance_filter) - if skipped_samples: - search['skipped_samples'] = skipped_samples - samples = samples.exclude(id__in=[s.id for s in skipped_samples]) - return samples - - -def _validate_search(search, samples, previous_search_results): - has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} and not previous_search_results.get('grouped_results') +def _validate_search(search, families): + has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) if has_comp_het_search: if not search.get('annotations'): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') - family_ids = {s.individual.family_id for s in samples.select_related('individual')} - if not has_location_filter and len(family_ids) > MAX_NO_LOCATION_COMP_HET_FAMILIES: + if not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: raise InvalidSearchException( 'Location must be specified to search for compound heterozygous variants across many families') - if search['secondary_dataset_type']: - invalid_type = next(( - dt for dt in [search['dataset_type'], search['secondary_dataset_type']] - if dt and dt != ALL_DATA_TYPES and samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dt]).count() < 1 - ), None) - if invalid_type: - raise InvalidSearchException( - f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' - ) - if search.get('no_access_project_genome_version'): raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - variant_samples = samples.filter(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - if not has_location_filter and variant_samples.values('individual__family__project_id').distinct().count() > 1: + if not has_location_filter and families.values('project_id').distinct().count() > 1: raise InvalidSearchException('Location must be specified to search across multiple projects') seqr_ac_filter = search.get('freqs', {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) - if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and variant_samples.values('individual__family_id').distinct().count() > 1: + if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: raise InvalidSearchException( f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' ) From f0c22a51fbb5a9ef248e6359d53bcbd4654b038a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 20:51:50 -0400 Subject: [PATCH 177/252] dataset type cleanup --- clickhouse_search/search.py | 1 - clickhouse_search/search_tests.py | 2 - seqr/utils/search/constants.py | 1 - seqr/utils/search/utils.py | 97 +++++++++++++------------------ 4 files changed, 42 insertions(+), 59 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index b999e91926..eba74f8704 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -45,7 +45,6 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno family_guid = None exclude_keys = search.pop('exclude_keys', None) or {} exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} - search.pop('dataset_type', None) for dataset_type, sample_data in sample_data_by_dataset_type.items(): logger.info(f'Loading {dataset_type} data for {sample_data["num_families"]} families', user) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index baf8aee7be..3dd6536c6f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -218,8 +218,6 @@ def _assert_expected_export_data(self, search_hash, export_data): def _assert_expected_search_error(self, error, **kwargs): response, search_hash, _ = self._execute_search(**kwargs) - if response.status_code != 400: - import pdb; pdb.set_trace() self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': error}) return search_hash diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py index ae6117128b..09627ce4f6 100644 --- a/seqr/utils/search/constants.py +++ b/seqr/utils/search/constants.py @@ -65,7 +65,6 @@ SV_CONSEQUENCES_FIELD = 'structural_consequence' SV_TYPE_FILTER_FIELD = 'structural' SV_ANNOTATION_TYPES = {SV_CONSEQUENCES_FIELD, SV_TYPE_FILTER_FIELD, NEW_SV_FIELD} -ALL_DATA_TYPES = 'ALL' CLINVAR_KEY = 'clinvar' CLINVAR_PATH_FILTER = 'pathogenic' diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 1870c9b650..5b17b4a598 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -10,7 +10,7 @@ from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE + MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom @@ -24,18 +24,6 @@ class InvalidSearchException(Exception): InvalidSearchException: 400, } -DATASET_TYPES_LOOKUP = { - data_types[0]: data_types for data_types in [ - [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS], - [Sample.DATASET_TYPE_MITO_CALLS], - [Sample.DATASET_TYPE_SV_CALLS], - ] -} -DATASET_TYPES_LOOKUP[ALL_DATA_TYPES] = [dt for dts in DATASET_TYPES_LOOKUP.values() for dt in dts] -DATASET_TYPE_SNP_INDEL_ONLY = f'{Sample.DATASET_TYPE_VARIANT_CALLS}_only' -DATASET_TYPES_LOOKUP[DATASET_TYPE_SNP_INDEL_ONLY] = [Sample.DATASET_TYPE_VARIANT_CALLS] -DATASET_TYPE_NO_MITO = f'{Sample.DATASET_TYPE_MITO_CALLS}_missing' -DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] MAX_GENES_FOR_FILTER = 10000 MIN_MULTI_FAMILY_SEQR_AC = 5000 @@ -72,12 +60,12 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) - dataset_type = _variant_ids_dataset_type([parsed_variant_id]) + dataset_types = _variant_ids_dataset_types([parsed_variant_id]) samples = _get_filtered_search_samples({'individual__family_id': family.id}) if len(samples) < 1: raise InvalidSearchException(f'No search data found for families {family.family_id}') variant = get_clickhouse_variant_by_id( - variant_id, parsed_variant_id, samples, family.project.genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], + variant_id, parsed_variant_id, samples, family.project.genome_version, dataset_types[0], ) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) @@ -96,7 +84,7 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ return variants parsed_variant_id = parse_variant_id(variant_id) - dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] + dataset_type = _variant_ids_dataset_types([parsed_variant_id])[0] _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) variants = clickhouse_variant_lookup(user, variant_id, parsed_variant_id, dataset_type, sample_type, genome_version, affected_only, hom_only) @@ -161,7 +149,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): _validate_sort(sort, families) parsed_search = _parse_search(search, genome_version, user) - search_dataset_type = _parse_dataset_type(parsed_search, genome_version) + search_dataset_types, comp_het_dts = _search_dataset_type(parsed_search, genome_version) _validate_search(parsed_search, families) @@ -171,10 +159,10 @@ def _query_variants(search_model, user, sort=None, **kwargs): return samples raise InvalidSearchException('No search data found for families {}'.format( ', '.join([f.family_id for f in families]))) - if search_dataset_type: - samples = samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[search_dataset_type]) + if search_dataset_types: + samples = samples.filter(dataset_type__in=search_dataset_types) if not samples: - raise InvalidSearchException(f'Unable to search against dataset type "{search_dataset_type}"') + raise InvalidSearchException(f'Unable to search against dataset type "{search_dataset_types[0]}"') if parsed_search.get('inheritance_mode') == X_LINKED_RECESSIVE: samples = samples.exclude(dataset_type=Sample.DATASET_TYPE_MITO_CALLS) if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): @@ -184,10 +172,9 @@ def _query_variants(search_model, user, sort=None, **kwargs): if skipped_samples: samples = samples.exclude(id__in=[s.id for s in skipped_samples]) - if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and parsed_search['secondary_dataset_type']: + if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and comp_het_dts: invalid_type = next(( - dt for dt in [parsed_search['dataset_type'], parsed_search['secondary_dataset_type']] - if dt and dt != ALL_DATA_TYPES and samples.filter(dataset_type__in=DATASET_TYPES_LOOKUP[dt]).count() < 1 + dts[0] for dts in comp_het_dts if samples.filter(dataset_type__in=dts).count() < 1 ), None) if invalid_type: raise InvalidSearchException( @@ -245,17 +232,6 @@ def _parse_search(search, genome_version, user): return parsed_search -def _parse_dataset_type(parsed_search, genome_version): - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) - parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) - search_dataset_type = None - if dataset_type and dataset_type != ALL_DATA_TYPES: - if secondary_dataset_type is None or secondary_dataset_type == dataset_type: - search_dataset_type = lookup_dataset_type or dataset_type - elif dataset_type == Sample.DATASET_TYPE_SV_CALLS: - search_dataset_type = DATASET_TYPE_NO_MITO - return search_dataset_type - def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) @@ -340,49 +316,60 @@ def _validate_sort(sort, families): def _search_dataset_type(search, genome_version): parsed_variant_ids = search.get('parsed_variant_ids') rsids = search.get('rs_ids') + comp_het_dts = [] if parsed_variant_ids or rsids: - lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) - return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type + search_dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS] if rsids else _variant_ids_dataset_types(parsed_variant_ids) + else: + chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ + interval['chrom'] for interval in (search.get('intervals') or []) + ] if not search.get('exclude_locations') else None + dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) + secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None - chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ - interval['chrom'] for interval in (search.get('intervals') or []) - ] if not search.get('exclude_locations') else None - dataset_type = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) - secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None + search_dataset_types = dataset_types + if dataset_types and secondary_dataset_types and dataset_types != secondary_dataset_types: + if len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: + secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] + search_dataset_types = {*dataset_types, *secondary_dataset_types} - return dataset_type, secondary_dataset_type, None + comp_het_dts = [dataset_types, secondary_dataset_types] + return search_dataset_types, comp_het_dts -def _variant_ids_dataset_type(all_variant_ids): + +def _variant_ids_dataset_types(all_variant_ids): variant_ids = [v for v in all_variant_ids if v] any_sv = len(variant_ids) < len(all_variant_ids) if len(variant_ids) == 0: - return Sample.DATASET_TYPE_SV_CALLS - return _chromosome_filter_dataset_type([vid[0] for vid in variant_ids], any_sv) + return [Sample.DATASET_TYPE_SV_CALLS] + return _chromosome_filter_dataset_types([vid[0] for vid in variant_ids], any_sv) -def _chromosome_filter_dataset_type(chroms, any_sv): +def _chromosome_filter_dataset_types(chroms, any_sv): has_mito = [chrom for chrom in chroms if chrom.replace('chr', '').startswith('M')] if len(has_mito) == len(chroms): - return Sample.DATASET_TYPE_MITO_CALLS - elif not has_mito: - return DATASET_TYPE_NO_MITO if any_sv else DATASET_TYPE_SNP_INDEL_ONLY - return ALL_DATA_TYPES if any_sv else Sample.DATASET_TYPE_VARIANT_CALLS + return [Sample.DATASET_TYPE_MITO_CALLS] + dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS] + if has_mito: + dataset_types.append(Sample.DATASET_TYPE_MITO_CALLS) + if any_sv: + dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) + return dataset_types def _annotation_dataset_type(annotations, chroms, pathogenicity=None, exclude_svs=False): if not (annotations or chroms): - return Sample.DATASET_TYPE_VARIANT_CALLS if (pathogenicity or exclude_svs) else None + return [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] if (pathogenicity or exclude_svs) else None annotation_types = set((annotations or {}).keys()) if annotations and annotation_types.issubset(SV_ANNOTATION_TYPES) and not pathogenicity: - return Sample.DATASET_TYPE_SV_CALLS + return [Sample.DATASET_TYPE_SV_CALLS] no_svs = exclude_svs or (annotations and annotation_types.isdisjoint(SV_ANNOTATION_TYPES)) if chroms: - return _chromosome_filter_dataset_type(chroms, any_sv=not no_svs) + return _chromosome_filter_dataset_types(chroms, any_sv=not no_svs) elif no_svs: - return Sample.DATASET_TYPE_VARIANT_CALLS - return ALL_DATA_TYPES + return [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] + return None def _parse_inheritance(search): From 7fb1e54dec73336f8c3e5bbf29a5e4b1cc2a1a0d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 20:57:11 -0400 Subject: [PATCH 178/252] clean up --- seqr/utils/search/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 5b17b4a598..0b754ce1f7 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -150,6 +150,8 @@ def _query_variants(search_model, user, sort=None, **kwargs): parsed_search = _parse_search(search, genome_version, user) search_dataset_types, comp_het_dts = _search_dataset_type(parsed_search, genome_version) + if parsed_search.get('inheritance_mode') == X_LINKED_RECESSIVE and Sample.DATASET_TYPE_MITO_CALLS in search_dataset_types: + search_dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) _validate_search(parsed_search, families) @@ -163,8 +165,6 @@ def _query_variants(search_model, user, sort=None, **kwargs): samples = samples.filter(dataset_type__in=search_dataset_types) if not samples: raise InvalidSearchException(f'Unable to search against dataset type "{search_dataset_types[0]}"') - if parsed_search.get('inheritance_mode') == X_LINKED_RECESSIVE: - samples = samples.exclude(dataset_type=Sample.DATASET_TYPE_MITO_CALLS) if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): samples = samples.select_related('individual') if samples: From a42b283ebdb8d0c63463e5be17e3f876f42cd205 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 21:10:12 -0400 Subject: [PATCH 179/252] clean up --- seqr/utils/search/utils.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0b754ce1f7..640aeac64c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -1,6 +1,8 @@ from collections import defaultdict from copy import deepcopy from datetime import timedelta + +from pyarrow.dataset import dataset from pyliftover.liftover import LiftOver from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ @@ -339,20 +341,21 @@ def _search_dataset_type(search, genome_version): def _variant_ids_dataset_types(all_variant_ids): variant_ids = [v for v in all_variant_ids if v] - any_sv = len(variant_ids) < len(all_variant_ids) - if len(variant_ids) == 0: - return [Sample.DATASET_TYPE_SV_CALLS] - return _chromosome_filter_dataset_types([vid[0] for vid in variant_ids], any_sv) + dataset_types = [] + if len(variant_ids) < len(all_variant_ids): + dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) + if variant_ids: + dataset_types += _chromosome_filter_dataset_types([vid[0] for vid in variant_ids]) + return dataset_types -def _chromosome_filter_dataset_types(chroms, any_sv): + +def _chromosome_filter_dataset_types(chroms): has_mito = [chrom for chrom in chroms if chrom.replace('chr', '').startswith('M')] if len(has_mito) == len(chroms): return [Sample.DATASET_TYPE_MITO_CALLS] dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS] if has_mito: dataset_types.append(Sample.DATASET_TYPE_MITO_CALLS) - if any_sv: - dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) return dataset_types @@ -364,12 +367,13 @@ def _annotation_dataset_type(annotations, chroms, pathogenicity=None, exclude_sv if annotations and annotation_types.issubset(SV_ANNOTATION_TYPES) and not pathogenicity: return [Sample.DATASET_TYPE_SV_CALLS] + dataset_types = _chromosome_filter_dataset_types(chroms) if chroms else [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] + no_svs = exclude_svs or (annotations and annotation_types.isdisjoint(SV_ANNOTATION_TYPES)) - if chroms: - return _chromosome_filter_dataset_types(chroms, any_sv=not no_svs) - elif no_svs: - return [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_MITO_CALLS] - return None + if not no_svs: + dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) + + return dataset_types def _parse_inheritance(search): From f69f5708ad434eaaa81bb04ae63aacc256943217 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 23:33:04 -0400 Subject: [PATCH 180/252] clean up --- seqr/utils/search/utils.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 640aeac64c..03b1e9e4a1 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -62,12 +62,12 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) - dataset_types = _variant_ids_dataset_types([parsed_variant_id]) + dataset_type = _variant_id_dataset_type(parsed_variant_id) samples = _get_filtered_search_samples({'individual__family_id': family.id}) if len(samples) < 1: raise InvalidSearchException(f'No search data found for families {family.family_id}') variant = get_clickhouse_variant_by_id( - variant_id, parsed_variant_id, samples, family.project.genome_version, dataset_types[0], + variant_id, parsed_variant_id, samples, family.project.genome_version, dataset_type, ) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) @@ -85,8 +85,9 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ if variants: return variants + # TODO move into search helper func parsed_variant_id = parse_variant_id(variant_id) - dataset_type = _variant_ids_dataset_types([parsed_variant_id])[0] + dataset_type = _variant_id_dataset_type(parsed_variant_id) _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) variants = clickhouse_variant_lookup(user, variant_id, parsed_variant_id, dataset_type, sample_type, genome_version, affected_only, hom_only) @@ -317,10 +318,9 @@ def _validate_sort(sort, families): def _search_dataset_type(search, genome_version): parsed_variant_ids = search.get('parsed_variant_ids') - rsids = search.get('rs_ids') comp_het_dts = [] - if parsed_variant_ids or rsids: - search_dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS] if rsids else _variant_ids_dataset_types(parsed_variant_ids) + if parsed_variant_ids: + search_dataset_types = _chromosome_filter_dataset_types([vid[0] for vid in parsed_variant_ids]) else: chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ interval['chrom'] for interval in (search.get('intervals') or []) @@ -339,14 +339,12 @@ def _search_dataset_type(search, genome_version): return search_dataset_types, comp_het_dts -def _variant_ids_dataset_types(all_variant_ids): - variant_ids = [v for v in all_variant_ids if v] - dataset_types = [] - if len(variant_ids) < len(all_variant_ids): - dataset_types.append(Sample.DATASET_TYPE_SV_CALLS) - if variant_ids: - dataset_types += _chromosome_filter_dataset_types([vid[0] for vid in variant_ids]) - return dataset_types +def _variant_id_dataset_type(parsed_variant_id): + if not parsed_variant_id: + return Sample.DATASET_TYPE_SV_CALLS + if parsed_variant_id[0].replace('chr', '').startswith('M'): + return Sample.DATASET_TYPE_MITO_CALLS + return Sample.DATASET_TYPE_VARIANT_CALLS def _chromosome_filter_dataset_types(chroms): From a0c445b0033db3290e3cf5297befddbea312990c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 23:33:56 -0400 Subject: [PATCH 181/252] clean up --- seqr/utils/search/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 03b1e9e4a1..26566a4afc 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -2,7 +2,6 @@ from copy import deepcopy from datetime import timedelta -from pyarrow.dataset import dataset from pyliftover.liftover import LiftOver from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ From c0b061cc73e34474d037dc53bf565eaf8ea1e6af Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 16 Mar 2026 23:42:54 -0400 Subject: [PATCH 182/252] move search excpetion --- clickhouse_search/search.py | 5 +++-- seqr/utils/search/utils.py | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index eba74f8704..18c875da22 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -24,6 +24,9 @@ logger = SeqrLogger(__name__) +class InvalidSearchException(Exception): + pass + BATCH_SIZE = 10000 TRANSCRIPT_CONSEQUENCES_FIELD = 'sortedTranscriptConsequences' @@ -111,7 +114,6 @@ def _get_search_results(*args, skip_entry_fields=False, **search_kwargs): def _evaluate_results(result_q, is_comp_het=False): results = [list(result[1:]) if is_comp_het else result for result in result_q[:MAX_VARIANTS + 1]] if len(results) > MAX_VARIANTS: - from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException('This search returned too many results') return results @@ -423,7 +425,6 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate affected=ArrayAgg('individual__affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: - from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 26566a4afc..a27dc3d5bc 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -5,7 +5,7 @@ from pyliftover.liftover import LiftOver from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ - get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id + get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger @@ -17,9 +17,6 @@ logger = SeqrLogger(__name__) -class InvalidSearchException(Exception): - pass - SEARCH_EXCEPTION_ERROR_MAP = { InvalidSearchException: 400, From 6e3acf2b2e7ac52136d17dcf596eb61a37fac6fd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 00:07:26 -0400 Subject: [PATCH 183/252] clean up --- seqr/utils/search/utils.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index a27dc3d5bc..f96fe04c7f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -148,10 +148,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): _validate_sort(sort, families) parsed_search = _parse_search(search, genome_version, user) - search_dataset_types, comp_het_dts = _search_dataset_type(parsed_search, genome_version) - if parsed_search.get('inheritance_mode') == X_LINKED_RECESSIVE and Sample.DATASET_TYPE_MITO_CALLS in search_dataset_types: - search_dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) - + dataset_types, secondary_dataset_types = _search_dataset_type(parsed_search, genome_version) _validate_search(parsed_search, families) samples = _get_filtered_search_samples({'individual__family__in': families}) @@ -160,10 +157,10 @@ def _query_variants(search_model, user, sort=None, **kwargs): return samples raise InvalidSearchException('No search data found for families {}'.format( ', '.join([f.family_id for f in families]))) - if search_dataset_types: - samples = samples.filter(dataset_type__in=search_dataset_types) + if dataset_types: + samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) if not samples: - raise InvalidSearchException(f'Unable to search against dataset type "{search_dataset_types[0]}"') + raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): samples = samples.select_related('individual') if samples: @@ -171,9 +168,10 @@ def _query_variants(search_model, user, sort=None, **kwargs): if skipped_samples: samples = samples.exclude(id__in=[s.id for s in skipped_samples]) - if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and comp_het_dts: + if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and secondary_dataset_types: invalid_type = next(( - dts[0] for dts in comp_het_dts if samples.filter(dataset_type__in=dts).count() < 1 + dts[0] for dts in [dataset_types, secondary_dataset_types] + if dts and samples.filter(dataset_type__in=dts).count() < 1 ), None) if invalid_type: raise InvalidSearchException( @@ -314,25 +312,22 @@ def _validate_sort(sort, families): def _search_dataset_type(search, genome_version): parsed_variant_ids = search.get('parsed_variant_ids') - comp_het_dts = [] + secondary_dataset_types = None if parsed_variant_ids: - search_dataset_types = _chromosome_filter_dataset_types([vid[0] for vid in parsed_variant_ids]) + dataset_types = _chromosome_filter_dataset_types([vid[0] for vid in parsed_variant_ids]) else: chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ interval['chrom'] for interval in (search.get('intervals') or []) ] if not search.get('exclude_locations') else None dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None + if secondary_dataset_types and len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: + secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] - search_dataset_types = dataset_types - if dataset_types and secondary_dataset_types and dataset_types != secondary_dataset_types: - if len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: - secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] - search_dataset_types = {*dataset_types, *secondary_dataset_types} - - comp_het_dts = [dataset_types, secondary_dataset_types] + if search.get('inheritance_mode') == X_LINKED_RECESSIVE and Sample.DATASET_TYPE_MITO_CALLS in dataset_types: + dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) - return search_dataset_types, comp_het_dts + return dataset_types, secondary_dataset_types def _variant_id_dataset_type(parsed_variant_id): From 6ebfb4bd3923949bd54f721af3243079615aaeed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 00:10:36 -0400 Subject: [PATCH 184/252] clean up --- seqr/utils/search/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f96fe04c7f..762f831f46 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -28,6 +28,7 @@ def _get_filtered_search_samples(search_filter, active_only=True): + # TODO clean up samples = Sample.objects.filter(**search_filter) if active_only: samples = samples.filter(is_active=True) @@ -151,7 +152,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): dataset_types, secondary_dataset_types = _search_dataset_type(parsed_search, genome_version) _validate_search(parsed_search, families) - samples = _get_filtered_search_samples({'individual__family__in': families}) + samples = Sample.objects.filter(individual__family__in=families, is_active=True) if len(samples) < 1: if parsed_search.get('no_access_project_genome_version'): return samples From e25d0b9c19e16b126b8f8aa3548e9a89b6b3a021 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 13:17:07 -0400 Subject: [PATCH 185/252] fix tests --- clickhouse_search/search_tests.py | 15 ++++++++------- seqr/utils/search/utils.py | 17 ++++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3dd6536c6f..646631a752 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -275,10 +275,10 @@ def test_single_family_search(self): ) self._assert_expected_search( - [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, - GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, - project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], + [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, + GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + gene_counts={**variant_gene_counts, **MITO_GENE_COUNTS, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, + project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) self._set_grch37_search() @@ -374,27 +374,28 @@ def test_all_project_search(self): ) self.login_manager() + locus = {'rawItems': 'chr1:1-100000000, chr13:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, chr21:1-100000000, M:1-100000000'} project_families += SV_PROJECT_FAMILIES results = [ VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, SV_VARIANT4, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, ] self._assert_expected_search( - results, request_body=request_body, project_families=project_families, additional_response=additional_response, + results, request_body=request_body, project_families=project_families, additional_response=additional_response, locus=locus, ) request_body['unsolvedFamiliesOnly'] = True project_families[0]['familyGuids'].remove('F000007_7') project_families[0]['familyGuids'].remove('F000010_10') self._assert_expected_search( - results, request_body=request_body, project_families=project_families, additional_response=additional_response, + results, request_body=request_body, project_families=project_families, additional_response=additional_response, locus=locus, ) request_body['trioFamiliesOnly'] = True self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - request_body=request_body, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + request_body=request_body, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, locus=locus, ) def test_both_sample_types_search(self): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 762f831f46..3261a4abd1 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -130,6 +130,7 @@ def _get_previous_search_results(search_model, sort): def query_variants(search_model, sort, page, num_results, user): + resp = _query_variants(search_model, user, sort=sort) previous_search_results, genome_version = _query_variants(search_model, user, sort=sort) all_results = previous_search_results.get('all_results') or [] @@ -154,13 +155,12 @@ def _query_variants(search_model, user, sort=None, **kwargs): samples = Sample.objects.filter(individual__family__in=families, is_active=True) if len(samples) < 1: - if parsed_search.get('no_access_project_genome_version'): - return samples - raise InvalidSearchException('No search data found for families {}'.format( - ', '.join([f.family_id for f in families]))) + if not parsed_search.get('no_access_project_genome_version'): + raise InvalidSearchException('No search data found for families {}'.format( + ', '.join([f.family_id for f in families]))) if dataset_types: samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) - if not samples: + if not samples and not parsed_search.get('no_access_project_genome_version'): raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): samples = samples.select_related('individual') @@ -325,8 +325,11 @@ def _search_dataset_type(search, genome_version): if secondary_dataset_types and len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] - if search.get('inheritance_mode') == X_LINKED_RECESSIVE and Sample.DATASET_TYPE_MITO_CALLS in dataset_types: - dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) + if search.get('inheritance_mode') == X_LINKED_RECESSIVE: + if not dataset_types: + dataset_types = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] + elif Sample.DATASET_TYPE_MITO_CALLS in dataset_types: + dataset_types.remove(Sample.DATASET_TYPE_MITO_CALLS) return dataset_types, secondary_dataset_types From 6058f1eab830309d8355bdadb58eef62b441fe49 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 13:44:45 -0400 Subject: [PATCH 186/252] clean up --- seqr/utils/search/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 3261a4abd1..9a5595fddf 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -130,7 +130,6 @@ def _get_previous_search_results(search_model, sort): def query_variants(search_model, sort, page, num_results, user): - resp = _query_variants(search_model, user, sort=sort) previous_search_results, genome_version = _query_variants(search_model, user, sort=sort) all_results = previous_search_results.get('all_results') or [] From 92dd74d0fd9080ad91ed6e052e7c07feb148538f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 14:31:49 -0400 Subject: [PATCH 187/252] add error handling for unmapped clickhouse lookup samples --- clickhouse_search/search_tests.py | 21 ++++++++++++++++++++- seqr/views/apis/variant_search_api.py | 18 +++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 626e792698..a537812ff1 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -38,7 +38,7 @@ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant +from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -1280,6 +1280,25 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) + # Test error handling when sample is present in clickhouse but absent in postgres + self.reset_logs() + Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') + missing_gt_variant = { + **GRCH37_VARIANT, 'genotypes': {k: v for k, v in GRCH37_VARIANT['genotypes'].items() if k != 'I000006_hg00733'}, + } + self._assert_expected_lookup( + '7-143270172-A-G', missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', + project_guids=['R0001_1kg'], family_guids=['F000002_2'], + individual_guids=['I000006_hg00733', 'I000005_hg00732', 'I000004_hg00731'] + ) + self.assert_json_logs(self.manager_user, [ + ('Looking up variant 7-143270172-A-G with data type SNV_INDEL', None), + ('Unable to map sample HG00733 in family F000002_2 to an individual for variant 7-143270172-A-G', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) + def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' if hom_only: diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 6646369f62..30273a01e8 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,15 +12,15 @@ from math import ceil import re -from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ variant_lookup, parse_variant_id, export_variants from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException -from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display +from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json @@ -32,6 +32,8 @@ from seqr.views.utils.variant_utils import get_variants_response from seqr.views.utils.vlm_utils import vlm_lookup +logger = SeqrLogger(__name__) + GENOTYPE_AC_LOOKUP = { 'ref_ref': [0, 0], @@ -544,12 +546,12 @@ def variant_lookup_handler(request): (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() } for variant in variants: - _update_lookup_variant(variant, response, individual_guid_map) + _update_lookup_variant(variant, response, individual_guid_map, request.user) return create_json_response(response) -def _update_lookup_variant(variant, response, individual_guid_map): +def _update_lookup_variant(variant, response, individual_guid_map, user): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) @@ -565,7 +567,13 @@ def _update_lookup_variant(variant, response, individual_guid_map): variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): - individual_guid = individual_guid_map[(family_guid, genotype['sampleId'])] + individual_guid = individual_guid_map.get((family_guid, genotype['sampleId'])) + if not individual_guid: + logger.error( + f'Unable to map sample {genotype["sampleId"]} in family {family_guid} to an individual for variant {variant["variantId"]}', + user, + ) + continue genotype = {**genotype, 'individualGuid': individual_guid} if individual_guid in variant['genotypes']: genotype = [variant['genotypes'][individual_guid], genotype] From 003b55853aac84bb9b4812e56ec68209f7be5466 Mon Sep 17 00:00:00 2001 From: hanars Date: Tue, 17 Mar 2026 15:05:40 -0400 Subject: [PATCH 188/252] pr comment feedback Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a537812ff1..095e9d909f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1280,7 +1280,7 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) - # Test error handling when sample is present in clickhouse but absent in postgres + # Test error handling when the ClickHouse sampleId cannot be mapped to any Postgres Individual self.reset_logs() Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') missing_gt_variant = { From 6164a63353e02be10543fe81ae3e77d3044e5989 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 16:32:19 -0400 Subject: [PATCH 189/252] move sample logic into clickhouse search --- clickhouse_search/search.py | 100 +++++++++++++----- .../check_for_new_samples_from_pipeline.py | 1 - .../commands/tag_seqr_prioritized_variants.py | 20 ++-- seqr/utils/search/utils.py | 58 +--------- seqr/views/utils/variant_utils.py | 6 +- 5 files changed, 88 insertions(+), 97 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 18c875da22..bff9a4dd95 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -15,7 +15,7 @@ from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization, Individual +from seqr.models import Sample, PhenotypePrioritization, Family, Individual from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ @@ -34,15 +34,21 @@ class InvalidSearchException(Exception): SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version, sort=None, **kwargs): +def get_clickhouse_variants(families, dataset_types, secondary_dataset_types, search, user, previous_search_results, genome_version, sort=None, **kwargs): inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) sample_data_by_dataset_type = _get_sample_data( - samples, + families, + dataset_types, + secondary_dataset_types=secondary_dataset_types, skip_multi_project_individual_guid=True, annotate_affected_males=has_x_chrom_comp_het or has_x_linked, + has_comp_het=has_comp_het, + inheritance_mode=inheritance_mode, + inheritance_filter=search.get('inheritance_filter'), + allow_no_samples=bool(search.get('no_access_project_genome_version')), ) results = [] family_guid = None @@ -79,13 +85,13 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno dataset_results += _evaluate_results(result_q, is_comp_het=True) if 'samples' not in sample_data: - add_individual_guids(dataset_results, samples) + add_individual_guids(dataset_results) results += dataset_results if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_by_dataset_type, user, exclude_key_pairs, **search) + results += _get_multi_data_type_comp_het_results(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, **search) if search.get('no_access_project_genome_version'): logger.info('Looking up variants in projects with no user access', user) @@ -117,7 +123,7 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -163,7 +169,7 @@ def _get_multi_data_type_comp_het_results(genome_version, samples, sample_data_b ) dataset_results = _evaluate_results(result_q, is_comp_het=True) if not sv_sample_data['samples']: - add_individual_guids(dataset_results, samples) + add_individual_guids(dataset_results) results += dataset_results return results @@ -289,14 +295,14 @@ def _result_as_tuple(results, field_prefix): return Tuple(*fields.keys(), output_field=NamedTupleField(list(fields.values()))) -def add_individual_guids(results, samples, encode_genotypes_json=False): +def add_individual_guids(results, encode_genotypes_json=False): families = set() for result in results: for r in (result if isinstance(result, list) else [result]): families.update(r.get('familyGenotypes', {}).keys()) sample_map = { - (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in samples.filter( - individual__family__guid__in=families, + (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Sample.objects.filter( + individual__family__guid__in=families, is_active=True, ).values_list('individual__family__guid', 'individual__guid', 'sample_id') } for result in results: @@ -419,7 +425,20 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): and transcript.get('spliceregion', {}).get('extended_intronic_splice_region_variant') == minimal_transcript.get('extendedIntronicSpliceRegionVariant')) -def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate_affected_males=False): +def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): + samples = Sample.objects.filter(individual__family__in=families, is_active=True) + if not samples.exists(): + if allow_no_samples: + return {} + raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}') + + if dataset_types: + samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) + if not samples and not allow_no_samples: + if allow_no_samples: + return {} + raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') + mismatch_affected_samples = samples.values('sample_id', 'dataset_type').annotate( projects=ArrayAgg('individual__family__project__name', distinct=True), affected=ArrayAgg('individual__affected', distinct=True), @@ -433,9 +452,16 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate skip_individual_guid = ( skip_multi_project_individual_guid and samples.values('individual__family__project_id').distinct().count() > 1 ) + individual_affected_status = (inheritance_filter or {}).get('affected') + genotype_filter = (inheritance_filter or {}).get('genotype') + affected_family_only = inheritance_mode and not individual_affected_status + family_array_kwargs = {'distinct': True} + if affected_family_only: + family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED) + annotations = { 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True), - 'family_guids': ArrayAgg('individual__family__guid', distinct=True), + 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs), } if skip_individual_guid: annotations['num_unaffected'] = Count( @@ -451,6 +477,9 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate )) sample_data = samples.values('dataset_type', 'sample_type').annotate(**annotations) + if affected_family_only: + sample_data = sample_data.filter(family_guids__len__gt=0) + samples_by_dataset_type = {} for data in sample_data: dataset_type = data.pop('dataset_type') @@ -458,6 +487,13 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate if dataset_type == Sample.DATASET_TYPE_SV_CALLS: dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + if genotype_filter and all(s['individual_guid'] not in genotype_filter for s in data.get('samples', [])): + continue + if inheritance_mode and individual_affected_status and all( + (individual_affected_status.get(s['individual_guid']) or s['affected']) != AFFECTED for s in data.get('samples', []) + ): + continue + family_guids = set(data.pop('family_guids')) if dataset_type in samples_by_dataset_type: other_type_data = samples_by_dataset_type[dataset_type] @@ -489,6 +525,23 @@ def _get_sample_data(samples, skip_multi_project_individual_guid=False, annotate 'num_families': len(set().union(*data['sample_type_families'].values())), } + if not samples_by_dataset_type: + raise InvalidSearchException( + 'Invalid custom inheritance' if genotype_filter else + 'Inheritance based search is disabled in families with no data loaded for affected individuals' + ) + + if has_comp_het and secondary_dataset_types: + invalid_type = next(( + dts[0] for dts in [dataset_types, secondary_dataset_types] if dts and not any( + dt for dt in dts if (Sample.DATASET_TYPE_SV_CALLS if dt.startswith(Sample.DATASET_TYPE_SV_CALLS) else dt) + in samples_by_dataset_type + )), None) + if invalid_type: + raise InvalidSearchException( + f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' + ) + _add_missing_multi_type_samples(samples, samples_by_dataset_type) return samples_by_dataset_type @@ -682,12 +735,10 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, samples=None, affected_only=False, hom_only=False): +def _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, sample_data=None, affected_only=False, hom_only=False): entry_cls = ENTRY_CLASS_MAP[genome_version][data_type] variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] - sample_data = _get_sample_data(samples)[data_type] if samples else None - entries = entry_cls.objects.filter_locus( variant_ids=[variant_id], parsed_variant_ids=[parsed_variant_id] if parsed_variant_id else [], ) @@ -768,23 +819,18 @@ def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_o variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) -def get_clickhouse_variant_by_id(variant_id, parsed_variant_id, samples, genome_version, dataset_type): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - data_types = [ - f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - for sample_type in samples.values_list('sample_type', flat=True).distinct() - ] - else: - data_types = [dataset_type] - for data_type in data_types: - variant = _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, samples) +def get_clickhouse_variant_by_id(variant_id, parsed_variant_id, family, dataset_type): + genome_version = family.project.genome_version + sample_data_by_type = _get_sample_data([family], [dataset_type]) + for data_type, sample_data in sample_data_by_type.items(): + variant = _clickhouse_variant_lookup(variant_id, parsed_variant_id, genome_version, data_type, sample_data) if variant: return variant return None -def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): - sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] +def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys): + sample_data = _get_sample_data(Family.objects.filter(guid__in=family_guids), [dataset_type])[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( project_guid=project_guid, family_guid__in=family_guids, key__in=keys, ) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index e98c02d7c4..0544214d06 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -405,7 +405,6 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type) variants = [] genotypes_by_key = get_clickhouse_genotypes( project.guid, [family_guid], project.genome_version, dataset_type, variant_models_by_key.keys(), - samples=Sample.objects.filter(is_active=True), ) for key, genotypes in genotypes_by_key.items(): variant = variant_models_by_key[key] diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 9ef1fca0d6..18d09fd19d 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -466,7 +466,7 @@ def handle(self, *args, **options): ) self._run_multi_data_type_comp_het_search( - updates, search_counts, samples_by_dataset_type, family_guid_map, project, sample_qs, genes=gene_by_moi[RECESSIVE_MOI], + updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes=gene_by_moi[RECESSIVE_MOI], ) new_tag_keys = updates['new_tag_keys'] @@ -526,7 +526,7 @@ def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, ) run_search_func = cls._run_comp_het_search if config_search['inheritance_mode'] == COMPOUND_HET else cls._run_search num_results = run_search_func( - search_name, family_variant_data, family_guid_map, dataset_type, sample_data, sample_qs, + search_name, family_variant_data, family_guid_map, dataset_type, sample_data, exclude_locations=exclude_locations, genes=search_genes, **config_search, **ALL_SEARCHES_CRITERIA, ) logger.info(f'Found {num_results} variants for criteria: {search_name}') @@ -567,7 +567,7 @@ def wrapped(v): return wrapped @classmethod - def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, samples, **kwargs): + def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, **kwargs): results_qs = get_search_queryset(GENOME_VERSION_GRCh38, dataset_type, sample_data, **kwargs) variant_fields = [] @@ -579,7 +579,7 @@ def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_ variant_fields += ['familyGenotypes'] results = results_qs.values(*variant_fields, 'key', 'familyGuids', **variant_values) - add_individual_guids(results, samples, encode_genotypes_json=True) + add_individual_guids(results, encode_genotypes_json=True) for variant in results: for family_guid in variant.pop('familyGuids'): @@ -590,16 +590,16 @@ def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_ return len(results) @classmethod - def _run_comp_het_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, samples, **kwargs): + def _run_comp_het_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, **kwargs): queryset = get_data_type_comp_het_results_queryset( GENOME_VERSION_GRCh38, dataset_type, sample_data, **kwargs, ) return cls._execute_comp_het_search( - queryset, search_name, family_variant_data, family_guid_map, samples, + queryset, search_name, family_variant_data, family_guid_map, ) @classmethod - def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, samples, genes): + def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes): sv_dataset_type = next(dt for dt in samples_by_dataset_type.keys() if dt.startswith('SV')) sample_type = sv_dataset_type.split('_')[-1] families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].keys()) @@ -623,16 +623,16 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by GENOME_VERSION_GRCh38, sv_dataset_type, sv_sample_data, snv_indel_sample_data, num_families=len(families), genes=genes, **config_search, **ALL_SEARCHES_CRITERIA, ) - num_results = cls._execute_comp_het_search(queryset, search_name, family_variant_data, family_guid_map, samples) + num_results = cls._execute_comp_het_search(queryset, search_name, family_variant_data, family_guid_map) logger.info(f'Found {num_results} variants for criteria: {search_name}') search_counts[search_name] = num_results cls._bulk_tag_variants(family_variant_data, updates) @classmethod - def _execute_comp_het_search(cls, queryset, search_name, family_variant_data, family_guid_map, samples): + def _execute_comp_het_search(cls, queryset, search_name, family_variant_data, family_guid_map): results = [list(v[1:]) for v in queryset] - add_individual_guids(results, samples, encode_genotypes_json=True) + add_individual_guids(results, encode_genotypes_json=True) for pair in results: for family_guid in pair[0]['familyGuids']: for variant, support_id in [(pair[0], pair[1]['key']), (pair[1], pair[0]['key'])]: diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 9a5595fddf..5fea0a172f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -57,14 +57,14 @@ def _get_search_genome_version(search_model): return next(iter(project_versions.keys())) -def get_single_variant(family, variant_id, user=None): +def get_single_variant(family, variant_id): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_id_dataset_type(parsed_variant_id) samples = _get_filtered_search_samples({'individual__family_id': family.id}) if len(samples) < 1: raise InvalidSearchException(f'No search data found for families {family.family_id}') variant = get_clickhouse_variant_by_id( - variant_id, parsed_variant_id, samples, family.project.genome_version, dataset_type, + variant_id, parsed_variant_id, family, dataset_type, ) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) @@ -82,7 +82,6 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ if variants: return variants - # TODO move into search helper func parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_id_dataset_type(parsed_variant_id) _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) @@ -152,33 +151,7 @@ def _query_variants(search_model, user, sort=None, **kwargs): dataset_types, secondary_dataset_types = _search_dataset_type(parsed_search, genome_version) _validate_search(parsed_search, families) - samples = Sample.objects.filter(individual__family__in=families, is_active=True) - if len(samples) < 1: - if not parsed_search.get('no_access_project_genome_version'): - raise InvalidSearchException('No search data found for families {}'.format( - ', '.join([f.family_id for f in families]))) - if dataset_types: - samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) - if not samples and not parsed_search.get('no_access_project_genome_version'): - raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') - if parsed_search.get('inheritance_mode') or parsed_search.get('inheritance_filter'): - samples = samples.select_related('individual') - if samples: - skipped_samples = _filter_inheritance_family_samples(samples, parsed_search['inheritance_filter']) - if skipped_samples: - samples = samples.exclude(id__in=[s.id for s in skipped_samples]) - - if (parsed_search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET}) and secondary_dataset_types: - invalid_type = next(( - dts[0] for dts in [dataset_types, secondary_dataset_types] - if dts and samples.filter(dataset_type__in=dts).count() < 1 - ), None) - if invalid_type: - raise InvalidSearchException( - f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' - ) - - get_clickhouse_variants(samples, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) + get_clickhouse_variants(families, dataset_types, secondary_dataset_types, parsed_search, user, previous_search_results, genome_version, sort=sort, **kwargs) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) @@ -405,31 +378,6 @@ def _validate_search(search, families): ) -def _filter_inheritance_family_samples(samples, inheritance_filter): - family_groups = defaultdict(set) - individual_affected_status = inheritance_filter.get('affected') or {} - genotype_filter = None if inheritance_filter.get(Individual.AFFECTED_STATUS_AFFECTED) else inheritance_filter.get('genotype') - for sample in samples: - if genotype_filter: - is_filtered_family = sample.individual.guid in genotype_filter - else: - affected_status = individual_affected_status.get(sample.individual.guid) or sample.individual.affected - is_filtered_family = affected_status == Individual.AFFECTED_STATUS_AFFECTED - - if is_filtered_family: - family_groups[sample.individual.family_id].add(sample.dataset_type) - - if not family_groups: - raise InvalidSearchException( - 'Invalid custom inheritance' if genotype_filter else - 'Inheritance based search is disabled in families with no data loaded for affected individuals' - ) - - return [ - s for s in samples if s.dataset_type not in family_groups[s.individual.family_id] - ] - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index d1a15bad7a..411864b760 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -187,9 +187,8 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_ variant_families[variant_id].append(family_id) families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} - samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) search_variants_by_id = { - v['variantId']: v for v in _get_clickhouse_variants(samples, families_by_id, family_variant_ids, genome_version) + v['variantId']: v for v in _get_clickhouse_variants(families_by_id, family_variant_ids, genome_version) } new_variants = {} @@ -212,7 +211,7 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_ return new_variants -def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: +def _get_clickhouse_variants(families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: variant_data = _get_clickhouse_variant_annotations( {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, ) @@ -226,7 +225,6 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), - samples, ) for key, genotypes in genotype_keys.items(): variants_by_key[key]['genotypes'].update(genotypes) From 2131ad41a1b13a4581a73bcf6e5bba8dca1de36b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 16:39:49 -0400 Subject: [PATCH 190/252] clean up --- .../commands/tag_seqr_prioritized_variants.py | 4 ++-- seqr/utils/search/utils.py | 17 +---------------- seqr/views/apis/variant_search_api.py | 7 +++---- seqr/views/utils/anvil_metadata_utils.py | 4 +--- seqr/views/utils/pedigree_info_utils.py | 5 ++--- 5 files changed, 9 insertions(+), 28 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 18d09fd19d..26d7897c1d 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -13,7 +13,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.gene_utils import get_genes from seqr.utils.search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO -from seqr.utils.search.utils import get_search_samples, COMPOUND_HET +from seqr.utils.search.utils import COMPOUND_HET from seqr.views.utils.orm_to_json_utils import SEQR_TAG_TYPE from seqr.views.utils.variant_utils import bulk_create_tagged_variants from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL @@ -458,7 +458,7 @@ def handle(self, *args, **options): updates = {update: set() for update in ['matched_families', 'new_tag_keys', 'update_tag_keys', 'skipped_tag_keys']} search_counts = {} samples_by_dataset_type = {} - sample_qs = get_search_samples([project]) + sample_qs = Sample.objects.filter(individual__family__project=project, is_active=True) for dataset_type, searches in SEARCHES.items(): self._run_dataset_type_searches( dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 5fea0a172f..0fdc7d6d13 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -7,7 +7,7 @@ from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Individual, Project, VariantSearchResults +from seqr.models import Sample, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ @@ -27,18 +27,6 @@ MIN_MULTI_FAMILY_SEQR_AC = 5000 -def _get_filtered_search_samples(search_filter, active_only=True): - # TODO clean up - samples = Sample.objects.filter(**search_filter) - if active_only: - samples = samples.filter(is_active=True) - return samples - - -def get_search_samples(projects, active_only=True): - return _get_filtered_search_samples({'individual__family__project__in': projects}, active_only=active_only) - - def _get_search_genome_version(search_model): projects = Project.objects.filter(family__in=search_model.families.all()).values_list('genome_version', 'name').distinct() project_versions = defaultdict(set) @@ -60,9 +48,6 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_id_dataset_type(parsed_variant_id) - samples = _get_filtered_search_samples({'individual__family_id': family.id}) - if len(samples) < 1: - raise InvalidSearchException(f'No search data found for families {family.family_id}') variant = get_clickhouse_variant_by_id( variant_id, parsed_variant_id, family, dataset_type, ) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 6646369f62..bc044dcc20 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,13 +12,12 @@ from math import ceil import re -from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ +from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ variant_lookup, parse_variant_id, export_variants from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException -from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.views.utils.json_utils import create_json_response, _to_snake_case @@ -366,7 +365,7 @@ def search_context_handler(request): **FAMILY_ADDITIONAL_VALUES, )} - family_sample_types = get_search_samples(projects).values('individual__family__guid').annotate( + family_sample_types = Sample.objects.filter(individual__family__project__in=projects, is_active=True).values('individual__family__guid').annotate( samples=ArrayAgg(JSONObject(sampleType='sample_type', datasetType='dataset_type', isActive=Value(True)), distinct=True)) project_dataset_types = defaultdict(set) for agg in family_sample_types: diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 01eb449849..5e002b119d 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -8,7 +8,6 @@ import requests from typing import Callable, Iterable -from clickhouse_search.backend.functions import ArrayFilter from clickhouse_search.search import get_variants_queryset, get_variant_main_transcripts_by_key from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP @@ -16,7 +15,6 @@ from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY @@ -284,7 +282,7 @@ def _get_all_project_individual_samples(projects): def _get_sorted_search_samples(projects): - return get_search_samples(projects, active_only=False).order_by('-loaded_date') + return Sample.objects.filter(individual__family__project__in=projects).order_by('-loaded_date') HET = 'Heterozygous' diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index fd122b4fb2..0a399afc50 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -13,10 +13,9 @@ from seqr.utils.communication_utils import send_html_email from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import get_search_samples from seqr.views.utils.json_utils import _to_snake_case, _to_title_case from seqr.views.utils.permissions_utils import user_is_pm, get_pm_user_emails -from seqr.models import Individual +from seqr.models import Individual, Sample logger = SeqrLogger(__name__) @@ -364,7 +363,7 @@ def get_validated_related_individuals(project, records_by_id, errors, related_gu affected = records_by_id[individual_id].get(JsonConstants.AFFECTED_COLUMN, Individual.AFFECTED_STATUS_UNKNOWN) affected_status_by_family[family_id].append(affected) - search_samples = get_search_samples([project]) + search_samples = Sample.objects.filter(individual__family__project=project, is_active=True) sample_type = None if search_dataset_type: From 71e6317be0a73d0a02f1cf93e84bcf443f12a023 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 17:41:46 -0400 Subject: [PATCH 191/252] fix comp het type validation --- clickhouse_search/search.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index bff9a4dd95..878fe9824e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -532,11 +532,13 @@ def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid ) if has_comp_het and secondary_dataset_types: + sample_dataset_types = { + Sample.DATASET_TYPE_SV_CALLS if dt.startswith(Sample.DATASET_TYPE_SV_CALLS) else dt + for dt in samples_by_dataset_type + } invalid_type = next(( - dts[0] for dts in [dataset_types, secondary_dataset_types] if dts and not any( - dt for dt in dts if (Sample.DATASET_TYPE_SV_CALLS if dt.startswith(Sample.DATASET_TYPE_SV_CALLS) else dt) - in samples_by_dataset_type - )), None) + dts[0] for dts in [dataset_types, secondary_dataset_types] if dts and not sample_dataset_types.intersection(dts) + ), None) if invalid_type: raise InvalidSearchException( f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' From 49fdf6a97b37336737227b6c9026214da614e7b5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 17:46:26 -0400 Subject: [PATCH 192/252] fix args --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 878fe9824e..00d62db624 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -832,7 +832,7 @@ def get_clickhouse_variant_by_id(variant_id, parsed_variant_id, family, dataset_ def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys): - sample_data = _get_sample_data(Family.objects.filter(guid__in=family_guids), [dataset_type])[dataset_type] + sample_data = _get_sample_data(Family.objects.filter(guid__in=family_guids), None)[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( project_guid=project_guid, family_guid__in=family_guids, key__in=keys, ) From cec21dcd36f4047263bc0d7da2191a804257fed1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 17 Mar 2026 18:52:27 -0400 Subject: [PATCH 193/252] codacy --- clickhouse_search/search.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 00d62db624..8b8b48523f 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -425,18 +425,18 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): and transcript.get('spliceregion', {}).get('extended_intronic_splice_region_variant') == minimal_transcript.get('extendedIntronicSpliceRegionVariant')) -def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): +def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples): samples = Sample.objects.filter(individual__family__in=families, is_active=True) if not samples.exists(): if allow_no_samples: - return {} + return None raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}') if dataset_types: samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) if not samples and not allow_no_samples: if allow_no_samples: - return {} + return None raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') mismatch_affected_samples = samples.values('sample_id', 'dataset_type').annotate( @@ -449,6 +449,13 @@ def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), ) + return samples + +def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): + samples = _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples) + if not samples: + return {} + skip_individual_guid = ( skip_multi_project_individual_guid and samples.values('individual__family__project_id').distinct().count() > 1 ) @@ -525,6 +532,15 @@ def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid 'num_families': len(set().union(*data['sample_type_families'].values())), } + _validate_dataset_types( + samples_by_dataset_type, dataset_types, secondary_dataset_types, has_comp_het, genotype_filter, + ) + _add_missing_multi_type_samples(samples, samples_by_dataset_type) + + return samples_by_dataset_type + + +def _validate_dataset_types(samples_by_dataset_type, dataset_types, secondary_dataset_types, has_comp_het, genotype_filter): if not samples_by_dataset_type: raise InvalidSearchException( 'Invalid custom inheritance' if genotype_filter else @@ -544,10 +560,6 @@ def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid f'Unable to search for comp-het pairs with dataset type "{invalid_type}". This may be because inheritance based search is disabled in families with no loaded affected individuals' ) - _add_missing_multi_type_samples(samples, samples_by_dataset_type) - - return samples_by_dataset_type - def _add_missing_multi_type_samples(samples, samples_by_dataset_type): for dataset_type, data in samples_by_dataset_type.items(): From d30860c64c77654bead95ef680154beef4476d7c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 10:18:04 -0400 Subject: [PATCH 194/252] codacy --- clickhouse_search/search.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 8b8b48523f..72877a3620 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -451,21 +451,14 @@ def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_n return samples -def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): - samples = _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples) - if not samples: - return {} +def _get_grouped_samples(samples, skip_multi_project_individual_guid, affected_family_only, annotate_affected_males): skip_individual_guid = ( skip_multi_project_individual_guid and samples.values('individual__family__project_id').distinct().count() > 1 ) - individual_affected_status = (inheritance_filter or {}).get('affected') - genotype_filter = (inheritance_filter or {}).get('genotype') - affected_family_only = inheritance_mode and not individual_affected_status family_array_kwargs = {'distinct': True} if affected_family_only: family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED) - annotations = { 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True), 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs), @@ -480,12 +473,25 @@ def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid )) else: annotations['samples'] = ArrayAgg(JSONObject( - affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), + affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', + family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), )) sample_data = samples.values('dataset_type', 'sample_type').annotate(**annotations) if affected_family_only: sample_data = sample_data.filter(family_guids__len__gt=0) + return sample_data + + +def _get_sample_data(families, dataset_types, skip_multi_project_individual_guid=False, annotate_affected_males=False, allow_no_samples=False, has_comp_het=False, secondary_dataset_types=None, inheritance_mode=None, inheritance_filter=None): + samples = _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_no_samples) + if not samples: + return {} + + individual_affected_status = (inheritance_filter or {}).get('affected') + genotype_filter = (inheritance_filter or {}).get('genotype') + affected_family_only = inheritance_mode and not individual_affected_status + sample_data = _get_grouped_samples(samples, skip_multi_project_individual_guid, affected_family_only, annotate_affected_males) samples_by_dataset_type = {} for data in sample_data: From 4876873dbc9d14061ab73581e0493fb362f25b10 Mon Sep 17 00:00:00 2001 From: hanars Date: Wed, 18 Mar 2026 10:32:49 -0400 Subject: [PATCH 195/252] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- clickhouse_search/search.py | 4 +++- seqr/utils/search/utils.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 72877a3620..29e9a86250 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -434,7 +434,9 @@ def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_n if dataset_types: samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) - if not samples and not allow_no_samples: + # After filtering by dataset type(s), there may be no matching samples even though + # the families have active samples. Honor allow_no_samples in this case. + if not samples.exists(): if allow_no_samples: return None raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0fdc7d6d13..4c32cc0707 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -279,7 +279,7 @@ def _search_dataset_type(search, genome_version): ] if not search.get('exclude_locations') else None dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None - if secondary_dataset_types and len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: + if secondary_dataset_types and dataset_types == [Sample.DATASET_TYPE_SV_CALLS]: secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] if search.get('inheritance_mode') == X_LINKED_RECESSIVE: From 2d366f61c159eb602e1f4b768f18225c76df32ff Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 11:22:29 -0400 Subject: [PATCH 196/252] Revert "Apply suggestions from code review" This reverts commit 4876873dbc9d14061ab73581e0493fb362f25b10. --- clickhouse_search/search.py | 4 +--- seqr/utils/search/utils.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 29e9a86250..72877a3620 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -434,9 +434,7 @@ def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_n if dataset_types: samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) - # After filtering by dataset type(s), there may be no matching samples even though - # the families have active samples. Honor allow_no_samples in this case. - if not samples.exists(): + if not samples and not allow_no_samples: if allow_no_samples: return None raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 4c32cc0707..0fdc7d6d13 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -279,7 +279,7 @@ def _search_dataset_type(search, genome_version): ] if not search.get('exclude_locations') else None dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None - if secondary_dataset_types and dataset_types == [Sample.DATASET_TYPE_SV_CALLS]: + if secondary_dataset_types and len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] if search.get('inheritance_mode') == X_LINKED_RECESSIVE: From 71c16e730250b6d27421b45c028b5a593d45962c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 11:25:19 -0400 Subject: [PATCH 197/252] properly apply code review --- clickhouse_search/search.py | 2 +- seqr/utils/search/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 72877a3620..44a9879b5c 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -434,7 +434,7 @@ def _get_valid_samples(families, dataset_types, secondary_dataset_types, allow_n if dataset_types: samples = samples.filter(dataset_type__in={*dataset_types, *(secondary_dataset_types or [])}) - if not samples and not allow_no_samples: + if not samples.exists(): if allow_no_samples: return None raise InvalidSearchException(f'Unable to search against dataset type "{dataset_types[0]}"') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0fdc7d6d13..2bef3d6f62 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -279,7 +279,7 @@ def _search_dataset_type(search, genome_version): ] if not search.get('exclude_locations') else None dataset_types = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity'), exclude_svs=search.pop('exclude_svs', False)) secondary_dataset_types = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None - if secondary_dataset_types and len(dataset_types) == 0 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: + if secondary_dataset_types and len(dataset_types or []) == 1 and dataset_types[0] == Sample.DATASET_TYPE_SV_CALLS: secondary_dataset_types = [dt for dt in secondary_dataset_types if dt != Sample.DATASET_TYPE_MITO_CALLS] if search.get('inheritance_mode') == X_LINKED_RECESSIVE: From 5d6a6e0f6b50ce1e4617ec33e30be5197860a5c6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 12:16:46 -0400 Subject: [PATCH 198/252] add high splice ai x linked prioritized searches --- .../commands/tag_seqr_prioritized_variants.py | 18 ++++++++++++++++++ .../tag_seqr_prioritized_variants_tests.py | 2 ++ 2 files changed, 20 insertions(+) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 9ef1fca0d6..95f715eff3 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -323,6 +323,24 @@ **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, + 'High Splice AI - X-Linked Recessive': { + 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + **HIGH_SPLICE_AI_SEARCH, + **RECESSIVE_SEARCH_NO_IN_SILICO, + 'family_filter': { + AFFECTED_MALE_FAMILY_FILTER: True, + CONFIRMED_FAMILY_FILTER: False, + }, + }, + 'High Splice AI - X-Linked Recessive Confirmed': { + 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + **CONFIRMED_HIGH_SPLICE_AI_SEARCH, + **RECESSIVE_SEARCH_NO_IN_SILICO, + 'family_filter': { + AFFECTED_MALE_FAMILY_FILTER: True, + CONFIRMED_FAMILY_FILTER: True, + }, + }, 'Recessive': { 'inheritance_mode': HOMOZYGOUS_RECESSIVE, 'annotations': HIGH_MODERATE_ANNOTATIONS, diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 284ed0e963..a4c185da1e 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -29,6 +29,8 @@ 'High Splice AI - De Novo/ Dominant Confirmed': 0, 'High Splice AI - Recessive': 0, 'High Splice AI - Recessive Confirmed': 0, + 'High Splice AI - X-Linked Recessive': 0, + 'High Splice AI - X-Linked Recessive Confirmed': 0, 'Recessive': 1, 'X-Linked Recessive': 0, } From 8b65d13ca73b7b96032abb57eb7011bbe76a374c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 12:24:24 -0400 Subject: [PATCH 199/252] fix family search ui error --- ui/redux/selectors.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index ea1daf9e45..3fe2aace45 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -181,7 +181,7 @@ export const getActiveDatasetsByFamily = createSelector( getFamiliesByGuid, getActiveDatasetsByIndividual, (familiesByGuid, activeDatasetsByIndividual) => Object.entries(familiesByGuid).reduce( - (acc, [familyGuid, { individualGuids }]) => ({ + (acc, [familyGuid, { individualGuids = [] }]) => ({ ...acc, [familyGuid]: individualGuids.reduce( (familyAcc, individualGuid) => [...familyAcc, ...(activeDatasetsByIndividual[individualGuid] || [])], [], From 296f80881ae3b0214dcd4549f21bf787f1f3875b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 13:53:29 -0400 Subject: [PATCH 200/252] separate manual variant creation endpoint --- seqr/urls.py | 2 + seqr/views/apis/saved_variant_api.py | 66 +++++++++++++++++++ .../components/CreateVariantButton.jsx | 4 +- ui/pages/Project/reducers.js | 7 ++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index 8204a9f8f4..4903826080 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -52,6 +52,7 @@ from seqr.views.apis.saved_variant_api import \ saved_variant_data, \ create_saved_variant_handler, \ + create_manual_saved_variant_handler, \ update_variant_tags_handler, \ update_variant_acmg_classification_handler, \ update_variant_functional_data_handler, \ @@ -265,6 +266,7 @@ 'saved_search/(?P[^/]+)/delete': delete_saved_search_handler, 'saved_variant/create': create_saved_variant_handler, + 'saved_variant/create_manual': create_manual_saved_variant_handler, 'saved_variant/(?P[^/]+)/update_tags': update_variant_tags_handler, 'saved_variant/(?P[^/]+)/update_acmg_classification': update_variant_acmg_classification_handler, 'saved_variant/(?P[^/]+)/update_functional_data': update_variant_functional_data_handler, diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index db43d776a4..09bda1c621 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -46,6 +46,72 @@ def saved_variant_data(request, project_guid, variant_guids=None): return create_json_response(response) +""" +familyGuid: "F027408_wal_oth17000" +tags: [{name: "Tier 1 - Novel mode of inheritance"}] +variant: [...] +--- +chrom: "1" +end: 456 +genomeVersion: "38" +genotypes: {I0072323_wal_oth17001a_3028568: {cn: 4}} +pos: 123 +svName: "DEL123" +svType: "INS" +transcripts: {ENSG00000169083: []} +variantId: "DEL123" +--- +alt: "CG" +chrom: "3" +genomeVersion: "38" +genotypes: {I0072323_wal_oth17001a_3028568: {numAlt: 2}} +pos: 123 +ref: "A" +transcripts: {ENSG00000180210: []} +variantId: "3-123-A-CG" +--- +alt: "T" +chrom: "4" +end: 5678 +genomeVersion: "38" +genotypes: {I0072323_wal_oth17001a_3028568: {numAlt: 1}} +mainTranscriptId: "ENST12344" +pos: 11234 +ref: "A" +transcripts: {ENSG00000169083: [{transcriptId: "ENST12344", hgvsc: "1234.c", hgvsp: "123.p"}]} +variantId: "4-11234-A-T" +""" +@login_and_policies_required +def create_manual_saved_variant_handler(request): + variant_json = json.loads(request.body) + family_guid = variant_json['familyGuid'] + + family = Family.objects.get(guid=family_guid) + check_project_permissions(family.project, request.user) + + variants_json = variant_json['variant'] + if not isinstance(variant_json['variant'], list): + variants_json = [variants_json] + + saved_variant_guids = [] + for single_variant_json in variants_json: + try: + create_json, update_json = parse_saved_variant_json(single_variant_json, family.id) + except ValueError as e: + return create_json_response({'error': str(e)}, status=400) + saved_variant, _ = get_or_create_model_from_json( + SavedVariant, create_json=create_json, update_json=update_json, + user=request.user, update_on_create_only=True) + saved_variant_guids.append(saved_variant.guid) + saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) + + response = {} + if variant_json.get('tags'): + _update_tags(saved_variants, variant_json, request.user) + + response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=family.project.genome_version)) + return create_json_response(response) + @login_and_policies_required def create_saved_variant_handler(request): diff --git a/ui/pages/Project/components/CreateVariantButton.jsx b/ui/pages/Project/components/CreateVariantButton.jsx index 9f031fa592..5bdf219fc2 100644 --- a/ui/pages/Project/components/CreateVariantButton.jsx +++ b/ui/pages/Project/components/CreateVariantButton.jsx @@ -3,7 +3,6 @@ import PropTypes from 'prop-types' import { connect } from 'react-redux' import { Grid, Divider, Accordion } from 'semantic-ui-react' -import { updateVariantTags } from 'redux/rootReducer' import { getUser, getSortedIndividualsByFamily } from 'redux/selectors' import UpdateButton from 'shared/components/buttons/UpdateButton' @@ -13,6 +12,7 @@ import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' import { GENOME_VERSION_FIELD, SV_TYPES } from 'shared/utils/constants' import { TAG_FORM_FIELD, TAG_FIELD_NAME } from '../constants' +import { createManualVariant } from '../reducers' import { getTaggedVariantsByFamilyType, getProjectTagTypeOptions, getCurrentProject } from '../selectors' import SelectSavedVariantsTable, { VARIANT_POS_COLUMN, TAG_COLUMN, GENES_COLUMN } from './SelectSavedVariantsTable' @@ -265,7 +265,7 @@ const mapDispatchToProps = (dispatch, ownProps) => ({ variant: [variant, ...variants], } - return dispatch(updateVariantTags(formattedValues)) + return dispatch(createManualVariant(formattedValues)) }, }) diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index 4a406efbc7..78390eac38 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -145,6 +145,13 @@ export const loadFamilySavedVariants = familyGuid => loadSavedVariants({ familyG export const unloadProject = () => dispatch => dispatch({ type: UPDATE_CURRENT_PROJECT, newValue: null }) +export const createManualVariant = values => dispatch => new HttpRequestHelper( + '/api/saved_variant/create_manual', + (responseJson) => { + dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) + }, +).post(values) + export const updateFamilies = values => (dispatch, getState) => { const action = values.delete ? 'delete' : 'edit' return new HttpRequestHelper(`/api/project/${getState().currentProjectGuid}/${action}_families`, From 084b6babb04d2ef6adbf84401fb9865636974b39 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 15:16:22 -0400 Subject: [PATCH 201/252] implement manual vvariant creation from unparsed --- seqr/urls.py | 2 +- seqr/views/apis/saved_variant_api.py | 100 ++++++++---------- seqr/views/utils/variant_utils.py | 1 + .../components/CreateVariantButton.jsx | 47 +------- ui/pages/Project/constants.js | 2 +- ui/pages/Project/reducers.js | 4 +- 6 files changed, 51 insertions(+), 105 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index 4903826080..f212f3c18a 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -266,7 +266,7 @@ 'saved_search/(?P[^/]+)/delete': delete_saved_search_handler, 'saved_variant/create': create_saved_variant_handler, - 'saved_variant/create_manual': create_manual_saved_variant_handler, + 'saved_variant/create_manual/(?P[\w.|-]+)': create_manual_saved_variant_handler, 'saved_variant/(?P[^/]+)/update_tags': update_variant_tags_handler, 'saved_variant/(?P[^/]+)/update_acmg_classification': update_variant_acmg_classification_handler, 'saved_variant/(?P[^/]+)/update_functional_data': update_variant_functional_data_handler, diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 09bda1c621..40fbf3189d 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -6,6 +6,7 @@ from clickhouse_search.search import get_variant_main_transcripts_by_key from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project, Sample +from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -46,70 +47,55 @@ def saved_variant_data(request, project_guid, variant_guids=None): return create_json_response(response) -""" -familyGuid: "F027408_wal_oth17000" -tags: [{name: "Tier 1 - Novel mode of inheritance"}] -variant: [...] ---- -chrom: "1" -end: 456 -genomeVersion: "38" -genotypes: {I0072323_wal_oth17001a_3028568: {cn: 4}} -pos: 123 -svName: "DEL123" -svType: "INS" -transcripts: {ENSG00000169083: []} -variantId: "DEL123" ---- -alt: "CG" -chrom: "3" -genomeVersion: "38" -genotypes: {I0072323_wal_oth17001a_3028568: {numAlt: 2}} -pos: 123 -ref: "A" -transcripts: {ENSG00000180210: []} -variantId: "3-123-A-CG" ---- -alt: "T" -chrom: "4" -end: 5678 -genomeVersion: "38" -genotypes: {I0072323_wal_oth17001a_3028568: {numAlt: 1}} -mainTranscriptId: "ENST12344" -pos: 11234 -ref: "A" -transcripts: {ENSG00000169083: [{transcriptId: "ENST12344", hgvsc: "1234.c", hgvsp: "123.p"}]} -variantId: "4-11234-A-T" -""" -@login_and_policies_required -def create_manual_saved_variant_handler(request): - variant_json = json.loads(request.body) - family_guid = variant_json['familyGuid'] +@login_and_policies_required +def create_manual_saved_variant_handler(request, family_guid): family = Family.objects.get(guid=family_guid) check_project_permissions(family.project, request.user) - variants_json = variant_json['variant'] - if not isinstance(variant_json['variant'], list): - variants_json = [variants_json] + variant_json = json.loads(request.body) + tags = variant_json.pop('tags') + saved_variant_guids = {guid for guid, is_selected in variant_json.pop('variants', {}).items() if is_selected} + + genome_version = family.project.genome_version + try: + xpos = get_xpos(variant_json['chrom'], variant_json['pos']) + except ValueError as e: + return create_json_response({'error': str(e)}, status=400) + + variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" + variant_json.update({ + 'genomeVersion': genome_version, + 'transcripts': {}, + 'variantId': variant_id, + 'xpos': xpos, + }) - saved_variant_guids = [] - for single_variant_json in variants_json: - try: - create_json, update_json = parse_saved_variant_json(single_variant_json, family.id) - except ValueError as e: - return create_json_response({'error': str(e)}, status=400) - saved_variant, _ = get_or_create_model_from_json( - SavedVariant, create_json=create_json, update_json=update_json, - user=request.user, update_on_create_only=True) - saved_variant_guids.append(saved_variant.guid) - saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) + gene_id = variant_json.pop('geneId', None) + if gene_id: + variant_json['transcripts'][gene_id] = [] + if variant_json.get('mainTranscriptId'): + variant_json['transcripts'][gene_id].append({ + 'transcriptId': variant_json['mainTranscriptId'], + 'hgvsc': variant_json.pop('hgvsc'), + 'hgvsp': variant_json.pop('hgvsp'), + }) - response = {} - if variant_json.get('tags'): - _update_tags(saved_variants, variant_json, request.user) + var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else 0 + saved_variant = create_model_from_json(SavedVariant, { + 'family': family, + 'xpos_end': xpos + var_length, + 'variant_id': variant_id, + 'gene_ids': [gene_id] if gene_id else [], + 'saved_variant_json': variant_json, + **{field: variant_json.get(field) for field in ['xpos', 'ref', 'alt']}, + }, request.user) + saved_variant_guids.add(saved_variant.guid) - response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=family.project.genome_version)) + saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) + _update_tags(saved_variants, {'tags': tags}, request.user) + + response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) return create_json_response(response) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index d1a15bad7a..f77d2e4ee3 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -40,6 +40,7 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_t alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 variant_id = variant_json.get('variantId', variant_id) + # TODO clean up if variant_json.get('key'): update_json = { 'key': variant_json['key'], diff --git a/ui/pages/Project/components/CreateVariantButton.jsx b/ui/pages/Project/components/CreateVariantButton.jsx index 5bdf219fc2..87836734b0 100644 --- a/ui/pages/Project/components/CreateVariantButton.jsx +++ b/ui/pages/Project/components/CreateVariantButton.jsx @@ -9,9 +9,9 @@ import UpdateButton from 'shared/components/buttons/UpdateButton' import { Select, IntegerInput, LargeMultiselect } from 'shared/components/form/Inputs' import { validators, configuredField } from 'shared/components/form/FormHelpers' import { AwesomeBarFormInput } from 'shared/components/page/AwesomeBar' -import { GENOME_VERSION_FIELD, SV_TYPES } from 'shared/utils/constants' +import { SV_TYPES } from 'shared/utils/constants' -import { TAG_FORM_FIELD, TAG_FIELD_NAME } from '../constants' +import { TAG_FORM_FIELD } from '../constants' import { createManualVariant } from '../reducers' import { getTaggedVariantsByFamilyType, getProjectTagTypeOptions, getCurrentProject } from '../selectors' import SelectSavedVariantsTable, { VARIANT_POS_COLUMN, TAG_COLUMN, GENES_COLUMN } from './SelectSavedVariantsTable' @@ -25,9 +25,6 @@ const TRANSCRIPT_ID_FIELD_NAME = 'mainTranscriptId' const HGVSC_FIELD_NAME = 'hgvsc' const HGVSP_FIELD_NAME = 'hgvsp' const VARIANTS_FIELD_NAME = 'variants' -const FORMAT_RESPONSE_FIELDS = [ - GENE_ID_FIELD_NAME, HGVSC_FIELD_NAME, HGVSP_FIELD_NAME, TAG_FIELD_NAME, VARIANTS_FIELD_NAME, -] const ZygosityInput = React.memo(({ individuals, name, title, individualField, error }) => (
@@ -88,8 +85,6 @@ const SAVED_VARIANT_COLUMNS = [ ] const SAVED_SV_COLUMNS = [{ name: 'svType', content: 'SV Type', width: 2 }, ...SAVED_VARIANT_COLUMNS] -const GENOME_FIELD = { ...GENOME_VERSION_FIELD, inline: false, width: null } - const ZYGOSITY_FIELD = { name: 'genotypes', component: connect(mapZygosityInputStateToProps)(ZygosityInput), @@ -127,7 +122,6 @@ const GENE_FIELD = { const SAVED_VARIANT_FIELD = { name: VARIANTS_FIELD_NAME, idField: 'variantGuid', - includeSelectedRowData: true, control: SavedVariantField, } @@ -156,7 +150,6 @@ const SNV_FIELDS = [ { name: TRANSCRIPT_ID_FIELD_NAME, label: 'Transcript ID', width: 6 }, { name: HGVSC_FIELD_NAME, label: 'HGVSC', width: 5, validate: validateHasTranscriptId }, { name: HGVSP_FIELD_NAME, label: 'HGVSP', width: 5, validate: validateHasTranscriptId }, - GENOME_FIELD, TAG_FIELD, { accordionLabel: 'Multinucleotide Variant', columns: SAVED_VARIANT_COLUMNS, ...SAVED_VARIANT_FIELD }, { @@ -176,7 +169,6 @@ const SV_FIELDS = [ START_FIELD, END_FIELD, GENE_FIELD, - GENOME_FIELD, TAG_FIELD, { name: SV_FIELD_NAME, validate: validators.required, label: 'SV Name', width: 8 }, { @@ -206,7 +198,6 @@ const BaseCreateVariantButton = React.memo(({ variantType, family, user, project (project.isAnalystProject ? user.isAnalyst : project.canEdit) ? ( ({ }) const mapDispatchToProps = (dispatch, ownProps) => ({ - onSubmit: (values) => { - const variant = ownProps.formFields.map(({ name }) => name).filter( - name => !FORMAT_RESPONSE_FIELDS.includes(name), - ).reduce( - (acc, name) => ({ ...acc, [name]: values[name] }), {}, - ) - - if (variant.svName) { - variant.variantId = values.svName - if (values[GENE_ID_FIELD_NAME]) { - variant.transcripts = { [values[GENE_ID_FIELD_NAME]]: [] } - } - } else { - variant.variantId = `${values.chrom}-${values.pos}-${values.ref}-${values.alt}` - variant.transcripts = { - [values[GENE_ID_FIELD_NAME]]: values[TRANSCRIPT_ID_FIELD_NAME] ? [{ - transcriptId: values[TRANSCRIPT_ID_FIELD_NAME], - [HGVSC_FIELD_NAME]: values[HGVSC_FIELD_NAME], - [HGVSP_FIELD_NAME]: values[HGVSP_FIELD_NAME], - }] : [], - } - } - - const variants = Object.values(values[VARIANTS_FIELD_NAME] || {}).filter(v => v) - - const formattedValues = { - familyGuid: ownProps.family.familyGuid, - tags: values[TAG_FIELD_NAME], - variant: [variant, ...variants], - } - - return dispatch(createManualVariant(formattedValues)) - }, + onSubmit: values => dispatch(createManualVariant(ownProps.family.familyGuid, values)), }) const CreateVariantButton = connect(mapStateToProps, mapDispatchToProps)(BaseCreateVariantButton) diff --git a/ui/pages/Project/constants.js b/ui/pages/Project/constants.js index 22854f020d..be1991a1f7 100644 --- a/ui/pages/Project/constants.js +++ b/ui/pages/Project/constants.js @@ -509,7 +509,7 @@ export const SAMPLE_EXPORT_DATA = [ { header: 'Dataset Type', field: 'datasetType' }, ] -export const TAG_FIELD_NAME = 'tags' +const TAG_FIELD_NAME = 'tags' export const TAG_FORM_FIELD = { name: TAG_FIELD_NAME, diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index 78390eac38..c2181b8dfe 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -145,8 +145,8 @@ export const loadFamilySavedVariants = familyGuid => loadSavedVariants({ familyG export const unloadProject = () => dispatch => dispatch({ type: UPDATE_CURRENT_PROJECT, newValue: null }) -export const createManualVariant = values => dispatch => new HttpRequestHelper( - '/api/saved_variant/create_manual', +export const createManualVariant = (familyGuid, values) => dispatch => new HttpRequestHelper( + `/api/saved_variant/create_manual/${familyGuid}`, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) }, From 44e462e1f12a74580bae0d4b8adb224608424dab Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 15:54:07 -0400 Subject: [PATCH 202/252] clean up parsing json --- seqr/views/apis/saved_variant_api.py | 26 +++++++++++------------ seqr/views/utils/variant_utils.py | 31 ++++++++++------------------ 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 40fbf3189d..2d9ec61a48 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -65,6 +65,7 @@ def create_manual_saved_variant_handler(request, family_guid): variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" variant_json.update({ + 'key': None, 'genomeVersion': genome_version, 'transcripts': {}, 'variantId': variant_id, @@ -81,15 +82,14 @@ def create_manual_saved_variant_handler(request, family_guid): 'hgvsp': variant_json.pop('hgvsp'), }) - var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else 0 - saved_variant = create_model_from_json(SavedVariant, { - 'family': family, - 'xpos_end': xpos + var_length, - 'variant_id': variant_id, - 'gene_ids': [gene_id] if gene_id else [], - 'saved_variant_json': variant_json, - **{field: variant_json.get(field) for field in ['xpos', 'ref', 'alt']}, - }, request.user) + model_json = parse_saved_variant_json( + variant_json, + family.id, + dataset_type=Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, + ) + model_json['saved_variant_json'] = variant_json + + saved_variant = create_model_from_json(SavedVariant, model_json, request.user) saved_variant_guids.add(saved_variant.guid) saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) @@ -113,12 +113,10 @@ def create_saved_variant_handler(request): saved_variant_guids = [] for single_variant_json in variants_json: - try: - create_json, update_json = parse_saved_variant_json(single_variant_json, family.id) - except ValueError as e: - return create_json_response({'error': str(e)}, status=400) + parsed_json = parse_saved_variant_json(single_variant_json, family.id) + create_json = {field: parsed_json[field] for field in ['key', 'dataset_type', 'family_id']} saved_variant, _ = get_or_create_model_from_json( - SavedVariant, create_json=create_json, update_json=update_json, + SavedVariant, create_json=create_json, update_json=parsed_json, user=request.user, update_on_create_only=True) saved_variant_guids.append(saved_variant.guid) saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index f77d2e4ee3..8672afca59 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -33,34 +33,23 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_type=None): - if 'xpos' not in variant_json: - variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos']) xpos = variant_json['xpos'] ref = variant_json.get('ref') - alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 variant_id = variant_json.get('variantId', variant_id) - # TODO clean up - if variant_json.get('key'): - update_json = { - 'key': variant_json['key'], - 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': dataset_type or variant_dataset_type({'variantId': variant_id, **variant_json}), - } - else: - update_json = {'saved_variant_json': variant_json} - if 'transcripts' in variant_json: - update_json['gene_ids'] = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) - elif 'gene_ids' in variant_json: - update_json['gene_ids'] = variant_json['gene_ids'] + gene_ids = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) return { + 'key': variant_json['key'], 'xpos': xpos, 'xpos_end': xpos + var_length, 'ref': ref, - 'alt': alt, + 'alt': variant_json.get('alt'), 'family_id': family_id, 'variant_id': variant_id, - }, update_json + 'genotypes': variant_json.get('genotypes', {}), + 'dataset_type': dataset_type or variant_dataset_type({'variantId': variant_id, **variant_json}), + 'gene_ids': gene_ids, + } def _transcript_sort(gene_id, saved_variant_json): @@ -92,8 +81,10 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): - create_json, update_json = parse_saved_variant_json(variant, family_id, variant_id=variant_id, dataset_type=dataset_type) - new_variant_models.append(SavedVariant(**create_json, **update_json)) + variant_json = parse_saved_variant_json(variant, family_id, variant_id=variant_id, dataset_type=dataset_type) + if not variant.get('key'): + variant_json['saved_variant_json'] = variant + new_variant_models.append(SavedVariant(**variant_json)) saved_variant_map.update({ (v.family_id, getattr(v, primary_id_field)): v for v in SavedVariant.bulk_create(user, new_variant_models) From 3e4fa0c5c66119653ca37aec24daf2ac5a187bf9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 17:02:31 -0400 Subject: [PATCH 203/252] fix genes --- seqr/management/commands/tag_seqr_prioritized_variants.py | 2 +- seqr/views/utils/variant_utils.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 95f715eff3..ecd753ff2b 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -596,7 +596,7 @@ def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_ else: variant_fields += ['familyGenotypes'] - results = results_qs.values(*variant_fields, 'key', 'familyGuids', **variant_values) + results = results_qs.values(*variant_fields, 'key', 'familyGuids', 'xpos', **variant_values) add_individual_guids(results, samples, encode_genotypes_json=True) for variant in results: diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 8672afca59..c6baeda00e 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -37,7 +37,9 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_t ref = variant_json.get('ref') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 variant_id = variant_json.get('variantId', variant_id) - gene_ids = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) + gene_ids = sorted( + variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json) + ) if 'transcripts' in variant_json else variant_json['gene_ids'] return { 'key': variant_json['key'], 'xpos': xpos, From 2a1a728cc31f335027c9eadafe27d2cf6dad9567 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 17:39:33 -0400 Subject: [PATCH 204/252] bulk save gregor variants; --- .../commands/tag_seqr_prioritized_variants.py | 2 +- seqr/views/apis/individual_api.py | 37 ++++++++++------- seqr/views/apis/individual_api_tests.py | 41 +++++++++---------- seqr/views/apis/summary_data_api.py | 2 +- seqr/views/utils/variant_utils.py | 13 +++--- 5 files changed, 51 insertions(+), 44 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index ecd753ff2b..0b58ad65b0 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -580,7 +580,7 @@ def _family_passes_filter(sample_data, family_filter): @staticmethod def _get_metadata(today, metadata_key): - def wrapped(v): + def wrapped(v, **kwargs): return {name: today for name in v[metadata_key]} if v[metadata_key] else None return wrapped diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index df12dce8ee..b74a968bb2 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -12,6 +12,7 @@ from seqr.models import Individual, Family, CAN_VIEW from seqr.utils.file_utils import file_iter from seqr.utils.gene_utils import get_genes, get_gene_ids_for_gene_symbols +from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.anvil_metadata_utils import PARTICIPANT_TABLE, PHENOTYPE_TABLE, EXPERIMENT_TABLE, \ EXPERIMENT_LOOKUP_TABLE, FINDINGS_TABLE, FINDING_METADATA_COLUMNS, TRANSCRIPT_FIELDS, GENE_COLUMN, parse_population from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file, parse_file @@ -822,7 +823,8 @@ def import_gregor_metadata(request, project_guid): family_variant_data = {} finding_id_map = {} - genes = set() + metadata = {} + genes = {} for row in _iter_metadata_table( metadata_files_path, FINDINGS_TABLE, request.user, lambda r: r['participant_id'] in participant_individual_map and r['variant_type'] in {'SNV/INDEL', 'SNV', 'INDEL'}, @@ -830,25 +832,31 @@ def import_gregor_metadata(request, project_guid): individual = participant_individual_map[row['participant_id']] variant_id = '-'.join([row[col] for col in ['chrom', 'pos', 'ref', 'alt']]) key = (individual['family_id'], variant_id) - variant = {k: v for k, v in row.items() if v and v != 'NA'} + row = {k: v for k, v in row.items() if v and v != 'NA'} + variant = {k: row.get(k) for k in ['chrom', 'pos', 'ref', 'alt', 'linked_variant']} + pos = int(variant['pos']) variant.update({ - 'pos': int(variant['pos']), - 'genomeVersion': variant['variant_reference_assembly'].replace('GRCh', ''), + 'key': None, + 'variantId': variant_id, + 'xpos': get_xpos(variant['chrom'], pos), + 'pos': pos, + 'genomeVersion': row['variant_reference_assembly'].replace('GRCh', ''), 'transcripts': {}, 'transcript': { - config.get('seqr_field', k): variant.pop(k, None) for k, config in TRANSCRIPT_FIELDS.items() + config.get('seqr_field', k): row.pop(k, None) for k, config in TRANSCRIPT_FIELDS.items() }, - 'genotypes': {individual['guid']: {'numAlt': 2 if variant['zygosity'] == 'Homozygous' else 1}}, + 'genotypes': {individual['guid']: {'numAlt': 2 if row['zygosity'] == 'Homozygous' else 1}}, 'support_vars': [], }) family_variant_data[key] = variant - genes.add(variant[GENE_COLUMN]) - finding_id_map[variant['genetic_findings_id']] = variant_id + genes[key] = row[GENE_COLUMN] + finding_id_map[row['genetic_findings_id']] = variant_id + metadata[key] = {k: row[k] for k in FINDING_METADATA_COLUMNS if k in row} - gene_symbols_to_ids = {k: v[0] for k, v in get_gene_ids_for_gene_symbols(genes, genome_version=project.genome_version).items()} + gene_symbols_to_ids = {k: v[0] for k, v in get_gene_ids_for_gene_symbols(genes.values(), genome_version=project.genome_version).items()} missing_genes = set() - for variant in family_variant_data.values(): - gene = variant[GENE_COLUMN] + for key, variant in family_variant_data.items(): + gene = genes[key] transcript = variant.pop('transcript') if gene in gene_symbols_to_ids: variant.update({ @@ -857,15 +865,16 @@ def import_gregor_metadata(request, project_guid): }) else: missing_genes.add(gene) - if variant.get('linked_variant') in finding_id_map: - variant['support_vars'].append(finding_id_map[variant['linked_variant']]) + linked_variant = variant.pop('linked_variant', None) + if linked_variant in finding_id_map: + variant['support_vars'].append(finding_id_map[linked_variant]) if missing_genes: warnings.append(f'The following unknown genes were omitted in the findings tags: {", ".join(sorted(missing_genes))}') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( family_variant_data, tag_name=GREGOR_FINDING_TAG_TYPE, user=request.user, project=project, - get_metadata=lambda v: {k: v[k] for k in FINDING_METADATA_COLUMNS if k in v}, + get_metadata=lambda v, key=None: metadata[key], ) loaded_info = f'Loaded {len(new_keys)} new and {len(update_keys)} updated findings tags' if skipped_keys: diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 015b3fd2a2..79abb4281a 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1219,9 +1219,7 @@ def test_import_gregor_metadata(self): saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__name='GREGoR Finding' ).order_by('family_id', 'variant_id').distinct().values( - 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json__genomeVersion', - 'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId', - 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes', 'gene_ids', + 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1229,11 +1227,7 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': 'F000012_12', - 'saved_variant_json__genomeVersion': None, - 'saved_variant_json__transcripts': None, - 'saved_variant_json__genotypes': None, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, + 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, @@ -1245,16 +1239,25 @@ def test_import_gregor_metadata(self): 'variant_id': '1-249045487-A-G', 'xpos': 1249045487, 'family__guid': 'F000012_12', - 'saved_variant_json__genomeVersion': '37', - 'saved_variant_json__transcripts': { - 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], + 'saved_variant_json': { + 'alt': 'G', + 'chrom': '1', + 'genomeVersion': '37', + 'genotypes': {created_individual_guid: {'numAlt': 1}}, + 'key': None, + 'mainTranscriptId': None, + 'pos': 249045487, + 'ref': 'A', + 'variantId': '1-249045487-A-G', + 'xpos': 1249045487, + 'transcripts': { + 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], + }, + 'support_vars': ['1-248367227-TC-T'], }, - 'saved_variant_json__genotypes': {created_individual_guid: {'numAlt': 1}}, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, 'key': None, - 'dataset_type': None, - 'genotypes': {}, + 'dataset_type': 'SNV_INDEL', + 'genotypes': {created_individual_guid: {'numAlt': 1}}, 'gene_ids': ['ENSG00000240361'], }) new_family_genotypes = {new_family_individual_guid: {'numAlt': 2}} @@ -1263,11 +1266,7 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': new_family_guid, - 'saved_variant_json__genomeVersion': None, - 'saved_variant_json__transcripts': None, - 'saved_variant_json__genotypes': None, - 'saved_variant_json__mainTranscriptId': None, - 'saved_variant_json__hgvsc': None, + 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 48ef125034..87efe9c3aa 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -216,7 +216,7 @@ def _load_aip_data(data: dict, user: User): today = datetime.now().strftime('%Y-%m-%d') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( family_variant_data, tag_name=AIP_TAG_TYPE, user=user, load_new_variant_data=True, - get_metadata=lambda pred: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, + get_metadata=lambda pred, **kwargs: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, ) summary_message = f'Loaded {len(new_keys)} new and {len(update_keys)} updated AIP tags for {len(family_id_map)} families' diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index c6baeda00e..7890f43b71 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -32,11 +32,10 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_type=None): +def parse_saved_variant_json(variant_json, family_id, dataset_type=None): xpos = variant_json['xpos'] ref = variant_json.get('ref') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - variant_id = variant_json.get('variantId', variant_id) gene_ids = sorted( variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json) ) if 'transcripts' in variant_json else variant_json['gene_ids'] @@ -47,9 +46,9 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None, dataset_t 'ref': ref, 'alt': variant_json.get('alt'), 'family_id': family_id, - 'variant_id': variant_id, + 'variant_id': variant_json['variantId'], 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': dataset_type or variant_dataset_type({'variantId': variant_id, **variant_json}), + 'dataset_type': dataset_type or variant_dataset_type(variant_json), 'gene_ids': gene_ids, } @@ -82,8 +81,8 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use ) new_variant_models = [] - for (family_id, variant_id), variant in new_variant_data.items(): - variant_json = parse_saved_variant_json(variant, family_id, variant_id=variant_id, dataset_type=dataset_type) + for (family_id, _), variant in new_variant_data.items(): + variant_json = parse_saved_variant_json(variant, family_id, dataset_type=dataset_type) if not variant.get('key'): variant_json['saved_variant_json'] = variant new_variant_models.append(SavedVariant(**variant_json)) @@ -104,7 +103,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_tag_keys = {} skipped_tag_keys = set() for key, variant in sorted(family_variant_data.items()): - metadata = get_metadata(variant) + metadata = get_metadata(variant, key=key) comp_het_metadata = get_comp_het_metadata(variant) if get_comp_het_metadata else None updated_tag = _set_updated_tags( key, metadata, comp_het_metadata, variant.get('support_vars', []), saved_variant_map, existing_tags, tag_type, user, From 11c960d350bb8bff3f226ae141593d70a4b2e090 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 17:48:25 -0400 Subject: [PATCH 205/252] fix summary data bulk --- clickhouse_search/search.py | 6 +++--- .../commands/check_for_new_samples_from_pipeline.py | 2 +- seqr/views/utils/variant_utils.py | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index b999e91926..21792a3219 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -783,15 +783,15 @@ def get_clickhouse_variant_by_id(variant_id, parsed_variant_id, samples, genome_ return None -def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): +def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples, additional_fields=None): sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( project_guid=project_guid, family_guid__in=family_guids, key__in=keys, ) gt_field, gt_expr = entries.genotype_expression(sample_data) return { - key: _clickhouse_genotypes_json(genotypes) for key, genotypes in - entries.annotate(**{gt_field: gt_expr}).values_list('key', 'genotypes') + e['key']: {**e, 'genotypes': _clickhouse_genotypes_json(e['genotypes'])} for e in + entries.annotate(**{gt_field: gt_expr}).values('key', 'genotypes', *(additional_fields or [])) } diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index e98c02d7c4..71b35b5fd0 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -409,7 +409,7 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type) ) for key, genotypes in genotypes_by_key.items(): variant = variant_models_by_key[key] - variant.genotypes = genotypes + variant.genotypes = genotypes['genotypes'] variants.append(variant) logger.info(f'Reloading genotypes for {len(variants)} {dataset_type} variants in family {family_guid}') SavedVariant.bulk_update_models(None, variants, ['genotypes']) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 7890f43b71..ecba1875f6 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -219,11 +219,12 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), - samples, + samples, additional_fields=['xpos'] ) - for key, genotypes in genotype_keys.items(): - variants_by_key[key]['genotypes'].update(genotypes) - variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in genotypes.values()}) + for key, entry_data in genotype_keys.items(): + variants_by_key[key]['xpos'] = entry_data['xpos'] + variants_by_key[key]['genotypes'].update(entry_data['genotypes']) + variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in entry_data['genotypes'].values()}) return list(variants_by_key.values()) From c1e395ea33da487521885dbf899cbb0df03136b6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 18:16:08 -0400 Subject: [PATCH 206/252] test manual variants --- seqr/views/apis/saved_variant_api_tests.py | 90 ++++++++++++---------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index ad51807ef9..62d32d8173 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -7,7 +7,7 @@ from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ - update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, \ + update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, create_manual_saved_variant_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ @@ -436,16 +436,13 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _assert_created_basic_variant(self, saved_variant, variant_json, gene_ids): + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', has_saved_variant_json=False): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) - - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL'): - self._assert_created_basic_variant(saved_variant, variant_json, gene_ids) self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) self.assertEqual(dataset_type, saved_variant.dataset_type) - self.assertDictEqual({}, saved_variant.saved_variant_json) + self.assertDictEqual(variant_json if has_saved_variant_json else {}, saved_variant.saved_variant_json) def test_create_saved_sv_variant(self): # SVs are only supported on build 38 @@ -574,29 +571,24 @@ def test_create_saved_compound_hets(self): saved_variants__guid__contains=new_compound_het_4_guid)]) def test_create_manual_variant(self): - create_saved_variant_url = reverse(create_saved_variant_handler) - self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) + create_saved_variant_url = reverse(create_manual_saved_variant_handler, args=['F000001_1']) + self.check_collaborator_login(create_saved_variant_url) manual_variant_request_body = { - 'familyGuid': 'F000001_1', 'tags': [{'name': 'Review'}], - 'variant': [{ - 'alt': 'A', - 'ref': 'GAG', - 'chrom': '7', - 'pos': 53417465, - 'genomeVersion': '37', - 'variantId': '7-53417465-GAG-A', - 'mainTranscriptId': 'ENST00000459627', - 'transcripts': { - 'ENSG00000277258': [{'transcriptId': 'ENST00000459627', 'hgvsc':'c.156GAG>A', 'hgvsp': 'p.Leu52Phe'}], - }, - 'genotypes': { - 'I000001_na19675': {'numAlt': 1}, - 'I000002_na19678': {'numAlt': 0}, - 'I000003_na19679': {'numAlt': 0}, - }, - }], + 'alt': 'A', + 'ref': 'GAG', + 'chrom': '7', + 'pos': 53417465, + 'mainTranscriptId': 'ENST00000459627', + 'geneId': 'ENSG00000277258', + 'hgvsc': 'c.156GAG>A', + 'hgvsp': 'p.Leu52Phe', + 'genotypes': { + 'I000001_na19675': {'numAlt': 1}, + 'I000002_na19678': {'numAlt': 0}, + 'I000003_na19679': {'numAlt': 0}, + }, } response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) self.assertEqual(response.status_code, 200) @@ -605,11 +597,16 @@ def test_create_manual_variant(self): variant_guid = next(iter(response.json()['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - variant_json = {'xpos': 7053417465} - variant_json.update(manual_variant_request_body['variant'][0]) - self._assert_created_basic_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258']) + variant_json = { + 'xpos': 7053417465, + 'key': None, + 'variantId': '7-53417465-GAG-A', + 'genomeVersion': '37', + 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, + **{k: v for k, v in manual_variant_request_body.items() if k in {'alt', 'ref', 'chrom', 'pos', 'genotypes', 'mainTranscriptId'}}, + } + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True) self.assertDictEqual(variant_json, saved_variant.saved_variant_json) - self.assertDictEqual({}, saved_variant.genotypes) base_variant_json = { 'variantGuid': variant_guid, @@ -627,36 +624,42 @@ def test_create_manual_variant(self): self.assertEqual(len(tag_guids), 1) tag = response_json['variantTagsByGuid'][tag_guids[0]] self.assertEqual('Review', tag['name']) + self.assertListEqual([variant_guid], tag['variantGuids']) self.assertDictEqual(variant_json, response_variant_json) # Create manual SV - manual_variant_request_body['variant'] = [{ + manual_variant_request_body = { 'chrom': '7', 'pos': 10417443, 'end': 11719832, - 'genomeVersion': '37', - 'variantId': 'Deletion:chr7:(?_10417443)_(11719832_?)', 'svName': 'Deletion:chr7:(?_10417443)_(11719832_?)', 'svType': 'DEL', - 'transcripts': {'ENSG00000240361': []}, + 'geneId': 'ENSG00000240361', 'genotypes': { 'I000001_na19675': {'cn': 1}, 'I000002_na19678': {'cn': 2}, 'I000003_na19679': {'cn': 2}, }, - }] + 'tags': manual_variant_request_body['tags'], + 'variants': {'SV0059956_11560662_f019313_1': True, 'SV0000002_1248367227_r0390_100': False}, + } response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) - sv_variant_guid = next(iter(response.json()['savedVariantsByGuid'])) + self.assertEqual(len(response.json()['savedVariantsByGuid']), 2) + self.assertTrue('SV0059956_11560662_f019313_1' in response.json()['savedVariantsByGuid']) + sv_variant_guid = next(guid for guid in response.json()['savedVariantsByGuid'] if guid != 'SV0059956_11560662_f019313_1') saved_sv_variant = SavedVariant.objects.get(guid=sv_variant_guid, family__guid='F000001_1') - sv_variant_json = {'xpos': 7010417443} - sv_variant_json.update(manual_variant_request_body['variant'][0]) - self._assert_created_basic_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361']) - self.assertDictEqual(sv_variant_json, saved_sv_variant.saved_variant_json) - self.assertDictEqual({}, saved_sv_variant.genotypes) + sv_variant_json = { + 'key': None, + 'xpos': 7010417443, + 'genomeVersion': '37', + 'variantId': 'Deletion:chr7:(?_10417443)_(11719832_?)', + 'transcripts': {'ENSG00000240361': []}, + **{k: v for k, v in manual_variant_request_body.items() if k in {'chrom', 'pos', 'end', 'svName', 'svType', 'genotypes'}}, + } + self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', has_saved_variant_json=True) sv_variant_json.update(base_variant_json) sv_variant_json.update({ @@ -668,6 +671,9 @@ def test_create_manual_variant(self): response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][sv_variant_guid] self.assertDictEqual(sv_variant_json, response_variant_json) + tag = response_json['variantTagsByGuid'][response_variant_json['tagGuids'][0]] + self.assertEqual('Review', tag['name']) + self.assertSetEqual({sv_variant_guid, 'SV0059956_11560662_f019313_1'}, set(tag['variantGuids'])) def test_create_update_and_delete_variant_note(self): create_variant_note_url = reverse(create_variant_note_handler, args=[VARIANT_GUID]) From e7f00ebd80d0d832d838afb1d9b86ba55c92d3e7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 18:26:06 -0400 Subject: [PATCH 207/252] fix test --- seqr/views/apis/saved_variant_api.py | 2 +- seqr/views/apis/saved_variant_api_tests.py | 33 +++++++++------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 2d9ec61a48..593b644040 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -114,7 +114,7 @@ def create_saved_variant_handler(request): saved_variant_guids = [] for single_variant_json in variants_json: parsed_json = parse_saved_variant_json(single_variant_json, family.id) - create_json = {field: parsed_json[field] for field in ['key', 'dataset_type', 'family_id']} + create_json = {field: parsed_json[field] for field in ['key', 'xpos', 'variant_id', 'family_id']} saved_variant, _ = get_or_create_model_from_json( SavedVariant, create_json=create_json, update_json=parsed_json, user=request.user, update_on_create_only=True) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 62d32d8173..80109e974d 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -99,10 +99,11 @@ 'topmed': {'ac': 0, 'af': 0.0, 'an': 0, 'het': 0, 'hom': 0}, }, 'pos': 61413835, + 'xpos': 2061413835, 'liftedOverGenomeVersion': '38', 'liftedOverChrom': None, 'liftedOverPos': None, - 'predictions': {'cadd': 21.9, + 'predictions': {'cadd': None, 'eigen': None, 'fathmm': None, 'mpc': None, @@ -380,36 +381,25 @@ def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) - create_variant_json = CREATE_VARIANT_JSON create_variant_request_body = { 'searchHash': 'd380ed0fd28c3127d07a64ea2ba907d7', 'familyGuid': 'F000001_1', 'tags': [{'name': 'Review', 'metadata': 'a note'}], 'note': '', 'functionalData': [], - 'variant': create_variant_json, + 'variant': CREATE_VARIANT_JSON, } - - invalide_create_variant_request_body = deepcopy(create_variant_request_body) - invalide_create_variant_request_body['variant']['chrom'] = '27' - response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - invalide_create_variant_request_body)) - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'error': 'Invalid chromosome: 27'}) - - response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps( - create_variant_request_body)) + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(create_variant_request_body)) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) variant_guid = next(iter(response.json()['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - variant_json = {'xpos': 2061413835} - variant_json.update(create_variant_json) - self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258', 'ENSG00000177000']) + self._assert_created_variant(saved_variant, CREATE_VARIANT_JSON, gene_ids=['ENSG00000277258', 'ENSG00000177000']) - variant_json.update({ + variant_json = { + **CREATE_VARIANT_JSON, 'variantGuid': variant_guid, 'familyGuids': ['F000001_1'], 'acmgClassification': None, @@ -418,7 +408,7 @@ def test_create_saved_variant(self): 'functionalDataGuids': [], 'transcripts': mock.ANY, 'mainTranscriptId': mock.ANY, - }) + } response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][variant_guid] tag_guids = response_variant_json.pop('tagGuids') @@ -578,7 +568,7 @@ def test_create_manual_variant(self): 'tags': [{'name': 'Review'}], 'alt': 'A', 'ref': 'GAG', - 'chrom': '7', + 'chrom': '27', 'pos': 53417465, 'mainTranscriptId': 'ENST00000459627', 'geneId': 'ENSG00000277258', @@ -591,6 +581,11 @@ def test_create_manual_variant(self): }, } response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'error': 'Invalid chromosome: 27'}) + + manual_variant_request_body['chrom'] = '7' + response = self.client.post(create_saved_variant_url, content_type='application/json', data=json.dumps(manual_variant_request_body)) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()['savedVariantsByGuid']), 1) From 31ade9191c365ffe5dea1f6444358acddded65ea Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 18 Mar 2026 18:39:07 -0400 Subject: [PATCH 208/252] fix tests --- seqr/views/apis/saved_variant_api_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 80109e974d..959d715038 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -103,7 +103,7 @@ 'liftedOverGenomeVersion': '38', 'liftedOverChrom': None, 'liftedOverPos': None, - 'predictions': {'cadd': None, + 'predictions': {'cadd': 21.9, 'eigen': None, 'fathmm': None, 'mpc': None, @@ -460,6 +460,7 @@ def test_create_saved_sv_variant(self): 'svType': 'DUP', 'variantId': 'batch_123_DUP', 'acmgClassification': None, + 'xpos': 2061413835, } request_body = { @@ -480,7 +481,6 @@ def test_create_saved_sv_variant(self): variant_guid = next(iter(response_json['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - variant_json.update({'xpos': 2061413835}) self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', gene_ids=['ENSG00000240361']) self.assertEqual(saved_variant.xpos_end, 2061414175) @@ -763,7 +763,7 @@ def test_create_partially_saved_compound_het_variant_note(self): request_body = { 'variant': [COMPOUND_HET_4_JSON, { 'variantId': '21-3343353-GAGA-G', 'xpos': 21003343353, 'ref': 'GAGA', 'alt': 'G', - 'variantGuid': 'SV0000001_2103343353_r0390_100', + 'variantGuid': 'SV0000001_2103343353_r0390_100', 'transcripts': {}, 'key': None, 'tagGuids': ['VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'], 'noteGuids': []}, ], 'note': 'one_saved_one_not_saved_compount_hets_note', @@ -1155,7 +1155,7 @@ def test_saved_variant_data(self, *args): def test_create_saved_variant(self): super(AnvilSavedVariantAPITest, self).test_create_saved_variant() - assert_no_list_ws_has_al(self, 4) + assert_no_list_ws_has_al(self, 3) def test_create_saved_sv_variant(self): super(AnvilSavedVariantAPITest, self).test_create_saved_sv_variant() From 8613f296bc258fba4998b8b39206cfc51c40437b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 10:09:56 -0400 Subject: [PATCH 209/252] move search variant logic to tighter scope --- seqr/views/apis/saved_variant_api.py | 13 ++- seqr/views/apis/saved_variant_api_tests.py | 2 - seqr/views/apis/summary_data_api.py | 65 ++++++++++++- seqr/views/utils/variant_utils.py | 103 +++++---------------- 4 files changed, 91 insertions(+), 92 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 593b644040..ea0de49a0a 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -65,7 +65,6 @@ def create_manual_saved_variant_handler(request, family_guid): variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" variant_json.update({ - 'key': None, 'genomeVersion': genome_version, 'transcripts': {}, 'variantId': variant_id, @@ -82,13 +81,13 @@ def create_manual_saved_variant_handler(request, family_guid): 'hgvsp': variant_json.pop('hgvsp'), }) - model_json = parse_saved_variant_json( - variant_json, - family.id, - dataset_type=Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, - ) - model_json['saved_variant_json'] = variant_json + variant_json['saved_variant_json'] = {**variant_json} + variant_json.update({ + 'key': None, + 'dataset_type': Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, + }) + model_json = parse_saved_variant_json(variant_json, family.id) saved_variant = create_model_from_json(SavedVariant, model_json, request.user) saved_variant_guids.add(saved_variant.guid) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 959d715038..a11c495417 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -594,7 +594,6 @@ def test_create_manual_variant(self): saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') variant_json = { 'xpos': 7053417465, - 'key': None, 'variantId': '7-53417465-GAG-A', 'genomeVersion': '37', 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, @@ -647,7 +646,6 @@ def test_create_manual_variant(self): saved_sv_variant = SavedVariant.objects.get(guid=sv_variant_guid, family__guid='F000001_1') sv_variant_json = { - 'key': None, 'xpos': 7010417443, 'genomeVersion': '37', 'variantId': 'Deletion:chr7:(?_10417443)_(11719832_?)', diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 87efe9c3aa..488169b04f 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -7,6 +7,7 @@ from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf import json +from clickhouse_search.search import get_clickhouse_genotypes from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission @@ -25,7 +26,7 @@ from seqr.views.utils.permissions_utils import analyst_required, user_is_analyst, get_project_guids_user_can_view, \ login_and_policies_required, get_project_and_check_permissions, get_internal_projects from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, DISCOVERY_ROW_TYPE -from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, DISCOVERY_CATEGORY +from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, get_saved_variant_annotations, DISCOVERY_CATEGORY from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL logger = SeqrLogger(__name__) @@ -215,7 +216,7 @@ def _load_aip_data(data: dict, user: User): today = datetime.now().strftime('%Y-%m-%d') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( - family_variant_data, tag_name=AIP_TAG_TYPE, user=user, load_new_variant_data=True, + family_variant_data, tag_name=AIP_TAG_TYPE, user=user, parse_new_saved_variants=_search_new_saved_variants, get_metadata=lambda pred, **kwargs: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, ) @@ -232,6 +233,66 @@ def _load_aip_data(data: dict, user: User): }) +def _search_new_saved_variants(family_variant_ids): + family_ids = set() + variant_families = defaultdict(list) + for family_id, variant_id in family_variant_ids: + family_ids.add(family_id) + variant_families[variant_id].append(family_id) + families_by_id = { + f['id']: f for f in Family.objects.filter(id__in=family_ids).values( + 'id', 'guid', 'family_id', 'project__guid', 'project__genome_version', + ) + } + + genome_version = next(family['project__genome_version'] for family in families_by_id.values()) + samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + search_variants_by_id = { + v['variantId']: v for v in _get_clickhouse_variants(samples, families_by_id, family_variant_ids, genome_version) + } + + new_variants = {} + missing = defaultdict(list) + for variant_id, family_ids in variant_families.items(): + variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} + for family_id in family_ids: + family = families_by_id[family_id] + if family['guid'] in variant['familyGuids']: + new_variants[(family_id, variant_id)] = variant + else: + missing[family['family_id']].append(variant_id) + + if missing: + missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] + raise ErrorsWarningsException([ + f"Unable to find the following family's variants in the search backend: {', '.join(missing_summary)}", + ]) + + return new_variants + + +def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, dict], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: + variant_data = get_saved_variant_annotations( + {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, + ) + variants_by_key = {variant['key']: variant for variant in variant_data.values() if variant.get('key')} + + families_by_project = defaultdict(list) + for family in families_by_id.values(): + families_by_project[family['project__guid']].append(family['guid']) + for project_guid, family_guids in families_by_project.items(): + genotype_keys = get_clickhouse_genotypes( + project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), + samples, additional_fields=['xpos'] + ) + for key, entry_data in genotype_keys.items(): + variants_by_key[key]['xpos'] = entry_data['xpos'] + variants_by_key[key]['genotypes'].update(entry_data['genotypes']) + variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in entry_data['genotypes'].values()}) + + return list(variants_by_key.values()) + + ALL_PROJECTS = 'all' GREGOR_CATEGORY = 'gregor' diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index ecba1875f6..a5e7025491 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -2,21 +2,19 @@ from clickhouse_backend.models import ArrayField, StringField from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import F, Q, Count, prefetch_related_objects +from django.db.models import F, Q, Count import json import logging import redis from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap -from clickhouse_search.search import get_variants_queryset, get_clickhouse_genotypes +from clickhouse_search.search import get_variants_queryset from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants -from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ @@ -32,7 +30,7 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def parse_saved_variant_json(variant_json, family_id, dataset_type=None): +def parse_saved_variant_json(variant_json, family_id): xpos = variant_json['xpos'] ref = variant_json.get('ref') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 @@ -48,8 +46,9 @@ def parse_saved_variant_json(variant_json, family_id, dataset_type=None): 'family_id': family_id, 'variant_id': variant_json['variantId'], 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': dataset_type or variant_dataset_type(variant_json), + 'dataset_type': variant_json.get('dataset_type') or variant_dataset_type(variant_json), 'gene_ids': gene_ids, + 'saved_variant_json': variant_json.get('saved_variant_json', {}), } @@ -60,7 +59,7 @@ def _transcript_sort(gene_id, saved_variant_json): return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) -def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, get_comp_het_metadata=None, load_new_variant_data=False, remove_missing_metadata=True, primary_id_field='variant_id', dataset_type=None, **kwargs): +def bulk_create_tagged_variants(family_variant_data, tag_name, parse_new_saved_variants, get_metadata, user, project=None, get_comp_het_metadata=None, remove_missing_metadata=True, primary_id_field='variant_id'): all_family_ids = {family_id for family_id, _ in family_variant_data.keys()} all_variant_ids = {variant_id for _, variant_id in family_variant_data.keys()} @@ -71,22 +70,21 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: - if load_new_variant_data: - genome_version = Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() - new_variant_data = _search_new_saved_variants(new_variant_keys, genome_version) - else: - new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} - new_variant_data = _get_clickhouse_variant_annotations( - new_variant_data, primary_id_field, project=project, dataset_type=dataset_type, **kwargs, - ) - - new_variant_models = [] - for (family_id, _), variant in new_variant_data.items(): - variant_json = parse_saved_variant_json(variant, family_id, dataset_type=dataset_type) - if not variant.get('key'): - variant_json['saved_variant_json'] = variant - new_variant_models.append(SavedVariant(**variant_json)) - + new_variant_data = parse_new_saved_variants(new_variant_keys) + # TODO + # if load_new_variant_data: + # genome_version = Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() + # new_variant_data = _search_new_saved_variants(new_variant_keys, genome_version) + # else: + # new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} + # new_variant_data = _get_clickhouse_variant_annotations( + # new_variant_data, primary_id_field, project=project, dataset_type=dataset_type, **kwargs, + # ) + + new_variant_models = [ + SavedVariant(**parse_saved_variant_json(variant, family_id)) + for (family_id, _), variant in new_variant_data.items() + ] saved_variant_map.update({ (v.family_id, getattr(v, primary_id_field)): v for v in SavedVariant.bulk_create(user, new_variant_models) }) @@ -172,64 +170,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ return updated_tag -def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], genome_version: str) -> dict[tuple[int, str], dict]: - family_ids = set() - variant_families = defaultdict(list) - for family_id, variant_id in family_variant_ids: - family_ids.add(family_id) - variant_families[variant_id].append(family_id) - families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} - - samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - search_variants_by_id = { - v['variantId']: v for v in _get_clickhouse_variants(samples, families_by_id, family_variant_ids, genome_version) - } - - new_variants = {} - missing = defaultdict(list) - for variant_id, family_ids in variant_families.items(): - variant = search_variants_by_id.get(variant_id) or {'familyGuids': []} - for family_id in family_ids: - family = families_by_id[family_id] - if family.guid in variant['familyGuids']: - new_variants[(family_id, variant_id)] = variant - else: - missing[family.family_id].append(variant_id) - - if missing: - missing_summary = [f'{family} ({", ".join(sorted(variant_ids))})' for family, variant_ids in missing.items()] - raise ErrorsWarningsException([ - f"Unable to find the following family's variants in the search backend: {', '.join(missing_summary)}", - ]) - - return new_variants - - -def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: - variant_data = _get_clickhouse_variant_annotations( - {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, - ) - variants_by_key = {variant['key']: variant for variant in variant_data.values() if variant.get('key')} - - families = list(families_by_id.values()) - prefetch_related_objects(families, 'project') - families_by_project = defaultdict(list) - for family in families: - families_by_project[family.project.guid].append(family.guid) - for project_guid, family_guids in families_by_project.items(): - genotype_keys = get_clickhouse_genotypes( - project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), - samples, additional_fields=['xpos'] - ) - for key, entry_data in genotype_keys.items(): - variants_by_key[key]['xpos'] = entry_data['xpos'] - variants_by_key[key]['genotypes'].update(entry_data['genotypes']) - variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in entry_data['genotypes'].values()}) - - return list(variants_by_key.values()) - - -def _get_clickhouse_variant_annotations(variant_data: dict[tuple[int, str], dict], primary_id_field: str = 'variant_id', genome_version: str = None, project: Project = None, dataset_type: str = None) -> dict[tuple[int, str], dict]: +def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], primary_id_field: str = 'variant_id', genome_version: str = None, project: Project = None, dataset_type: str = None) -> dict[tuple[int, str], dict]: dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS variant_ids = { variant_id for (_, variant_id), variant in variant_data.items() From bb96d70a3b323e51031515a66ba1c73bb1380a76 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 10:34:55 -0400 Subject: [PATCH 210/252] explicitly se saved variant json for parsing --- .../commands/tag_seqr_prioritized_variants.py | 15 ++++-- seqr/views/apis/individual_api.py | 47 +++++++++++-------- seqr/views/apis/individual_api_tests.py | 2 - seqr/views/apis/summary_data_api.py | 4 +- seqr/views/utils/variant_utils.py | 19 ++------ 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 0b58ad65b0..0f2a52edfe 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -15,7 +15,7 @@ from seqr.utils.search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO from seqr.utils.search.utils import get_search_samples, COMPOUND_HET from seqr.views.utils.orm_to_json_utils import SEQR_TAG_TYPE -from seqr.views.utils.variant_utils import bulk_create_tagged_variants +from seqr.views.utils.variant_utils import bulk_create_tagged_variants, get_saved_variant_annotations from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL import logging @@ -580,10 +580,19 @@ def _family_passes_filter(sample_data, family_filter): @staticmethod def _get_metadata(today, metadata_key): - def wrapped(v, **kwargs): + def wrapped(v): return {name: today for name in v[metadata_key]} if v[metadata_key] else None return wrapped + @staticmethod + def _parse_new_saved_variants(dataset_type): + def wrapped(new_variant_keys, family_variant_data): + return get_saved_variant_annotations( + {k: v for k, v in family_variant_data.items() if k in new_variant_keys}, + dataset_type=dataset_type, genome_version=GENOME_VERSION_GRCh38, primary_id_field='key', + ) + return wrapped + @classmethod def _run_search(cls, search_name, family_variant_data, family_guid_map, dataset_type, sample_data, samples, **kwargs): results_qs = get_search_queryset(GENOME_VERSION_GRCh38, dataset_type, sample_data, **kwargs) @@ -667,7 +676,7 @@ def _bulk_tag_variants(cls, family_variant_data, updates, dataset_type=None): new_tag_keys, update_tag_keys, skipped_tag_keys = bulk_create_tagged_variants( family_variant_data, tag_name=SEQR_TAG_TYPE, get_metadata=cls._get_metadata(today, 'matched_searches'), get_comp_het_metadata=cls._get_metadata(today, 'matched_comp_het_searches'), user=None, - remove_missing_metadata=False, primary_id_field='key', dataset_type=dataset_type, genome_version=GENOME_VERSION_GRCh38, + remove_missing_metadata=False, primary_id_field='key', parse_new_saved_variants=cls._parse_new_saved_variants(dataset_type), ) updates['new_tag_keys'].update(new_tag_keys) updates['update_tag_keys'].update(update_tag_keys - updates['new_tag_keys']) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index b74a968bb2..6458b4a2b6 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -28,7 +28,7 @@ pm_or_data_manager_required, check_workspace_perm from seqr.views.utils.project_context_utils import add_project_tag_type_counts from seqr.views.utils.individual_utils import delete_individuals, add_or_update_individuals_and_families -from seqr.views.utils.variant_utils import bulk_create_tagged_variants +from seqr.views.utils.variant_utils import bulk_create_tagged_variants, get_saved_variant_annotations @login_and_policies_required @@ -823,8 +823,7 @@ def import_gregor_metadata(request, project_guid): family_variant_data = {} finding_id_map = {} - metadata = {} - genes = {} + genes = set() for row in _iter_metadata_table( metadata_files_path, FINDINGS_TABLE, request.user, lambda r: r['participant_id'] in participant_individual_map and r['variant_type'] in {'SNV/INDEL', 'SNV', 'INDEL'}, @@ -832,31 +831,28 @@ def import_gregor_metadata(request, project_guid): individual = participant_individual_map[row['participant_id']] variant_id = '-'.join([row[col] for col in ['chrom', 'pos', 'ref', 'alt']]) key = (individual['family_id'], variant_id) - row = {k: v for k, v in row.items() if v and v != 'NA'} - variant = {k: row.get(k) for k in ['chrom', 'pos', 'ref', 'alt', 'linked_variant']} + variant = {k: v for k, v in row.items() if v and v != 'NA'} pos = int(variant['pos']) variant.update({ - 'key': None, 'variantId': variant_id, 'xpos': get_xpos(variant['chrom'], pos), 'pos': pos, - 'genomeVersion': row['variant_reference_assembly'].replace('GRCh', ''), + 'genomeVersion': variant['variant_reference_assembly'].replace('GRCh', ''), 'transcripts': {}, 'transcript': { - config.get('seqr_field', k): row.pop(k, None) for k, config in TRANSCRIPT_FIELDS.items() + config.get('seqr_field', k): variant.pop(k, None) for k, config in TRANSCRIPT_FIELDS.items() }, - 'genotypes': {individual['guid']: {'numAlt': 2 if row['zygosity'] == 'Homozygous' else 1}}, + 'genotypes': {individual['guid']: {'numAlt': 2 if variant['zygosity'] == 'Homozygous' else 1}}, 'support_vars': [], }) family_variant_data[key] = variant - genes[key] = row[GENE_COLUMN] - finding_id_map[row['genetic_findings_id']] = variant_id - metadata[key] = {k: row[k] for k in FINDING_METADATA_COLUMNS if k in row} + genes.add(variant[GENE_COLUMN]) + finding_id_map[variant['genetic_findings_id']] = variant_id - gene_symbols_to_ids = {k: v[0] for k, v in get_gene_ids_for_gene_symbols(genes.values(), genome_version=project.genome_version).items()} + gene_symbols_to_ids = {k: v[0] for k, v in get_gene_ids_for_gene_symbols(genes, genome_version=project.genome_version).items()} missing_genes = set() - for key, variant in family_variant_data.items(): - gene = genes[key] + for variant in family_variant_data.values(): + gene = variant[GENE_COLUMN] transcript = variant.pop('transcript') if gene in gene_symbols_to_ids: variant.update({ @@ -865,16 +861,15 @@ def import_gregor_metadata(request, project_guid): }) else: missing_genes.add(gene) - linked_variant = variant.pop('linked_variant', None) - if linked_variant in finding_id_map: - variant['support_vars'].append(finding_id_map[linked_variant]) + if variant.get('linked_variant') in finding_id_map: + variant['support_vars'].append(finding_id_map[variant['linked_variant']]) if missing_genes: warnings.append(f'The following unknown genes were omitted in the findings tags: {", ".join(sorted(missing_genes))}') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( family_variant_data, tag_name=GREGOR_FINDING_TAG_TYPE, user=request.user, project=project, - get_metadata=lambda v, key=None: metadata[key], + get_metadata=lambda v: {k: v[k] for k in FINDING_METADATA_COLUMNS if k in v}, parse_new_saved_variants=_parse_new_aip_saved_variants, ) loaded_info = f'Loaded {len(new_keys)} new and {len(update_keys)} updated findings tags' if skipped_keys: @@ -887,6 +882,20 @@ def import_gregor_metadata(request, project_guid): return create_json_response(response_json) +def _parse_new_aip_saved_variants(new_variant_keys, family_variant_data): + family_id = next(family_id for family_id, _ in new_variant_keys) + genome_version = Family.objects.filter(id=family_id).values_list('project__genome_version', flat=True).first() + new_variant_data = get_saved_variant_annotations( + {k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version, + ) + for variant in new_variant_data.values(): + if not variant.get('key'): + variant.update({'key': None, 'saved_variant_json': {k: v for k, v in variant.items() if k in { + 'chrom', 'pos', 'ref', 'alt', 'variantId', 'xpos', 'genomeVersion', 'genotypes', 'transcripts', 'mainTranscriptId', + }}}) + return new_variant_data + + def _iter_metadata_table(file_path, table_name, user, filter_row): file_name = f'{file_path}/{table_name}.tsv' file_rows = parse_file(file_name, file_iter(file_name, user=user, no_project=True), iter_file=True) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 79abb4281a..c0b9e18fa3 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1244,7 +1244,6 @@ def test_import_gregor_metadata(self): 'chrom': '1', 'genomeVersion': '37', 'genotypes': {created_individual_guid: {'numAlt': 1}}, - 'key': None, 'mainTranscriptId': None, 'pos': 249045487, 'ref': 'A', @@ -1253,7 +1252,6 @@ def test_import_gregor_metadata(self): 'transcripts': { 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], }, - 'support_vars': ['1-248367227-TC-T'], }, 'key': None, 'dataset_type': 'SNV_INDEL', diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 488169b04f..e88ed4c73f 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -217,7 +217,7 @@ def _load_aip_data(data: dict, user: User): today = datetime.now().strftime('%Y-%m-%d') new_keys, update_keys, skipped_keys = bulk_create_tagged_variants( family_variant_data, tag_name=AIP_TAG_TYPE, user=user, parse_new_saved_variants=_search_new_saved_variants, - get_metadata=lambda pred, **kwargs: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, + get_metadata=lambda pred: {category: {'name': category_map[category], 'date': today} for category in pred['categories']}, ) summary_message = f'Loaded {len(new_keys)} new and {len(update_keys)} updated AIP tags for {len(family_id_map)} families' @@ -233,7 +233,7 @@ def _load_aip_data(data: dict, user: User): }) -def _search_new_saved_variants(family_variant_ids): +def _search_new_saved_variants(family_variant_ids, *args, **kwargs): family_ids = set() variant_families = defaultdict(list) for family_id, variant_id in family_variant_ids: diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a5e7025491..8fcb24ea70 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -70,17 +70,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, parse_new_saved_v new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: - new_variant_data = parse_new_saved_variants(new_variant_keys) - # TODO - # if load_new_variant_data: - # genome_version = Family.objects.filter(id__in=all_family_ids).values_list('project__genome_version', flat=True).first() - # new_variant_data = _search_new_saved_variants(new_variant_keys, genome_version) - # else: - # new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} - # new_variant_data = _get_clickhouse_variant_annotations( - # new_variant_data, primary_id_field, project=project, dataset_type=dataset_type, **kwargs, - # ) - + new_variant_data = parse_new_saved_variants(new_variant_keys, family_variant_data) new_variant_models = [ SavedVariant(**parse_saved_variant_json(variant, family_id)) for (family_id, _), variant in new_variant_data.items() @@ -101,7 +91,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, parse_new_saved_v new_tag_keys = {} skipped_tag_keys = set() for key, variant in sorted(family_variant_data.items()): - metadata = get_metadata(variant, key=key) + metadata = get_metadata(variant) comp_het_metadata = get_comp_het_metadata(variant) if get_comp_het_metadata else None updated_tag = _set_updated_tags( key, metadata, comp_het_metadata, variant.get('support_vars', []), saved_variant_map, existing_tags, tag_type, user, @@ -170,7 +160,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ return updated_tag -def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], primary_id_field: str = 'variant_id', genome_version: str = None, project: Project = None, dataset_type: str = None) -> dict[tuple[int, str], dict]: +def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], genome_version: str, primary_id_field: str = 'variant_id', dataset_type: str = None) -> dict[tuple[int, str], dict]: dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS variant_ids = { variant_id for (_, variant_id), variant in variant_data.items() @@ -181,7 +171,7 @@ def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], pri keys = variant_ids variant_ids = None qs = get_variants_queryset( - genome_version or project.genome_version, dataset_type, keys=keys, variant_ids=variant_ids, + genome_version, dataset_type, keys=keys, variant_ids=variant_ids, ) key_field = 'variantId' if primary_id_field == 'variant_id' else primary_id_field variant_fields = ['key'] @@ -200,6 +190,7 @@ def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], pri for (_, variant_id), variant in variant_data.items(): if variant_id in variants_by_id: variant.update(variants_by_id[variant_id]) + variant['dataset_type'] = dataset_type return variant_data From 0b58377adc29472642eadd45bc4a1945313c4aaa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 11:02:03 -0400 Subject: [PATCH 211/252] better heloer separation --- .../commands/tag_seqr_prioritized_variants.py | 9 +++++++-- seqr/views/apis/individual_api.py | 14 +++++++++----- seqr/views/apis/summary_data_api.py | 13 ++++++------- seqr/views/utils/variant_utils.py | 16 ++++------------ 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 0f2a52edfe..28a9c54b01 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -587,10 +587,15 @@ def wrapped(v): @staticmethod def _parse_new_saved_variants(dataset_type): def wrapped(new_variant_keys, family_variant_data): - return get_saved_variant_annotations( - {k: v for k, v in family_variant_data.items() if k in new_variant_keys}, + new_variant_data = {k: v for k, v in family_variant_data.items() if k in new_variant_keys} + variants_by_id = get_saved_variant_annotations( + {k for k, v in new_variant_data.items() if not (v.get('key') and v.get('variantId'))}, dataset_type=dataset_type, genome_version=GENOME_VERSION_GRCh38, primary_id_field='key', ) + return { + k: {**v, 'dataset_type': dataset_type, **(variants_by_id.get(k[1], {}))} + for k, v in family_variant_data.items() if k in new_variant_keys + } return wrapped @classmethod diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 6458b4a2b6..75846a1bb1 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -885,14 +885,18 @@ def import_gregor_metadata(request, project_guid): def _parse_new_aip_saved_variants(new_variant_keys, family_variant_data): family_id = next(family_id for family_id, _ in new_variant_keys) genome_version = Family.objects.filter(id=family_id).values_list('project__genome_version', flat=True).first() - new_variant_data = get_saved_variant_annotations( - {k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version, - ) - for variant in new_variant_data.values(): - if not variant.get('key'): + variants_by_id = get_saved_variant_annotations(new_variant_keys, genome_version) + new_variant_data = {} + for key in new_variant_keys: + variant = family_variant_data[key] + variant_id = key[1] + if variant_id in variants_by_id: + variant.update(variants_by_id[variant_id]) + else: variant.update({'key': None, 'saved_variant_json': {k: v for k, v in variant.items() if k in { 'chrom', 'pos', 'ref', 'alt', 'variantId', 'xpos', 'genomeVersion', 'genotypes', 'transcripts', 'mainTranscriptId', }}}) + new_variant_data[key] = variant return new_variant_data diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index e88ed4c73f..f9f85d6d1f 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -272,11 +272,7 @@ def _search_new_saved_variants(family_variant_ids, *args, **kwargs): def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, dict], family_variant_ids: set[tuple[int, str]], genome_version: str) -> list[dict]: - variant_data = get_saved_variant_annotations( - {variant_id: {'genotypes': {}, 'familyGuids': []} for variant_id in family_variant_ids}, genome_version=genome_version, - ) - variants_by_key = {variant['key']: variant for variant in variant_data.values() if variant.get('key')} - + variants_by_key = get_saved_variant_annotations(family_variant_ids, genome_version=genome_version, group_by_field='key') families_by_project = defaultdict(list) for family in families_by_id.values(): families_by_project[family['project__guid']].append(family['guid']) @@ -287,8 +283,11 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, ) for key, entry_data in genotype_keys.items(): variants_by_key[key]['xpos'] = entry_data['xpos'] - variants_by_key[key]['genotypes'].update(entry_data['genotypes']) - variants_by_key[key]['familyGuids'] += sorted({g['familyGuid'] for g in entry_data['genotypes'].values()}) + variants_by_key[key]['genotypes'] = {**entry_data['genotypes'], **variants_by_key[key].get('genotypes', {})} + variants_by_key[key]['familyGuids'] = sorted([ + *{g['familyGuid'] for g in entry_data['genotypes'].values()}, + *(variants_by_key[key].get('familyGuids', [])), + ]) return list(variants_by_key.values()) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 8fcb24ea70..6edcc91a67 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -1,4 +1,4 @@ -from collections import defaultdict +from collections import defaultdict, abc from clickhouse_backend.models import ArrayField, StringField from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg @@ -160,12 +160,9 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ return updated_tag -def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], genome_version: str, primary_id_field: str = 'variant_id', dataset_type: str = None) -> dict[tuple[int, str], dict]: +def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], genome_version: str, primary_id_field: str = 'variant_id', group_by_field=None, dataset_type: str = None) -> dict[str, dict]: dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS - variant_ids = { - variant_id for (_, variant_id), variant in variant_data.items() - if not (variant.get('key') and variant.get('variantId')) - } + variant_ids = {variant_id for _, variant_id in variant_keys} keys = None if primary_id_field == 'key': keys = variant_ids @@ -186,12 +183,7 @@ def get_saved_variant_annotations(variant_data: dict[tuple[int, str], dict], gen variant_fields += ['chrom', 'pos', 'end'] else: variant_values.update(qs.split_variant_id_annotations()) - variants_by_id = {v[key_field]: v for v in qs.join_variant_id().values(*variant_fields, **variant_values)} - for (_, variant_id), variant in variant_data.items(): - if variant_id in variants_by_id: - variant.update(variants_by_id[variant_id]) - variant['dataset_type'] = dataset_type - return variant_data + return {v[group_by_field or key_field]: v for v in qs.join_variant_id().values(*variant_fields, **variant_values)} def reset_cached_search_results(project, reset_index_metadata=False): From 9e7ca36390fe615e7095a26de6fc70c805953b4f Mon Sep 17 00:00:00 2001 From: hanars Date: Thu, 19 Mar 2026 11:27:44 -0400 Subject: [PATCH 212/252] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- seqr/views/apis/saved_variant_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index ea0de49a0a..2fe24827a3 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -54,7 +54,7 @@ def create_manual_saved_variant_handler(request, family_guid): check_project_permissions(family.project, request.user) variant_json = json.loads(request.body) - tags = variant_json.pop('tags') + tags = variant_json.pop('tags', []) saved_variant_guids = {guid for guid, is_selected in variant_json.pop('variants', {}).items() if is_selected} genome_version = family.project.genome_version @@ -77,8 +77,8 @@ def create_manual_saved_variant_handler(request, family_guid): if variant_json.get('mainTranscriptId'): variant_json['transcripts'][gene_id].append({ 'transcriptId': variant_json['mainTranscriptId'], - 'hgvsc': variant_json.pop('hgvsc'), - 'hgvsp': variant_json.pop('hgvsp'), + 'hgvsc': variant_json.pop('hgvsc', None), + 'hgvsp': variant_json.pop('hgvsp', None), }) variant_json['saved_variant_json'] = {**variant_json} From f96c90a80ca32864245754e79e426f4b8e8be6eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 11:30:40 -0400 Subject: [PATCH 213/252] pr feedback --- seqr/views/apis/saved_variant_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index ea0de49a0a..a94776bb51 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -60,10 +60,10 @@ def create_manual_saved_variant_handler(request, family_guid): genome_version = family.project.genome_version try: xpos = get_xpos(variant_json['chrom'], variant_json['pos']) - except ValueError as e: + variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" + except (KeyError, ValueError) as e: return create_json_response({'error': str(e)}, status=400) - variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" variant_json.update({ 'genomeVersion': genome_version, 'transcripts': {}, @@ -91,7 +91,7 @@ def create_manual_saved_variant_handler(request, family_guid): saved_variant = create_model_from_json(SavedVariant, model_json, request.user) saved_variant_guids.add(saved_variant.guid) - saved_variants = SavedVariant.objects.filter(guid__in=saved_variant_guids) + saved_variants = SavedVariant.objects.filter(family=family, guid__in=saved_variant_guids) _update_tags(saved_variants, {'tags': tags}, request.user) response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) From 60d639f74b7dbc6f5a5e1559c24e43f4526ed2bb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 15:06:39 -0400 Subject: [PATCH 214/252] add and populate saved variant summary cols --- clickhouse_search/backend/functions.py | 7 +- .../commands/set_saved_variant_key.py | 1 + ...nt_main_transcript_savedvariant_sv_type.py | 121 ++++++++++++++++++ seqr/models.py | 3 + 4 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 2bd6461c6d..2bc99c5ade 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -2,7 +2,8 @@ from django.db.models import Func, Subquery, lookups, BooleanField, Aggregate from django.db.models.sql.datastructures import BaseTable, Join -from clickhouse_search.backend.fields import NestedField +from clickhouse_search.backend.fields import NestedField, NamedTupleField + class Array(Func): function = 'array' @@ -124,9 +125,9 @@ def as_sql(self, compiler, connection, *args, **kwargs): class ArrayIndex(IndexTransform): - def __init__(self, index, *args, **kwargs): + def __init__(self, index, *args, base_field=None, **kwargs): super().__init__(index+1, None, *args, **kwargs) - self.base_field = self.source_expressions[0].output_field.base_field + self.base_field = base_field or self.source_expressions[0].output_field.base_field @ArrayField.register_lookup diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 49a6d0c0d8..15ee50e5d1 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -159,6 +159,7 @@ class Command(BaseCommand): def handle(self, *args, **options): + # TODO num_updated = SavedVariant.objects.filter( genotypes={}, saved_variant_json__genotypes__isnull=False, ).exclude(saved_variant_json__genotypes={}).update(genotypes=F('saved_variant_json__genotypes')) diff --git a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py new file mode 100644 index 0000000000..deacceac7c --- /dev/null +++ b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py @@ -0,0 +1,121 @@ +# Generated by Django 4.2.24 on 2026-03-19 15:57 + +from collections import defaultdict +from django.db import migrations, models + +from clickhouse_search.backend.fields import NamedTupleField +from clickhouse_search.backend.functions import ArrayIndex, ArrayObjectSort +from clickhouse_search.models.search_models import VARIANT_DETAILS_CLASS_MAP +from clickhouse_search.search import _get_variant_details_queryset +import seqr.views.utils.json_utils + +BATCH_SIZE = 10000 + + +def populate_saved_variant_fields(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + no_key_variants = SavedVariant.objects.using(db_alias).filter(key__isnull=True) + no_key_variants.filter(saved_variant_json__genotypes__isnull=False).update( + genotypes=models.F('saved_variant_json__genotypes'), + ) + no_key_variants.filter(saved_variant_json__svType__isnull=False).update( + sv_type=models.F('saved_variant_json__svType'), + dataset_type='SV', + ) + no_key_variants.filter(sv_type__isnull=True).update(dataset_type='SNV_INDEL') + + to_update = [] + transcript_variants = no_key_variants.filter(sv_type__isnull=True, saved_variant_json__transcripts__isnull=False).exclude(saved_variant_json__transcripts={}) + for variant in transcript_variants: + main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') + if main_transcript_id: + variant.main_transcript = next( + t for ts in variant.saved_variant_json['transcripts'].values() for t in ts + if t['transcriptId'] == main_transcript_id + ) + to_update.append(variant) + SavedVariant.objects.using(db_alias).bulk_update(to_update, ['main_transcript']) + + +def populate_clickhouse_main_transcript(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'MITO', + expression=models.F('sorted_transcript_consequences__0'), + ) + + for genome_version in ['38', '37']: + output_field = VARIANT_DETAILS_CLASS_MAP[genome_version].transcripts.field.clone() + output_field.group_by_key = 'transcriptId' + output_field.flatten_groups = True + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, + family__project__genome_version=genome_version, selected_main_transcript_id__isnull=False, + expression=ArrayObjectSort('transcripts', sort_field='transcriptId', output_field=output_field), + process_value=lambda transcripts, variant: transcripts.get(variant.selected_main_transcript_id, {}), + ) + + _update_field_from_clickhouse( + SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, + family__project__genome_version=genome_version, selected_main_transcript_id__isnull=True, + expression=ArrayIndex(0, ArrayObjectSort('transcripts', sort_field='transcriptRank'), base_field=NamedTupleField(output_field.base_fields)), + ) + + +def populate_clickhouse_sv_type(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + for dataset_type in ['SV_WGS', 'SV_WES']: + _update_field_from_clickhouse(SavedVariant, db_alias, 'sv_type', dataset_type) + + +def _update_field_from_clickhouse(SavedVariant, db_alias, field, dataset_type, genome_version='38', expression=None, process_value=None, **filters): + variants = SavedVariant.objects.using(db_alias).filter(key__isnull=False, dataset_type=dataset_type, **filters) + if not variants.exists(): + return + variants_by_key = defaultdict(list) + for v in variants: + variants_by_key[v.key].append(v) + all_keys = sorted(variants_by_key.keys()) + print(f'Populating {field} for {len(all_keys)} {dataset_type} ({genome_version}) clickhouse variants') + for i in range(0, len(all_keys), BATCH_SIZE): + batch_keys = all_keys[i:i + BATCH_SIZE] + clickhouse_qs = _get_variant_details_queryset(genome_version, dataset_type, batch_keys) + to_update = [] + if expression: + clickhouse_qs = clickhouse_qs.annotate(**{field: expression}) + for key, value in clickhouse_qs.using('clickhouse').values_list('key', field): + for variant in variants_by_key[key]: + if process_value: + value = process_value(value, variant) + setattr(variant, field, value) + to_update.append(variant) + num_updated = SavedVariant.objects.using(db_alias).bulk_update(to_update, [field]) + print(f'Updated {num_updated} variants') + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0084_rename_restrict_hpo_sharing_project_restrict_sharing'), + ] + + operations = [ + migrations.AddField( + model_name='savedvariant', + name='main_transcript', + field=models.JSONField(default=dict, encoder=seqr.views.utils.json_utils.DjangoJSONEncoderWithSets), + ), + migrations.AddField( + model_name='savedvariant', + name='sv_type', + field=models.CharField(max_length=15, null=True), + ), + migrations.RunPython(populate_saved_variant_fields, reverse_code=migrations.RunPython.noop), + migrations.RunPython(populate_clickhouse_main_transcript, reverse_code=migrations.RunPython.noop), + migrations.RunPython(populate_clickhouse_sv_type, reverse_code=migrations.RunPython.noop), + ] diff --git a/seqr/models.py b/seqr/models.py index c407d33787..36363d728c 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -14,6 +14,7 @@ from seqr.utils.logging_utils import log_model_update, log_model_bulk_update, SeqrLogger from seqr.utils.xpos_utils import get_chrom_pos +from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.terra_api_utils import anvil_enabled from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_CHOICES from settings import MME_DEFAULT_CONTACT_NAME, MME_DEFAULT_CONTACT_HREF, MME_DEFAULT_CONTACT_INSTITUTION, \ @@ -843,7 +844,9 @@ class SavedVariant(ModelWithGUID): selected_main_transcript_id = models.CharField(max_length=20, null=True) saved_variant_json = models.JSONField(default=dict) genotypes = models.JSONField(default=dict) + main_transcript = models.JSONField(default=dict, encoder=DjangoJSONEncoderWithSets) gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True) + sv_type = models.CharField(max_length=15, null=True) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True) acmg_classification = models.JSONField(null=True) # ACMG based classification From c5d91248f52db24525a245f564fcceca5f9f1772 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 15:25:46 -0400 Subject: [PATCH 215/252] fix sleected transcript fallback --- ...variant_main_transcript_savedvariant_sv_type.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py index deacceac7c..777c2a9687 100644 --- a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py +++ b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py @@ -49,19 +49,19 @@ def populate_clickhouse_main_transcript(apps, schema_editor): for genome_version in ['38', '37']: output_field = VARIANT_DETAILS_CLASS_MAP[genome_version].transcripts.field.clone() - output_field.group_by_key = 'transcriptId' - output_field.flatten_groups = True _update_field_from_clickhouse( SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, - family__project__genome_version=genome_version, selected_main_transcript_id__isnull=False, - expression=ArrayObjectSort('transcripts', sort_field='transcriptId', output_field=output_field), - process_value=lambda transcripts, variant: transcripts.get(variant.selected_main_transcript_id, {}), + family__project__genome_version=genome_version, + expression=ArrayIndex(0, ArrayObjectSort('transcripts', sort_field='transcriptRank'), base_field=NamedTupleField(output_field.base_fields)), ) + output_field.group_by_key = 'transcriptId' + output_field.flatten_groups = True _update_field_from_clickhouse( SavedVariant, db_alias, 'main_transcript', 'SNV_INDEL', genome_version=genome_version, - family__project__genome_version=genome_version, selected_main_transcript_id__isnull=True, - expression=ArrayIndex(0, ArrayObjectSort('transcripts', sort_field='transcriptRank'), base_field=NamedTupleField(output_field.base_fields)), + family__project__genome_version=genome_version, selected_main_transcript_id__isnull=False, + expression=ArrayObjectSort('transcripts', sort_field='transcriptId', output_field=output_field), + process_value=lambda transcripts, variant: transcripts.get(variant.selected_main_transcript_id, variant.main_transcript), ) From 62114412198dfeac01b89cac7b4f50702871b15a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 15:28:48 -0400 Subject: [PATCH 216/252] clean up --- .../management/commands/set_saved_variant_key.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 15ee50e5d1..6b2515bc57 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -159,12 +159,6 @@ class Command(BaseCommand): def handle(self, *args, **options): - # TODO - num_updated = SavedVariant.objects.filter( - genotypes={}, saved_variant_json__genotypes__isnull=False, - ).exclude(saved_variant_json__genotypes={}).update(genotypes=F('saved_variant_json__genotypes')) - logger.info(f'Updated genotypes for {num_updated} variants') - variant_ids = SavedVariant.objects.filter( key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh38, saved_variant_json__populations__isnull=False, # Omit manual variants @@ -226,7 +220,6 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ mapped_variant_ids = list(variant_key_map.keys()) - update_fields = ['key', 'dataset_type'] total_num_updated = 0 for i in range(0, len(mapped_variant_ids), BATCH_SIZE): batch_ids = mapped_variant_ids[i:i + BATCH_SIZE] @@ -235,8 +228,7 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ ) for variant in saved_variants: variant.key = variant_key_map[variant.variant_id] - variant.dataset_type = dataset_type - num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields) + num_updated = SavedVariant.objects.bulk_update(saved_variants, ['key']) logger.info(f'Updated batch of {num_updated}') total_num_updated += num_updated From b14f95e339377365677714c68413c2bac3e483cf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 15:47:46 -0400 Subject: [PATCH 217/252] clean up --- seqr/views/apis/saved_variant_api.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 9441217815..784bd14faa 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -3,7 +3,6 @@ from django.db.models import Q -from clickhouse_search.search import get_variant_main_transcripts_by_key from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project, Sample from seqr.utils.xpos_utils import get_xpos @@ -122,7 +121,7 @@ def create_saved_variant_handler(request): response = {} if variant_json.get('note'): - _, response = _create_variant_note(saved_variants, variant_json, request.user, family.project.genome_version) + _, response = _create_variant_note(saved_variants, variant_json, request.user) elif variant_json.get('tags'): _update_tags(saved_variants, variant_json, request.user) @@ -149,7 +148,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response({'error': 'Note is required'}, status=400) # update saved_variants - note, response = _create_variant_note(saved_variants, request_json, request.user, family.project.genome_version) + note, response = _create_variant_note(saved_variants, request_json, request.user) note_json = get_json_for_variant_note(note) note_json['variantGuids'] = all_variant_guids response.update({ @@ -162,7 +161,7 @@ def create_variant_note_handler(request, variant_guids): return create_json_response(response) -def _create_variant_note(saved_variants, note_json, user, genome_version): +def _create_variant_note(saved_variants, note_json, user): note = create_model_from_json(VariantNote, { 'note': note_json.get('note'), 'report': note_json.get('report') or False, @@ -172,7 +171,9 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = _variant_gene_id(saved_variants[0], genome_version) + variant = saved_variants[0] + gene_id = variant.gene_ids[0] if len(variant.gene_ids) == 1 or not variant.selected_main_transcript_id \ + else variant.main_transcript.get('geneId') create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -181,14 +182,6 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): return note, response -def _variant_gene_id(variant, genome_version): - if len(variant.gene_ids) == 1 or not variant.selected_main_transcript_id: - return variant.gene_ids[0] - return get_variant_main_transcripts_by_key( - genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, {variant.key: [variant.selected_main_transcript_id]}, - )[variant.key]['main_transcripts'][variant.selected_main_transcript_id].get('geneId') - - @login_and_policies_required def update_variant_note_handler(request, variant_guids, note_guid): note = VariantNote.objects.get(guid=note_guid) From b518f6b3408cd6f602799e67bd6fecf51819afd8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 16:25:03 -0400 Subject: [PATCH 218/252] use model fields instead of clickhouse --- clickhouse_search/search.py | 42 +-------- ...nt_main_transcript_savedvariant_sv_type.py | 4 +- seqr/views/apis/saved_variant_api.py | 1 + seqr/views/utils/anvil_metadata_utils.py | 92 +++++-------------- seqr/views/utils/project_context_utils.py | 34 +------ seqr/views/utils/variant_utils.py | 1 + 6 files changed, 36 insertions(+), 138 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fc9e6fbb53..02d8f9ce19 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -9,7 +9,7 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ - ArrayMap, Modulo, ArrayObjectSort + ArrayMap, Modulo from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ @@ -334,7 +334,7 @@ def get_clickhouse_cache_results(results, sort, family_guid): def format_clickhouse_export_results(results, genome_version): formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} - detail_qs = _get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) + detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) details_by_key = { detail['key']: detail for detail in detail_qs.values( 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), @@ -365,7 +365,7 @@ def format_clickhouse_results(results, genome_version): } details_by_key = { detail['key']: detail for detail in - _get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() } formatted_results = [] @@ -806,14 +806,14 @@ def get_variants_queryset(genome_version, dataset_type, keys, variant_ids=None): return variants_cls.objects.filter(key__in=keys) -def _get_variant_details_queryset(genome_version, dataset_type, keys): +def get_variant_details_queryset(genome_version, dataset_type, keys): if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: return VARIANT_DETAILS_CLASS_MAP[genome_version].objects.filter(key__in=keys) return get_variants_queryset(genome_version, dataset_type, keys) def get_clickhouse_variant_annotations(genome_version, dataset_type, keys): - qs = _get_variant_details_queryset(genome_version, dataset_type, keys) + qs = get_variant_details_queryset(genome_version, dataset_type, keys) return qs.join_annotations().result_values(skip_entry_fields=True) @@ -827,38 +827,6 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids): return lookup -def get_variant_main_transcripts_by_key(genome_version, dataset_type, selected_transcripts_by_key, include_clinvar=False, additional_values=None): - qs = _get_variant_details_queryset(genome_version, dataset_type, selected_transcripts_by_key.keys()) - if hasattr(qs.model, 'transcripts'): - transcript_field = 'transcripts' - expr = ArrayObjectSort - expr_kwargs = {'sort_field': 'transcriptRank'} - else: - transcript_field = qs.TRANSCRIPT_FIELD - expr = ArrayMap - expr_kwargs = {'mapped_expression': 'x'} - output_field = getattr(qs.model, transcript_field).field.clone() - output_field.group_by_key = None - qs = qs.annotate(sorted_transcripts=expr(transcript_field, output_field=output_field, **expr_kwargs)) - - fields = ['key', 'sorted_transcripts'] - if include_clinvar: - fields.append('clinvar') - qs = qs.join_clinvar() - - variants_by_key = {} - for variant in qs.values(*fields, **(additional_values or {})): - key = variant['key'] - transcripts = variant.pop('sorted_transcripts') - variant['main_transcripts'] = { - selected_transcript_id: _main_transcript(selected_transcript_id, transcripts) - for selected_transcript_id in selected_transcripts_by_key[key] - } - variants_by_key[key] = variant - - return variants_by_key - - def _main_transcript(selected_transcript_id, sorted_transcripts): if not sorted_transcripts: return {} diff --git a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py index 777c2a9687..a5d9674b7d 100644 --- a/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py +++ b/seqr/migrations/0085_savedvariant_main_transcript_savedvariant_sv_type.py @@ -6,7 +6,7 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import ArrayIndex, ArrayObjectSort from clickhouse_search.models.search_models import VARIANT_DETAILS_CLASS_MAP -from clickhouse_search.search import _get_variant_details_queryset +from clickhouse_search.search import get_variant_details_queryset import seqr.views.utils.json_utils BATCH_SIZE = 10000 @@ -84,7 +84,7 @@ def _update_field_from_clickhouse(SavedVariant, db_alias, field, dataset_type, g print(f'Populating {field} for {len(all_keys)} {dataset_type} ({genome_version}) clickhouse variants') for i in range(0, len(all_keys), BATCH_SIZE): batch_keys = all_keys[i:i + BATCH_SIZE] - clickhouse_qs = _get_variant_details_queryset(genome_version, dataset_type, batch_keys) + clickhouse_qs = get_variant_details_queryset(genome_version, dataset_type, batch_keys) to_update = [] if expression: clickhouse_qs = clickhouse_qs.annotate(**{field: expression}) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 784bd14faa..20d106bd7f 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -345,6 +345,7 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) check_project_permissions(saved_variant.family.project, request.user, can_edit=True) + # TODO update_model_from_json(saved_variant, {'selected_main_transcript_id': transcript_id}, request.user) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 01eb449849..f3610f7a39 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -2,14 +2,13 @@ from datetime import datetime from django.db.models import F, Q, Value, CharField, Aggregate -from django.db.models.functions import Replace +from django.db.models.functions import Coalesce, Replace from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg import requests from typing import Callable, Iterable -from clickhouse_search.backend.functions import ArrayFilter -from clickhouse_search.search import get_variants_queryset, get_variant_main_transcripts_by_key +from clickhouse_search.search import get_variant_details_queryset from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType @@ -352,7 +351,7 @@ def _get_parsed_saved_discovery_variants_by_family( variant_json = variant_json_by_guid[variant.guid] main_transcript = variant_json['main_transcript'] gene_ids.add(variant_json['gene_id']) - sv_type = variant_json.get('svType') + sv_type = variant_json.get('sv_type') variant_type = next( (variant_type for variant_type, has_type in VARIANT_TYPES if has_type(variant.ref, variant.alt)), 'INDEL') @@ -407,79 +406,34 @@ def _get_variant_json_by_guid(saved_variants, include_clinvar): variant_json_by_guid = {} variant_keys_by_search_type = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) for v in saved_variants: - if v.key: - end_chrom, end = get_chrom_pos(v.xpos_end) - variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids} - variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id)) - else: - main_transcript = _get_variant_main_transcript(v) - gene_id = main_transcript.get('geneId') - gene_ids = [gene_id] if gene_id else v.gene_ids - variant_json_by_guid[v.guid] = { - **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, - } + end_chrom, end = get_chrom_pos(v.xpos_end) + variant_json_by_guid[v.guid] = { + 'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids, + 'main_transcript': v.main_transcript, 'gene_id': Coalesce('main_transcript__geneId', 'gene_ids__0'), + 'sv_type': v.sv_type, 'CAID': None, + } + if include_clinvar: + variant_json_by_guid[v.guid]['clinvar'] = None + if not v.dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): + variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append(v.guid) for genome_version, keys_by_dataset_type in variant_keys_by_search_type.items(): for dataset_type, key_map in keys_by_dataset_type.items(): - set_json_func = _set_clickhouse_sv_json if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) else _set_clickhouse_snv_indel_json - set_json_func(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar) + _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar) return variant_json_by_guid -def _set_clickhouse_sv_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): - annotations = get_variants_queryset(genome_version, dataset_type, key_map.keys()).values( - 'key', svType=F('sv_type'), - ) - defaults = {'CAID': None, 'gene_id': None, 'main_transcript': {}} - if include_clinvar: - defaults['clinvar'] = None - for anns in annotations: - for guid, _ in key_map[anns['key']]: - variant_json_by_guid[guid].update({**anns, **defaults}) - - def _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): - annotation_values = {field: Value(None, output_field=CharField()) for field in ['CAID', 'svType']} - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - annotation_values['CAID'] = F('caid') - - selected_transcripts_by_key = { - key: [transcript_id for (_, transcript_id) in variants] for key, variants in key_map.items() - } - annotations_by_key = get_variant_main_transcripts_by_key( - genome_version, dataset_type, selected_transcripts_by_key, include_clinvar=include_clinvar, additional_values=annotation_values, - ) - for key, anns in annotations_by_key.items(): - for guid, selected_main_transcript_id in key_map[key]: - main_transcript = anns['main_transcripts'][selected_main_transcript_id] - gene_id = main_transcript.get('geneId') - variant_json_by_guid[guid].update({ - **anns, - 'main_transcript': main_transcript, - 'gene_id': gene_id, - }) - if gene_id: - variant_json_by_guid[guid]['gene_ids'] = [gene_id] - - -def _get_variant_main_transcript(variant_model): - variant = variant_model.saved_variant_json - main_transcript_id = variant_model.selected_main_transcript_id or variant.get('mainTranscriptId') - if main_transcript_id: - for gene_id, transcripts in variant.get('transcripts', {}).items(): - main_transcript = next((t for t in transcripts if t['transcriptId'] == main_transcript_id), None) - if main_transcript: - if 'geneId' not in main_transcript: - main_transcript['geneId'] = gene_id - return main_transcript - elif len(variant.get('transcripts', {})) == 1: - gene_id = next(k for k in variant['transcripts'].keys()) - # Handle manually created SNPs - if variant['transcripts'][gene_id] == []: - return {'geneId': gene_id} - return {} - + qs = get_variant_details_queryset(genome_version, dataset_type, key_map.keys()) + fields = ['key'] + if include_clinvar: + fields.append('clinvar') + qs = qs.join_clinvar() + caid_expr = F('caid') if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else Value(None, output_field=CharField()) + for anns in qs.values(*fields, CAID=caid_expr): + for guid in key_map[anns['key']]: + variant_json_by_guid[guid].update(anns) def _get_transcript_field(field, config, transcript): value = transcript.get(config.get('seqr_field', field)) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index adce3b5d5f..30943d8204 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,9 +1,8 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models.functions import JSONObject +from django.db.models.functions import Coalesce, JSONObject -from clickhouse_search.search import get_variant_main_transcripts_by_key from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample from seqr.utils.gene_utils import get_genes @@ -124,18 +123,9 @@ def families_discovery_tags(families, genome_version, project=None): varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, ).annotate(family_guid=F('family__guid')) - family_discovery_genes = list(discovery_variants.filter( - Q(gene_ids__len=1) | (Q(selected_main_transcript_id__isnull=True) & ~Q(dataset_type__startswith='SV')) - ).values_list('family_guid', F('gene_ids__0'))) - try: - _get_clickhouse_selected_transcript_gene_id( - family_discovery_genes, - discovery_variants.filter(selected_main_transcript_id__isnull=False, gene_ids__len__gt=1), - genome_version, - ) - except Exception as e: - logger.error(f'Error loading discovery genes from clickhouse: {e}', None) - + family_discovery_genes = list(discovery_variants.values_list( + 'family_guid', Coalesce('main_transcript__geneId', 'gene_ids__0'), + )) gene_ids = set() for family_guid, gene_id in family_discovery_genes: gene_ids.add(gene_id) @@ -147,22 +137,6 @@ def families_discovery_tags(families, genome_version, project=None): } -def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): - tags_by_dataset_type = discovery_variants.values('dataset_type').annotate( - keys=ArrayAgg('key', distinct=True), - variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')), - ) - - for dataset_type, keys, variants in tags_by_dataset_type.values_list('dataset_type', 'keys', 'variants'): - selected_transcripts_by_key = defaultdict(list) - for v in variants: - selected_transcripts_by_key[v['key']].append(v['selected_main_transcript_id']) - transcripts_by_key = get_variant_main_transcripts_by_key(genome_version, dataset_type, selected_transcripts_by_key) - for v in variants: - gene_id = transcripts_by_key[v['key']]['main_transcripts'][v['selected_main_transcript_id']].get('geneId') - family_discovery_genes.append((v['family_guid'], gene_id)) - - MME_TAG_NAME = 'MME Submission' diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 6edcc91a67..fb30092895 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -37,6 +37,7 @@ def parse_saved_variant_json(variant_json, family_id): gene_ids = sorted( variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json) ) if 'transcripts' in variant_json else variant_json['gene_ids'] + # TODO save sv_type and main_transcript return { 'key': variant_json['key'], 'xpos': xpos, From d88827108ca4cf572ad6d1f481cab84e81ab1d81 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 16:39:57 -0400 Subject: [PATCH 219/252] update fixtures --- seqr/fixtures/1kg_project.json | 9 +++++++++ seqr/fixtures/report_variants.json | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 2d93844b42..f7b34ca2a9 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1869,7 +1869,9 @@ "ref": "GAGA", "alt": "G", "variant_id": "21-3343353-GAGA-G", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, "saved_variant_json": { "variantId": "21-3343353-GAGA-G", "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, @@ -1923,6 +1925,7 @@ }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], "saved_variant_json": {}, + "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": null, "consequenceTerms": [], "geneId": "ENSG00000135953", "hgvsc": null, "hgvsp": null, "loftee": [null, []], "majorConsequence": null, "transcriptId": "ENST00000371839", "transcriptRank": 1}, "selected_main_transcript_id": "ENST00000371839", "family": 2 } @@ -1940,7 +1943,9 @@ "ref": "G", "alt": "A", "variant_id": "1-46859832-G-A", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000197530"], + "main_transcript": {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "CELSR1", "majorConsequence": "splice_region_variant", "canonical": "YES", "hgvs": "ENST00000505820.2:c.1067-4G>A", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000505820.2:c.1067-4G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"}, "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -1981,7 +1986,9 @@ "ref": "G", "alt": "CA", "variant_id": "1-1562437-G-CA", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000197530"], + "main_transcript": {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"}, "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -2090,7 +2097,9 @@ "ref": null, "alt": null, "variant_id": "prefix_19107_DEL", + "dataset_type": "SV", "gene_ids": ["ENSG00000135953", "ENSG00000223972", "ENSG00000240361"], + "sv_type": "DEL", "saved_variant_json": { "liftedOverGenomeVersion": null, "pos": 249045487, diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 02a9f039fe..83efbed124 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -12,6 +12,7 @@ "ref": "C", "alt": "T", "variant_id": "19-1912634-C-T", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000240361"], "saved_variant_json": { "liftedOverGenomeVersion": "37", @@ -58,7 +59,9 @@ "ref": "G", "alt": "T", "variant_id": "19-1912633-G-T", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361"}, "saved_variant_json": { "liftedOverGenomeVersion": "37", "pos": 1912633, @@ -107,7 +110,9 @@ "ref": "G", "alt": "C", "variant_id": "19-1912632-G-C", + "dataset_type": "SNV_INDEL", "gene_ids": ["ENSG00000135953"], + "main_transcript": {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361", "hgvsc": "c.586_587delinsTT", "hgvsp": "p.Ala196Leu"}, "saved_variant_json": { "pos": 1912632, "end": 1912632, @@ -149,6 +154,7 @@ "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} }, "gene_ids": ["ENSG00000198727"], + "main_transcript": {"aminoAcids": "L", "biotype": "protein_coding", "canonical": 1, "codons": "Tta/Cta", "consequenceTerms": ["synonymous_variant"], "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "loftee": [null, []], "majorConsequence": "synonymous_variant", "transcriptId": "ENST00000361789", "transcriptRank": 0}, "saved_variant_json": {}, "family": 14 } @@ -167,6 +173,7 @@ "variant_id": "suffix_19107_DEL_013746", "key": 101, "dataset_type": "SV_WGS", + "sv_type": "SV", "genotypes": { "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, From e9d3f22ffb33b4d2a4d2a35cbc6882f8bb64de51 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 16:45:37 -0400 Subject: [PATCH 220/252] clean up saved variant json usage --- .../commands/check_for_new_samples_from_pipeline.py | 5 +---- seqr/views/apis/family_api.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 71b35b5fd0..c0af42bf98 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -395,10 +395,7 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type) for family_guid in family_guids: variant_models_by_key = { v.key: v for v in - SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid).filter( - Q(saved_variant_json__genomeVersion__isnull=True) | - Q(saved_variant_json__genomeVersion=project.genome_version.replace('GRCh', '')) - ) + SavedVariant.objects.filter(dataset_type=dataset_type, family__guid=family_guid, key__isnull=False) } if not variant_models_by_key: continue diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index f0152e154e..d6facef3ec 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -58,7 +58,7 @@ def family_page_data(request, family_guid): discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( 'xpos', 'xpos_end', 'gene_ids', 'key', 'dataset_type', - svType=F('saved_variant_json__svType') + svType=F('sv_type') ) gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} discovery_variant_intervals = [dict(zip( From d8e1e7a8881f313321181c328f7c8674d384acfc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 16:47:32 -0400 Subject: [PATCH 221/252] remove unused imports --- clickhouse_search/backend/functions.py | 2 +- seqr/views/utils/project_context_utils.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 2bc99c5ade..d2f166764c 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -2,7 +2,7 @@ from django.db.models import Func, Subquery, lookups, BooleanField, Aggregate from django.db.models.sql.datastructures import BaseTable, Join -from clickhouse_search.backend.fields import NestedField, NamedTupleField +from clickhouse_search.backend.fields import NestedField class Array(Func): diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 30943d8204..a1c0213762 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,10 +1,9 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects -from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models.functions import Coalesce, JSONObject +from django.db.models.functions import Coalesce from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ - VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample + VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ From 2638c6670638fd8c9f79b7ba79798abe8d12720a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 17:41:33 -0400 Subject: [PATCH 222/252] actually save new fields --- clickhouse_search/managers.py | 2 +- .../tag_seqr_prioritized_variants_tests.py | 18 +++++---- seqr/views/apis/individual_api_tests.py | 6 ++- seqr/views/apis/saved_variant_api_tests.py | 17 ++++++--- seqr/views/apis/summary_data_api_tests.py | 6 +++ seqr/views/utils/variant_utils.py | 37 ++++++++++++++----- 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 31ce7895c9..8f5c0561c0 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -287,7 +287,7 @@ def annotation_fields(self): @property def variant_detail_field(self): - return next(obj.name for obj in self.model._meta.related_objects if obj.name.startswith('variantdetails')) + return next((obj.name for obj in self.model._meta.related_objects if obj.name.startswith('variantdetails')), None) @property def entry_field(self): diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index a4c185da1e..0633e7b0e8 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -72,24 +72,28 @@ def test_command(self, mock_datetime, mock_slack, mock_email): }) new_saved_variants = SavedVariant.objects.filter(key__in=[2, 3, 4, 18, 19]).order_by('key').values( - 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', + 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', 'main_transcript', 'sv_type', ) + self.maxDiff = None self.assertListEqual(list(new_saved_variants), [{ 'key': 2, 'variant_id': '1-38724419-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1038724419, 'xpos_end': 1038724419, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000177000', 'ENSG00000277258'], - 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, + 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT2['transcripts']['ENSG00000177000'][0], }, {'key': 3, 'variant_id': '1-91502721-G-A', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091502721, 'xpos_end': 1091502721, 'ref': 'G', 'alt': 'A', 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, - }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, + 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT3['transcripts']['ENSG00000097046'][0], + }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, 'xpos_end': 1091511686, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000097046'], - 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, + 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'main_transcript': VARIANT4['transcripts']['ENSG00000097046'][0], }, {'key': 18, 'variant_id': 'suffix_140593_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038717327, 'xpos_end': 17038719636, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023'], - 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, + 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DUP', }, {'key': 19, 'variant_id': 'suffix_140608_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038721781, 'xpos_end': 17038735703, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023', 'ENSG00000277258', 'ENSG00000277972'], - 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, + 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DEL', }]) expected_tags = { diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index c0b9e18fa3..676efe002c 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1219,8 +1219,9 @@ def test_import_gregor_metadata(self): saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__name='GREGoR Finding' ).order_by('family_id', 'variant_id').distinct().values( - 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', + 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', ) + self.maxDiff = None self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { 'guid': 'SV0000006_1248367227_r0003_tes', @@ -1232,6 +1233,7 @@ def test_import_gregor_metadata(self): 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, 'gene_ids': ['ENSG00000240361', 'ENSG00000135953'], + 'main_transcript': {}, }) self.assertEqual(len(saved_variants[0]['genotypes']), 2) self.assertDictEqual(saved_variants[1], { @@ -1253,6 +1255,7 @@ def test_import_gregor_metadata(self): 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], }, }, + 'main_transcript': {}, 'key': None, 'dataset_type': 'SNV_INDEL', 'genotypes': {created_individual_guid: {'numAlt': 1}}, @@ -1269,6 +1272,7 @@ def test_import_gregor_metadata(self): 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, 'gene_ids': ['ENSG00000135953'], + 'main_transcript': {'hgvsc': 'c.3955G>A','hgvsp': 'c.1586-17C>G', 'transcriptId': 'ENST00000505820'}, }) variant_tags = VariantTag.objects.filter(variant_tag_type__name='GREGoR Finding') diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index a11c495417..04a40ac9fd 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -396,7 +396,10 @@ def test_create_saved_variant(self): variant_guid = next(iter(response.json()['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - self._assert_created_variant(saved_variant, CREATE_VARIANT_JSON, gene_ids=['ENSG00000277258', 'ENSG00000177000']) + self._assert_created_variant( + saved_variant, CREATE_VARIANT_JSON, gene_ids=['ENSG00000277258', 'ENSG00000177000'], + main_transcript=CREATE_VARIANT_JSON['transcripts']['ENSG00000277258'][0], + ) variant_json = { **CREATE_VARIANT_JSON, @@ -426,12 +429,14 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', has_saved_variant_json=False): + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None, has_saved_variant_json=False): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) self.assertEqual(dataset_type, saved_variant.dataset_type) + self.assertEqual(sv_type, saved_variant.sv_type) + self.assertDictEqual(main_transcript or {}, saved_variant.main_transcript) self.assertDictEqual(variant_json if has_saved_variant_json else {}, saved_variant.saved_variant_json) def test_create_saved_sv_variant(self): @@ -481,7 +486,7 @@ def test_create_saved_sv_variant(self): variant_guid = next(iter(response_json['savedVariantsByGuid'])) saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') - self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', gene_ids=['ENSG00000240361']) + self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', sv_type='DUP', gene_ids=['ENSG00000240361']) self.assertEqual(saved_variant.xpos_end, 2061414175) variant_json.update({ @@ -599,7 +604,9 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, **{k: v for k, v in manual_variant_request_body.items() if k in {'alt', 'ref', 'chrom', 'pos', 'genotypes', 'mainTranscriptId'}}, } - self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True) + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True, main_transcript={ + 'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627', + }) self.assertDictEqual(variant_json, saved_variant.saved_variant_json) base_variant_json = { @@ -652,7 +659,7 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000240361': []}, **{k: v for k, v in manual_variant_request_body.items() if k in {'chrom', 'pos', 'end', 'svName', 'svType', 'genotypes'}}, } - self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', has_saved_variant_json=True) + self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL', has_saved_variant_json=True) sv_variant_json.update(base_variant_json) sv_variant_json.update({ diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 9511a0d195..df4b151ef1 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -645,6 +645,12 @@ def _assert_expected_new_saved_variant(self, new_saved_variant): 'familyGuid': 'F000002_2', 'individualGuid': 'I000006_hg00733', 'sampleType': 'WES', }, }) + self.assertDictEqual(new_saved_variant.main_transcript, { + 'aminoAcids': None, 'biotype': 'protein_coding', 'canonical': 1, 'codons': None, + 'consequenceTerms': ['inframe_deletion'], 'geneId': 'ENSG00000135953', 'hgvsc': 'ENST00000262738.3:c.3955G>A', + 'hgvsp': 'ENST00000505820.2:c.1586-17C>G', 'loftee': {'isLofNagnag': None, 'lofFilters': None}, + 'majorConsequence': 'inframe_deletion', 'transcriptId': 'ENST00000505820', 'transcriptRank': 0, + }) def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False): self.assertEqual(response.status_code, 200) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index fb30092895..7a3aed8dc1 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -2,7 +2,7 @@ from clickhouse_backend.models import ArrayField, StringField from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import F, Q, Count +from django.db.models import F, Q, Count, Value import json import logging import redis @@ -34,28 +34,37 @@ def parse_saved_variant_json(variant_json, family_id): xpos = variant_json['xpos'] ref = variant_json.get('ref') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - gene_ids = sorted( - variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json) - ) if 'transcripts' in variant_json else variant_json['gene_ids'] - # TODO save sv_type and main_transcript + if 'transcripts' in variant_json: + main_transcript_id = variant_json.get('mainTranscriptId') + gene_ids = sorted( + variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json, main_transcript_id) + ) + main_gene_transcripts = variant_json['transcripts'][gene_ids[0]] if gene_ids else [] + main_transcript = next( + t for t in main_gene_transcripts if t.get('transcriptId') == main_transcript_id + ) if main_transcript_id else {} + else: + gene_ids = variant_json['gene_ids'] + main_transcript = variant_json['main_transcript'] return { 'key': variant_json['key'], 'xpos': xpos, 'xpos_end': xpos + var_length, 'ref': ref, 'alt': variant_json.get('alt'), + 'sv_type': variant_json.get('svType'), 'family_id': family_id, 'variant_id': variant_json['variantId'], 'genotypes': variant_json.get('genotypes', {}), 'dataset_type': variant_json.get('dataset_type') or variant_dataset_type(variant_json), 'gene_ids': gene_ids, + 'main_transcript': main_transcript, 'saved_variant_json': variant_json.get('saved_variant_json', {}), } -def _transcript_sort(gene_id, saved_variant_json): +def _transcript_sort(gene_id, saved_variant_json, main_transcript_id): gene_transcripts = saved_variant_json['transcripts'][gene_id] - main_transcript_id = saved_variant_json.get('mainTranscriptId') is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) @@ -170,7 +179,7 @@ def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], g variant_ids = None qs = get_variants_queryset( genome_version, dataset_type, keys=keys, variant_ids=variant_ids, - ) + ).join_variant_id() key_field = 'variantId' if primary_id_field == 'variant_id' else primary_id_field variant_fields = ['key'] variant_values = { @@ -181,10 +190,18 @@ def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], g ), } if dataset_type.startswith('SV'): - variant_fields += ['chrom', 'pos', 'end'] + variant_fields += ['chrom', 'pos', 'end', 'sv_type'] + variant_values.update({ + 'svType': F('sv_type'), + 'main_transcript': Value(None, output_field=StringField()), + }) else: variant_values.update(qs.split_variant_id_annotations()) - return {v[group_by_field or key_field]: v for v in qs.join_variant_id().values(*variant_fields, **variant_values)} + if qs.variant_detail_field: + variant_values['main_transcript'] = F(f'{qs.variant_detail_field}__transcripts__0') + else: + variant_values['main_transcript'] = F('sorted_transcript_consequences__0') + return {v[group_by_field or key_field]: v for v in qs.values(*variant_fields, **variant_values)} def reset_cached_search_results(project, reset_index_metadata=False): From a1ee74bcfc9fcc958e981f27768e9a943d5e80b1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 19 Mar 2026 17:52:11 -0400 Subject: [PATCH 223/252] properly update main transcript --- seqr/views/apis/saved_variant_api.py | 6 ++++-- seqr/views/apis/saved_variant_api_tests.py | 4 +++- ui/redux/rootReducer.js | 6 +++--- ui/shared/components/panel/variants/Transcripts.jsx | 6 +++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 20d106bd7f..3a1b289226 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -345,8 +345,10 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) check_project_permissions(saved_variant.family.project, request.user, can_edit=True) - # TODO - update_model_from_json(saved_variant, {'selected_main_transcript_id': transcript_id}, request.user) + update_model_from_json(saved_variant, { + 'selected_main_transcript_id': transcript_id, + 'main_transcript': json.loads(request.body), + }, request.user) return create_json_response({'savedVariantsByGuid': {variant_guid: {'selectedMainTranscriptId': transcript_id}}}) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 04a40ac9fd..80549e3dc1 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -1094,13 +1094,15 @@ def test_update_variant_main_transcript(self): update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) self.check_manager_login(update_main_transcript_url) - response = self.client.post(update_main_transcript_url) + transcript = {'transcriptId': transcript_id, 'majorConsequence': 'intron_variant'} + response = self.client.post(update_main_transcript_url, content_type='application/json', data=json.dumps(transcript)) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'savedVariantsByGuid': {VARIANT_GUID: {'selectedMainTranscriptId': transcript_id}}}) saved_variants = SavedVariant.objects.filter(guid=VARIANT_GUID) self.assertEqual(len(saved_variants), 1) self.assertEqual(saved_variants.first().selected_main_transcript_id, transcript_id) + self.assertDictEqual(saved_variants.first().main_transcript, transcript) self.assertEqual(get_json_for_saved_variants(saved_variants, add_details=True)[0]['selectedMainTranscriptId'], transcript_id) def test_update_variant_acmg_classification(self): diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index b93f60411d..d28acefe1f 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -225,12 +225,12 @@ export const updateVariantTags = (values, tagType = 'tags') => { export const updateVariantClassification = values => updateSavedVariant(values, `${values.variant.variantGuid}/update_acmg_classification`) -export const updateVariantMainTranscript = (variantGuid, transcriptId) => dispatch => new HttpRequestHelper( - `/api/saved_variant/${variantGuid}/update_transcript/${transcriptId}`, +export const updateVariantMainTranscript = (variantGuid, transcript) => dispatch => new HttpRequestHelper( + `/api/saved_variant/${variantGuid}/update_transcript/${transcript.transcriptId}`, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) }, -).post() +).post(transcript) export const updateLocusList = values => (dispatch) => { let action = 'create' diff --git a/ui/shared/components/panel/variants/Transcripts.jsx b/ui/shared/components/panel/variants/Transcripts.jsx index 5406ebbe5b..c049d9dae8 100644 --- a/ui/shared/components/panel/variants/Transcripts.jsx +++ b/ui/shared/components/panel/variants/Transcripts.jsx @@ -143,7 +143,7 @@ const transcriptIdDetails = (transcript, variant, { transcriptsById, project, up transcript.transcriptId === variant.selectedMainTranscriptId ?