From 2a3cdcf0674bd3bce1119136f0a50052f68f12cf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 Aug 2025 14:57:37 -0400 Subject: [PATCH 001/109] fix gene search dataset type for clickhouse --- seqr/utils/search/search_utils_tests.py | 28 ++++++++++++++++++------- seqr/utils/search/utils.py | 2 +- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 681b438f3f..746dde4d13 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -310,7 +310,7 @@ def test_invalid_search_query_variants(self): def _test_expected_search_call(self, mock_get_variants, results_cache, search_fields=None, has_gene_search=False, rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', dataset_type=None, secondary_dataset_type=None, omitted_sample_guids=None, - exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, **kwargs): + exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): expected_search = { 'inheritance_mode': inheritance_mode, 'inheritance_filter': {}, @@ -335,6 +335,9 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi if has_gene_search: gene_ids = ['ENSG00000186092', 'ENSG00000227232'] intervals = [['2', 1234, 5678], ['7', 1, 11100], ['1', 14404, 29570], ['1', 65419, 71585]] + if single_gene_search: + gene_ids = gene_ids[1:] + intervals = intervals[2:3] self._assert_expected_search_locus( mock_get_variants.call_args.args[1], dataset_type='MITO_missing' if has_included_gene_search else dataset_type, gene_ids=gene_ids, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, @@ -417,15 +420,23 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v rs_ids=['rs9876'], variant_ids=[], parsed_variant_ids=[], omitted_sample_guids=SV_SAMPLES, dataset_type='SNV_INDEL', ) - self.search_model.search['locus']['rawItems'] = 'WASH7P, chr2:1234-5678, chr7:100-10100%10, ENSG00000186092' + locus_items = 'WASH7P, chr2:1234-5678, chr7:100-10100%10, ENSG00000186092' + self.search_model.search['locus']['rawItems'] = locus_items query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, has_gene_search=True, ) - locus = self.search_model.search.pop('locus') - self.search_model.search['exclude'] = {'clinvar': ['benign'], 'rawItems': locus['rawItems']} + self.search_model.search['locus']['rawItems'] = 'WASH7P' + query_variants(self.results_model, user=self.user) + self._test_expected_search_call( + mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, + has_gene_search=True, single_gene_search=True, + ) + + del self.search_model.search['locus'] + self.search_model.search['exclude'] = {'clinvar': ['benign'], 'rawItems': locus_items} query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, @@ -539,7 +550,7 @@ def _assert_expected_search_locus(self, search_body, dataset_type, gene_ids=None intervals = [ {'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}, - ] + ] if len(gene_ids) > 1 else [] dataset_type = None if dataset_type == 'MITO_missing' else dataset_type super()._assert_expected_search_locus( @@ -551,7 +562,8 @@ def _assert_expected_search_locus(self, search_body, dataset_type, gene_ids=None for gene in parsed_genes.values(): self.assertSetEqual(set(gene.keys()), GENE_FIELDS) self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') - self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') + if len(gene_ids) > 1: + self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) @@ -710,8 +722,8 @@ def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, ex gene_ids = None if exclude_locations else gene_ids gene_intervals = None if gene_ids: - gene_intervals = intervals[2:] - intervals = intervals[:2] + gene_intervals = intervals[2:] if len(gene_ids) > 1 else intervals + intervals = intervals[:2] if len(gene_ids) > 1 else None super()._assert_expected_search_locus( *args, gene_ids=gene_ids, gene_intervals=gene_intervals, intervals=intervals, variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 4b1e01d86d..cb8fcf792e 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -487,7 +487,7 @@ def _search_dataset_type(search): lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type - intervals = locus['intervals'] if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None + intervals = (locus['intervals'] or locus.get('gene_intervals')) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None dataset_type = _annotation_dataset_type(search.get('annotations'), intervals, pathogenicity=search.get('pathogenicity')) secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], intervals) if search.get('annotations_secondary') else None From 60461453150c981558f86beacf0303e1e9aa7518 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 10:38:08 -0400 Subject: [PATCH 002/109] codacy fixes --- .../tests/check_bam_cram_paths_tests.py | 4 +-- ...eck_for_new_samples_from_pipeline_tests.py | 6 ++-- ...fer_families_to_different_project_tests.py | 1 - seqr/models.py | 32 +++++++++---------- seqr/views/apis/anvil_workspace_api.py | 22 ++++++------- seqr/views/apis/anvil_workspace_api_tests.py | 14 +++----- seqr/views/apis/data_manager_api_tests.py | 2 +- seqr/views/utils/airflow_utils.py | 2 +- seqr/views/utils/export_utils.py | 2 -- seqr/views/utils/orm_to_json_utils_tests.py | 2 +- seqr/views/utils/variant_utils.py | 2 +- 11 files changed, 40 insertions(+), 49 deletions(-) diff --git a/seqr/management/tests/check_bam_cram_paths_tests.py b/seqr/management/tests/check_bam_cram_paths_tests.py index d8105f1931..89ee7b0aa5 100644 --- a/seqr/management/tests/check_bam_cram_paths_tests.py +++ b/seqr/management/tests/check_bam_cram_paths_tests.py @@ -46,8 +46,8 @@ def _check_results(self, did_delete, mock_logger, mock_safe_post_to_slack, mock_ self.assertListEqual(sorted(igv_file_paths), expected_remaining_files) mock_subprocess.assert_has_calls([ - mock.call('gsutil ls gs://readviz/NA20870.cram', stdout=-1, stderr=-2, shell=True), - mock.call('gsutil ls gs://datasets-gcnv/NA20870.bed.gz', stdout=-1, stderr=-2, shell=True), + mock.call('gsutil ls gs://readviz/NA20870.cram', stdout=-1, stderr=-2, shell=True), # nosec + mock.call('gsutil ls gs://datasets-gcnv/NA20870.bed.gz', stdout=-1, stderr=-2, shell=True), # nosec ], any_order=True) calls = [ diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 53294bd9c5..550fdc393c 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -45,7 +45,7 @@ f'We are following up on the request to load data from AnVIL on March 12, 2017.
' \ f'We have loaded 1 new WES samples from the AnVIL workspace {anvil_link} to the corresponding seqr project {seqr_link}.' \ f'
Let us know if you have any questions.

All the best,
The seqr team' -ANVIL_ERROR_TEXT_EMAIL = f"""Dear seqr user, +ANVIL_ERROR_TEXT_EMAIL = """Dear seqr user, We are following up on the request to load data from AnVIL workspace ext-data/empty on March 12, 2017. This request could not be loaded due to the following error(s): - Missing the following expected contigs:chr17 @@ -1018,7 +1018,7 @@ def _set_empty_loading_files(self): def _assert_has_expected_empty_list_file_calls(self): self.mock_subprocess.assert_called_with( - 'gsutil ls gs://seqr-hail-search-data/v3.1/GRCh37/MITO/runs/*/*', stdout=-1, stderr=-1, shell=True + 'gsutil ls gs://seqr-hail-search-data/v3.1/GRCh37/MITO/runs/*/*', stdout=-1, stderr=-1, shell=True # nosec ) def _set_reloading_loading_files(self): @@ -1047,7 +1047,7 @@ def _assert_expected_loading_file_calls(self, single_call): ('gsutil mv /mock/tmp/* gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', -2), ] self.mock_subprocess.assert_has_calls( - [mock.call(command, stdout=-1, stderr=stderr, shell=True) for (command, stderr) in calls] + [mock.call(command, stdout=-1, stderr=stderr, shell=True) for (command, stderr) in calls] # nosec ) def _additional_loading_logs(self, data_type, version): diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index de7d9e7db0..d2c6c78f2c 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -1,7 +1,6 @@ import responses from django.core.management import call_command import mock -import json from seqr.models import Family, VariantTagType, VariantTag, Sample from seqr.views.utils.test_utils import AirflowTestCase, AuthenticationTestCase diff --git a/seqr/models.py b/seqr/models.py index 180ad8e04f..4f4951e90d 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -6,7 +6,7 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import PermissionDenied, ValidationError from django.db import models -from django.db.models import base, options, ForeignKey, JSONField, prefetch_related_objects +from django.db.models import base, options, prefetch_related_objects from django.utils import timezone from django.utils.text import slugify as __slugify @@ -340,7 +340,7 @@ class Family(ModelWithGUID): description = models.TextField(null=True, blank=True) pedigree_image = models.ImageField(null=True, blank=True, upload_to='pedigree_images') - pedigree_dataset = JSONField(null=True, blank=True) + pedigree_dataset = models.JSONField(null=True, blank=True) assigned_analyst = models.ForeignKey(User, null=True, on_delete=models.SET_NULL, related_name='assigned_families') # type: ForeignKey @@ -639,19 +639,19 @@ class Individual(ModelWithGUID): expected_inheritance = ArrayField(models.CharField(max_length=1, choices=INHERITANCE_CHOICES), null=True) # features are objects with an id field for HPO id and optional notes and qualifiers fields - features = JSONField(null=True) - absent_features = JSONField(null=True) + features = models.JSONField(null=True) + absent_features = models.JSONField(null=True) # nonstandard_features are objects with an id field for a free text label and optional # notes, qualifiers, and categories fields - nonstandard_features = JSONField(null=True) - absent_nonstandard_features = JSONField(null=True) + nonstandard_features = models.JSONField(null=True) + absent_nonstandard_features = models.JSONField(null=True) # Disorders are a list of MIM IDs disorders = ArrayField(models.CharField(max_length=10), null=True) # genes are objects with required key gene (may be blank) and optional key comments - candidate_genes = JSONField(null=True) - rejected_genes = JSONField(null=True) + candidate_genes = models.JSONField(null=True) + rejected_genes = models.JSONField(null=True) ar_fertility_meds = models.BooleanField(null=True) ar_iui = models.BooleanField(null=True) @@ -661,10 +661,10 @@ class Individual(ModelWithGUID): ar_donoregg = models.BooleanField(null=True) ar_donorsperm = models.BooleanField(null=True) - filter_flags = JSONField(null=True) - pop_platform_filters = JSONField(null=True) + filter_flags = models.JSONField(null=True) + pop_platform_filters = models.JSONField(null=True) population = models.CharField(max_length=5, null=True) - sv_flags = JSONField(null=True) + sv_flags = models.JSONField(null=True) def __unicode__(self): return self.individual_id.strip() @@ -839,11 +839,11 @@ class SavedVariant(ModelWithGUID): key = models.PositiveBigIntegerField(null=True, blank=True) selected_main_transcript_id = models.CharField(max_length=20, null=True) - saved_variant_json = JSONField(default=dict) - genotypes = JSONField(default=dict) + saved_variant_json = models.JSONField(default=dict) + genotypes = models.JSONField(default=dict) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True) - acmg_classification = JSONField(null=True) # ACMG based classification + acmg_classification = models.JSONField(null=True) # ACMG based classification def __unicode__(self): chrom, pos = get_chrom_pos(self.xpos) @@ -1120,7 +1120,7 @@ class Meta: class DynamicAnalysisGroup(ModelWithGUID): project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, blank=True) name = models.TextField() - criteria = JSONField() + criteria = models.JSONField() def __unicode__(self): return self.name.strip() @@ -1136,7 +1136,7 @@ class Meta: class VariantSearch(ModelWithGUID): name = models.CharField(max_length=200, null=True) order = models.FloatField(null=True, blank=True) - search = JSONField() + search = models.JSONField() def __unicode__(self): return self.name or str(self.id) diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 023832e187..9de58abb3f 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -3,7 +3,6 @@ import time from datetime import datetime from functools import wraps -from collections import defaultdict from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.views import redirect_to_login @@ -271,9 +270,11 @@ def _trigger_add_workspace_data(project, pedigree_records, user, data_path, samp # use airflow api to trigger AnVIL dags reload_summary = f' and {len(previous_loaded_ids)} re-loaded' if previous_loaded_ids else '' - success_message = f""" - *{user.email}* requested to load {num_updated_individuals} new{reload_summary} {sample_type} samples ({GENOME_VERSION_LOOKUP.get(project.genome_version)}) from AnVIL workspace *{project.workspace_namespace}/{project.workspace_name}* at - {data_path} to seqr project <{_get_seqr_project_url(project)}|*{project.name}*> (guid: {project.guid})""" + success_message = ( + f"*{user.email}* requested to load {num_updated_individuals} new{reload_summary} {sample_type} samples " + f"({GENOME_VERSION_LOOKUP.get(project.genome_version)}) from AnVIL workspace *{project.workspace_namespace}/{project.workspace_name}* at " + f"{data_path} to seqr project <{_get_seqr_project_url(project)}|*{project.name}*> (guid: {project.guid})" + ) trigger_success = trigger_airflow_data_loading( [project], individual_ids, sample_type, Sample.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message, success_slack_channel=SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, error_message=f'ERROR triggering AnVIL loading for project {project.guid}', @@ -291,13 +292,12 @@ def _trigger_add_workspace_data(project, pedigree_records, user, data_path, samp loading_warning_date = ANVIL_LOADING_DELAY_EMAIL_START_DATE and datetime.strptime(ANVIL_LOADING_DELAY_EMAIL_START_DATE, '%Y-%m-%d') if loading_warning_date and loading_warning_date <= datetime.now(): try: - email_body = f"""Hi {user.get_full_name() or user.email}, - We have received your request to load data to seqr from AnVIL. Currently, the Broad Institute is holding an - internal retreat or closed for the winter break so we may not be able to load data until mid-January - {loading_warning_date.year + 1}. We appreciate your understanding and support of our research team taking - some well-deserved time off and hope you also have a nice break. - - The seqr team - """ + email_body = (f"Hi {user.get_full_name() or user.email},\n" + "We have received your request to load data to seqr from AnVIL. Currently, the Broad Institute is holding an " + "internal retreat or closed for the winter break so we may not be able to load data until mid-January " + f"{loading_warning_date.year + 1}. We appreciate your understanding and support of our research team taking " + "some well-deserved time off and hope you also have a nice break.\n" + "- The seqr team") send_html_email(email_body, subject='Delay in loading AnVIL in seqr', to=[user.email]) except Exception as e: logger.error('AnVIL loading delay email error: {}'.format(e), user) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 957f687115..ab6c6d60a3 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -823,9 +823,7 @@ def _assert_valid_operation(self, project, test_add_data=True): 'sample_source': 'AnVIL', } sample_summary = '13 new and 7 re-loaded' if test_add_data else '3 new' - slack_message = """ - *test_user_manager@test.com* requested to load {sample_summary} WES samples ({version}) from AnVIL workspace *my-seqr-billing/{workspace_name}* at - gs://test_bucket/test_path.vcf to seqr project (guid: {guid}) + slack_message = """*test_user_manager@test.com* requested to load {sample_summary} WES samples ({version}) from AnVIL workspace *my-seqr-billing/{workspace_name}* at gs://test_bucket/test_path.vcf to seqr project (guid: {guid}) Pedigree files have been uploaded to gs://seqr-loading-temp/v3.1/{version}/SNV_INDEL/pedigrees/WES @@ -892,7 +890,7 @@ def _test_mv_file_and_triggering_dag_exception(self, url, workspace, sample_data slack_message_on_failure = """ERROR triggering AnVIL loading for project {guid}: LOADING_PIPELINE DAG is running and cannot be triggered again. - DAG LOADING_PIPELINE should be triggered with following: + DAG LOADING_PIPELINE should be triggered with following: ```{dag}``` """.format( guid=project.guid, @@ -969,12 +967,8 @@ def _test_after_email_date(self, url, request_body): response = self.client.post(url, content_type='application/json', data=json.dumps(request_body)) self.assertEqual(response.status_code, 200) self.mock_send_email.assert_called_with("""Hi Test Manager User, - We have received your request to load data to seqr from AnVIL. Currently, the Broad Institute is holding an - internal retreat or closed for the winter break so we may not be able to load data until mid-January - 2022. We appreciate your understanding and support of our research team taking - some well-deserved time off and hope you also have a nice break. - - The seqr team - """, subject='Delay in loading AnVIL in seqr', to=['test_user_manager@test.com']) +We have received your request to load data to seqr from AnVIL. Currently, the Broad Institute is holding an internal retreat or closed for the winter break so we may not be able to load data until mid-January 2022. We appreciate your understanding and support of our research team taking some well-deserved time off and hope you also have a nice break. +- The seqr team""", subject='Delay in loading AnVIL in seqr', to=['test_user_manager@test.com']) self.mock_api_logger.error.assert_called_with( 'AnVIL loading delay email error: Unable to send email', self.manager_user) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 8e14871f70..50a96415fd 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1884,7 +1884,7 @@ def _assert_trigger_error(self, response, body, dag_json, **kwargs): 'WGS', 'WES').replace('SNV_INDEL', 'GCNV').replace('v01', 'v3.1') error_message = f"""ERROR triggering internal WES SV loading: {errors[0]} - DAG LOADING_PIPELINE should be triggered with following: + DAG LOADING_PIPELINE should be triggered with following: ```{dag_json}``` """ self.mock_slack.assert_called_once_with(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, error_message) diff --git a/seqr/views/utils/airflow_utils.py b/seqr/views/utils/airflow_utils.py index 4a8b7030cd..634903a192 100644 --- a/seqr/views/utils/airflow_utils.py +++ b/seqr/views/utils/airflow_utils.py @@ -79,7 +79,7 @@ def _send_load_data_slack_msg(messages: list[str], channel: str, dag: dict): def _send_slack_msg_on_failure_trigger(e, dag, error_message): message_content = f"""{error_message}: {e} - DAG {LOADING_PIPELINE_DAG_NAME} should be triggered with following: + DAG {LOADING_PIPELINE_DAG_NAME} should be triggered with following: ```{json.dumps(dag, indent=4)}``` """ safe_post_to_slack(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, message_content) diff --git a/seqr/views/utils/export_utils.py b/seqr/views/utils/export_utils.py index bd72a50b93..71261e921c 100644 --- a/seqr/views/utils/export_utils.py +++ b/seqr/views/utils/export_utils.py @@ -1,5 +1,3 @@ -from collections import OrderedDict -import json import openpyxl as xl import os from tempfile import NamedTemporaryFile, TemporaryDirectory diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index 4a539a787a..1b553d3f30 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -3,7 +3,7 @@ from copy import deepcopy from seqr.models import Project, Sample, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch from seqr.views.utils.orm_to_json_utils import get_json_for_user, _get_json_for_project, \ - get_json_for_sample, get_json_for_saved_variants, get_json_for_variant_note, get_json_for_locus_list, \ + get_json_for_sample, get_json_for_variant_note, get_json_for_locus_list, \ get_json_for_saved_searches, get_json_for_saved_variants_with_tags, get_json_for_current_user from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ PROJECT_FIELDS, SAMPLE_FIELDS, SAVED_VARIANT_FIELDS, \ diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 6f35ba9ae1..5f71832f7a 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -19,7 +19,7 @@ from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos -from seqr.views.utils.json_to_orm_utils import update_model_from_json, create_model_from_json +from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ get_json_for_matchmaker_submissions From a702076745e7a648f9872c2705c6f06934f0dff5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 12:07:33 -0400 Subject: [PATCH 003/109] explicitly include project guid in call filter --- clickhouse_search/managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 3012a33e2a..c41447da6d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -843,6 +843,7 @@ def _family_calls_q(self, call_q, family_sample_data, sample_filters, sample_typ sample_q = Q( calls__array_exists={**sample_inheritance_filter, **sample_quality_filter}, family_guid=family_sample_data['family_guid'], + project_guid= family_sample_data['project_guid'], ) if filter_sample_type: sample_q &= Q(sample_type=sample_type) From e34f145a65276c1c1c67c6fe2484721148375ce2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 12:52:08 -0400 Subject: [PATCH 004/109] do individual guid mapping outside of clcikhouse --- clickhouse_search/managers.py | 9 +++--- clickhouse_search/search.py | 53 +++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 3012a33e2a..12b2f97f88 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -735,7 +735,7 @@ def result_values(self, sample_data): def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') - def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, **kwargs): + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): project_guids = {s['project_guid'] for s in sample_data} project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) @@ -803,7 +803,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit q |= clinvar_override_q entries = entries.filter(q) - return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, multi_sample_type_families) + return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) @staticmethod def _get_family_sample_types(sample_data): @@ -948,7 +948,7 @@ def _annotate_failed_family_samples(cls, entries, family_sample_gt_filters): ) return entries - def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, multi_sample_type_families=None): + def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None): carriers_expression = self._carriers_expression(sample_data) if annotate_carriers else None if carriers_expression: entries = entries.annotate(carriers=carriers_expression) @@ -970,9 +970,10 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an if self._has_clinvar(): fields += ['clinvar', 'clinvar_key'] if multi_sample_type_families or sample_data is None or len(sample_data) > 1: + genotype_sample_data = None if skip_individual_guid else sample_data entries = entries.values(*fields).annotate( familyGuids=ArraySort(ArrayDistinct(GroupArray('family_guid'))), - **{'genotypes' if sample_data else 'familyGenotypes': GroupArrayArray(self.genotype_expression(sample_data))}, + **{'genotypes' if genotype_sample_data else 'familyGenotypes': GroupArrayArray(self.genotype_expression(genotype_sample_data))}, **{col: GroupArrayArray(col) for col in genotype_override_annotations} ) if carriers_expression: diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 851d3b1a66..f454a894d3 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -45,13 +45,20 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] family_guid = sample_data[0]['family_guid'] + is_multi_project = any(s['project_guid'] != sample_data[0]['project_guid'] for s in sample_data) + skip_individual_guid = is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS + dataset_results = [] if inheritance_mode != COMPOUND_HET: - result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, **search) - results += list(result_q[:MAX_VARIANTS + 1]) + result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) + dataset_results += list(result_q[:MAX_VARIANTS + 1]) if has_comp_het: - result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, **search) - results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) + dataset_results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + + if skip_individual_guid: + _add_individual_guids(dataset_results, sample_data) + results += dataset_results if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type @@ -90,9 +97,15 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b families = snv_indel_families.intersection(sv_families) if not families: continue + + sample_data = [ + s for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] if s['family_guid'] in families + ] + skip_individual_guid = any(s['project_guid'] != sample_data[0]['project_guid'] for s in sample_data) + entries = entry_cls.objects.search([ s for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] if s['family_guid'] in families - ], **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True) + ], skip_individual_guid=skip_individual_guid, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True) snv_indel_q = annotations_cls.objects.subquery_join(entries).search(**search_kwargs, annotations=annotations) sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search([ @@ -102,7 +115,10 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations) result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) - results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + dataset_results = [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + if skip_individual_guid: + _add_individual_guids(dataset_results, sample_data, single_data_type=False) + results += dataset_results return results @@ -177,6 +193,31 @@ def _result_as_tuple(results, field_prefix): return Tuple(*fields.keys(), output_field=NamedTupleField(list(fields.values()))) +def _add_individual_guids(results, sample_data, single_data_type=True): + sample_map = {} + for data in sample_data: + for s in data['samples']: + sample_map.update({ + (data['family_guid'], sample_id): s['individual_guid'] for sample_id in set(s['sample_ids_by_type'].values()) + }) + for result in results: + if isinstance(result, list): + for variant in result: + if single_data_type or 'svType' not in variant: + _set_individual_guids(variant, sample_map) + else: + _set_individual_guids(result, sample_map) + + +def _set_individual_guids(result, sample_map): + result['familyGuids'] = sorted(result['familyGenotypes'].keys()) + result['genotypes'] = {} + for family_guid, genotypes in result.pop('familyGenotypes').items(): + for genotype in genotypes: + individual_guid = sample_map[(genotype['familyGuid'], genotype['sampleId'])] + result['genotypes'][individual_guid] = {**genotype, 'individualGuid': individual_guid} + + def get_clickhouse_cache_results(results, sort, family_guid): sort_metadata = _get_sort_gene_metadata(sort, results, family_guid) sort_key = _get_sort_key(sort, sort_metadata) From 7a38ccb3bd36844c430c6d7559a6f1aaa5c50685 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 13:16:53 -0400 Subject: [PATCH 005/109] fix compound hets --- clickhouse_search/managers.py | 6 +++--- clickhouse_search/search.py | 29 ++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 12b2f97f88..0901500283 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -203,7 +203,7 @@ def _get_join_query_values(self, query, alias, conditional_selects): query_select = query.annotation_values for select_func in (conditional_selects or []): query_select.update(select_func(query, prefix=f'{alias}_')) - annotation_fields = query.annotation_fields + self.ENTRY_FIELDS + annotation_fields = query.annotation_fields return query.values( **{f'{alias}_{field}': F(field) for field in annotation_fields if field not in query_select}, **{f'{alias}_{field}': value for field, value in query_select.items()}, @@ -312,14 +312,14 @@ def search_compound_hets(self, primary_q, secondary_q): secondary_q = secondary_q.explode_gene_id(secondary_gene_field) conditional_fields = lambda query, **kwargs: { - field: F(field) for field in [self.SELECTED_GENE_FIELD, 'clinvar', 'family_carriers', 'carriers', 'has_hom_alt', 'no_hom_alt_families'] + field: F(field) for field in [self.SELECTED_GENE_FIELD, 'clinvar', 'family_carriers', 'carriers', 'has_hom_alt', 'no_hom_alt_families', 'familyGenotypes'] + self.ENTRY_FIELDS if field in query.query.annotations } results = self.cross_join( query=primary_q, alias='primary', join_query=secondary_q, join_alias='secondary', conditional_selects=[ - self._conditional_selected_transcript_values, self._genotype_override_values, conditional_fields, + conditional_fields, self._conditional_selected_transcript_values, self._genotype_override_values, ], ) return results.filter( diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f454a894d3..d87690a6e4 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -163,18 +163,33 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ )) if num_families > 1: + primary_family_expr = 'primary_familyGuids' if results.has_annotation('primary_familyGuids') else ArrayMap( + 'primary_familyGenotypes', mapped_expression='x.1', + ) + secondary_family_expr = 'secondary_familyGuids' if results.has_annotation('secondary_familyGuids') else ArrayMap( + 'secondary_familyGenotypes', mapped_expression='x.1', + ) + genotype_expressions = { + 'primary_genotypes': ArrayFilter('primary_genotypes', conditions=[ + {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, + ]), + 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ + {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, + ]), + 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ + {1: (None, 'arrayExists(g -> g.1 = {field}, secondary_familyGenotypes)')}, + ]), + 'secondary_familyGenotypes': ArrayFilter('secondary_familyGenotypes', conditions=[ + {1: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, + ]), + } results = results.annotate( primary_familyGuids=ArrayIntersect( - 'primary_familyGuids', 'secondary_familyGuids', output_field=ArrayField(StringField()), + primary_family_expr, secondary_family_expr, output_field=ArrayField(StringField()), ), ).filter(primary_familyGuids__not_empty=True).annotate( secondary_familyGuids=F('primary_familyGuids'), - primary_genotypes=ArrayFilter('primary_genotypes', conditions=[ - {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, - ]), - secondary_genotypes=ArrayFilter('secondary_genotypes', conditions=[ - {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, - ]), + **{k: v for k, v in genotype_expressions.items() if results.has_annotation(k)}, ) return results.annotate( From de40cf0d81be6745b9c27d0f06e416d74b434807 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 13:24:23 -0400 Subject: [PATCH 006/109] properly handle multi sample types --- clickhouse_search/search.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index d87690a6e4..1380362e1d 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -226,11 +226,12 @@ def _add_individual_guids(results, sample_data, single_data_type=True): def _set_individual_guids(result, sample_map): result['familyGuids'] = sorted(result['familyGenotypes'].keys()) - result['genotypes'] = {} + individual_genotypes = defaultdict(list) for family_guid, genotypes in result.pop('familyGenotypes').items(): for genotype in genotypes: individual_guid = sample_map[(genotype['familyGuid'], genotype['sampleId'])] - result['genotypes'][individual_guid] = {**genotype, 'individualGuid': individual_guid} + individual_genotypes[individual_guid].append({**genotype, 'individualGuid': individual_guid}) + result['genotypes'] = {k: v[0] if len(v) == 1 else v for k, v in individual_genotypes.items()} def get_clickhouse_cache_results(results, sort, family_guid): From 37e1d8a544bbf9297d8f1b3685c1ec035dae83d1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 14:12:14 -0400 Subject: [PATCH 007/109] codacy --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1380362e1d..e6145a40a3 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -229,7 +229,7 @@ def _set_individual_guids(result, sample_map): individual_genotypes = defaultdict(list) for family_guid, genotypes in result.pop('familyGenotypes').items(): for genotype in genotypes: - individual_guid = sample_map[(genotype['familyGuid'], genotype['sampleId'])] + individual_guid = sample_map[(family_guid, genotype['sampleId'])] individual_genotypes[individual_guid].append({**genotype, 'individualGuid': individual_guid}) result['genotypes'] = {k: v[0] if len(v) == 1 else v for k, v in individual_genotypes.items()} From f12abb8d06b58988e6891b1b967e949f93868a46 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 17:39:02 -0400 Subject: [PATCH 008/109] first pass group genotype filters --- clickhouse_search/backend/functions.py | 7 ++ clickhouse_search/managers.py | 102 ++++++++++++++++++------- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 85e242665e..1a3300a220 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -95,6 +95,13 @@ def process_rhs(self, compiler, connection): return rhs_params[0], [] +@NestedField.register_lookup +@ArrayField.register_lookup +class ArrayAll(ArrayExists): + lookup_name = "array_all" + function = "arrayAll" + + class ArrayFilter(lookups.Transform): def __init__(self, *args, conditions=None, **kwargs): super().__init__(*args, **kwargs) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 739d7d785b..ba548cf409 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -622,12 +622,20 @@ def has_annotation(self, field): class EntriesManager(SearchQuerySet): + # # TODO used for something else? + # GENOTYPE_LOOKUP = { + # REF_REF: (0,), + # REF_ALT: (1,), + # ALT_ALT: (2,), + # HAS_ALT: (0, '{field} > {value}'), + # HAS_REF: (2, '{field} < {value}'), + # } GENOTYPE_LOOKUP = { - REF_REF: (0,), - REF_ALT: (1,), - ALT_ALT: (2,), - HAS_ALT: (0, '{field} > {value}'), - HAS_REF: (2, '{field} < {value}'), + REF_REF: [0], + REF_ALT: [1], + ALT_ALT: [2], + HAS_ALT: [1, 2], + HAS_REF: [0, 1], } COMP_HET_ALT = 'COMP_HET_ALT' GENOTYPE_LOOKUP[COMP_HET_ALT] = GENOTYPE_LOOKUP[REF_ALT] @@ -762,29 +770,68 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None - call_q = None - multi_sample_type_quality_q = None - multi_sample_type_any_affected_q = None + + gt_map = [] + no_quality_map = [] + quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: - sample_filters, any_affected_samples = self._family_sample_filters(s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected) - if not (sample_filters or any_affected_samples): - continue - if s['family_guid'] in multi_sample_type_families: - multi_sample_type_quality_q, multi_sample_type_any_affected_q = self._multi_sample_type_family_calls_q( - multi_sample_type_quality_q, multi_sample_type_any_affected_q, s, sample_filters, any_affected_samples, - clinvar_override_q, multi_sample_type_families, - ) - else: - sample_type = s['sample_types'][0] - call_q = self._family_calls_q( - call_q, s, sample_filters, sample_type, clinvar_override_q, - any_affected_samples=any_affected_samples, filter_sample_type=not self.single_sample_type - ) + sample_gt_map = [] + no_quality_samples = [] + for sample in s['samples']: + affected = custom_affected.get(sample['individual_guid']) or sample['affected'] + genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) + if genotype is not None: + sample_id = next(iter(sample['sample_ids_by_type'].values())) + sample_gt_map.append(f"'{sample_id}', {self.GENOTYPE_LOOKUP[genotype]}") # TODO NULLABLE_GENOTYPE_LOOKUP + if quality_filter_conditions and quality_filter.get('affected_only') and affected != AFFECTED: + no_quality_samples += list(sample['sample_ids_by_type'].values()) + # TODO any affected + + if sample_gt_map: + gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") + if no_quality_samples: + no_quality_map.append(f"'{s['family_guid']}', {no_quality_samples}") + + if gt_map: + entries = entries.filter(calls__array_all={ + 'gt': (', '.join(gt_map), 'has(map({value})[family_guid][x.sampleId], {field})'), + }) + + if no_quality_map: + quality_filter_conditions = {'OR': [ + quality_filter_conditions, + {'sampleId': (', '.join(no_quality_map), 'has(map({value})[family_guid], {field})')}, + ]} + if quality_filter_conditions: + quality_q = Q(calls__array_all=quality_filter_conditions) + if clinvar_override_q: + quality_q |= clinvar_override_q + entries = entries.filter(quality_q) + + # call_q = None + # multi_sample_type_quality_q = None + # multi_sample_type_any_affected_q = None + # for s in sample_data: + # sample_filters, any_affected_samples = self._family_sample_filters(s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected) + # if not (sample_filters or any_affected_samples): + # continue + # if s['family_guid'] in multi_sample_type_families: + # multi_sample_type_quality_q, multi_sample_type_any_affected_q = self._multi_sample_type_family_calls_q( + # multi_sample_type_quality_q, multi_sample_type_any_affected_q, s, sample_filters, any_affected_samples, + # clinvar_override_q, multi_sample_type_families, + # ) + # else: + # sample_type = s['sample_types'][0] + # call_q = self._family_calls_q( + # call_q, s, sample_filters, sample_type, clinvar_override_q, + # any_affected_samples=any_affected_samples, filter_sample_type=not self.single_sample_type + # ) # With families with multiple sample types, can only filter rows after aggregating filtered_multi_sample_type_families = { family_guid: filters for family_guid, filters in multi_sample_type_families.items() if filters } + # TODO multi sample type if filtered_multi_sample_type_families: multi_type_q = Q(family_guid__in=filtered_multi_sample_type_families.keys()) call_q = (call_q | multi_type_q) if call_q else multi_type_q @@ -794,9 +841,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit else: entries = self._annotate_failed_family_samples(entries, filtered_multi_sample_type_families) - if call_q: - entries = entries.filter(call_q) - if quality_filter.get('vcf_filter'): q = Q(filters__len=0) if clinvar_override_q: @@ -883,8 +927,7 @@ def _multi_sample_type_family_calls_q(self, call_q, any_affected_q, family_sampl return call_q, any_affected_q - def _sample_genotype_filter(self, sample, affected, inheritance_mode, individual_genotype_filter): - sample_filter = {} + def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None if individual_genotype_filter: genotype = individual_genotype_filter.get(sample['individual_guid']) @@ -892,6 +935,11 @@ def _sample_genotype_filter(self, sample, affected, inheritance_mode, individual genotype = self.INHERITANCE_FILTERS.get(inheritance_mode, {}).get(affected) if (inheritance_mode == X_LINKED_RECESSIVE and affected == UNAFFECTED and sample['sex'] in MALE_SEXES): genotype = REF_REF + return genotype + + def _sample_genotype_filter(self, sample, affected, inheritance_mode, individual_genotype_filter): + sample_filter = {} + genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) if genotype: sample_filter['gt'] = self.genotype_lookup[genotype] return sample_filter From f5b9bdfabca4fd4be467b7fb8f8ecbab07ce9190 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 17:49:58 -0400 Subject: [PATCH 009/109] handle unfiltered samples in entry data --- clickhouse_search/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index ba548cf409..b933c4847e 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -794,7 +794,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if gt_map: entries = entries.filter(calls__array_all={ - 'gt': (', '.join(gt_map), 'has(map({value})[family_guid][x.sampleId], {field})'), + 'gt': (', '.join(gt_map), 'or(has(map({value})[family_guid][x.sampleId], {field}), not mapContains(map({value})[family_guid], x.sampleId))'), }) if no_quality_map: From 144738accc530e143d991e6fe771d980584fe043 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 18:09:01 -0400 Subject: [PATCH 010/109] fix gcnv inheritance search --- clickhouse_search/managers.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index b933c4847e..2e08a809da 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -641,11 +641,12 @@ class EntriesManager(SearchQuerySet): GENOTYPE_LOOKUP[COMP_HET_ALT] = GENOTYPE_LOOKUP[REF_ALT] NULLABLE_GENOTYPE_LOOKUP = { **GENOTYPE_LOOKUP, - REF_REF: (0, 'or(isNull({field}), {field} = {value})'), - HAS_REF: (2, 'or(isNull({field}), {field} < {value})'), - HAS_ALT: (0, 'and(isNotNull({field}), {field} > {value})'), + # REF_REF: (0, 'or(isNull({field}), {field} = {value})'), + # HAS_REF: (2, 'or(isNull({field}), {field} < {value})'), + # HAS_ALT: (0, 'and(isNotNull({field}), {field} > {value})'), } NULLABLE_GENOTYPE_LOOKUP[COMP_HET_ALT] = NULLABLE_GENOTYPE_LOOKUP[HAS_ALT] + NULLABLE_GENOTYPES = {REF_REF, HAS_REF} INVALID_NUM_ALT_LOOKUP = { (0,): [1, 2], (1,): [0, 2], @@ -673,6 +674,10 @@ def call_fields(self): def genotype_lookup(self): return self.NULLABLE_GENOTYPE_LOOKUP if self.annotations_model.GENOTYPE_OVERRIDE_FIELDS else self.GENOTYPE_LOOKUP + @property + def nullable_genotypes(self): + return self.NULLABLE_GENOTYPES if self.annotations_model.GENOTYPE_OVERRIDE_FIELDS else [] + @property def quality_filters(self): return [config for config in [('gq', 1), ('ab', 100, 'x.gt != 1'), ('qs', 1), ('hl', 100)] if config[0] in self.call_fields] @@ -772,30 +777,41 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit ) if self._has_clinvar() else None gt_map = [] + null_gt_map = [] no_quality_map = [] quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: sample_gt_map = [] + null_gt_samples = [] no_quality_samples = [] for sample in s['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) if genotype is not None: sample_id = next(iter(sample['sample_ids_by_type'].values())) - sample_gt_map.append(f"'{sample_id}', {self.GENOTYPE_LOOKUP[genotype]}") # TODO NULLABLE_GENOTYPE_LOOKUP + sample_gt_map.append(f"'{sample_id}', {self.genotype_lookup[genotype]}") + if genotype in self.nullable_genotypes: + null_gt_samples.append(sample_id) if quality_filter_conditions and quality_filter.get('affected_only') and affected != AFFECTED: no_quality_samples += list(sample['sample_ids_by_type'].values()) # TODO any affected if sample_gt_map: gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") + if null_gt_samples: + null_gt_map.append(f"'{s['family_guid']}', {null_gt_samples}") if no_quality_samples: no_quality_map.append(f"'{s['family_guid']}', {no_quality_samples}") if gt_map: - entries = entries.filter(calls__array_all={ - 'gt': (', '.join(gt_map), 'or(has(map({value})[family_guid][x.sampleId], {field}), not mapContains(map({value})[family_guid], x.sampleId))'), - }) + gt_map = ', '.join(gt_map) + gt_conditions = [ + (gt_map, 'has(map({value})[family_guid][x.sampleId], {field})'), + (gt_map, 'not mapContains(map({value})[family_guid], x.sampleId)'), + ] + if null_gt_map: + gt_conditions.append((', '.join(null_gt_map), 'and(isNull({field}), has(map({value})[family_guid], x.sampleId))')) + entries = entries.filter(calls__array_all={'OR': [{'gt': condition} for condition in gt_conditions]}) if no_quality_map: quality_filter_conditions = {'OR': [ From 5da61c1560ba2005345c952f64606d74444d0991 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 18:25:11 -0400 Subject: [PATCH 011/109] any affected --- clickhouse_search/managers.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 2e08a809da..bfbd40ec1e 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -778,28 +778,33 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit gt_map = [] null_gt_map = [] + affected_sample_map = [] no_quality_map = [] quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: sample_gt_map = [] null_gt_samples = [] + any_affected_samples = [] no_quality_samples = [] for sample in s['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) + sample_id = next(iter(sample['sample_ids_by_type'].values())) if genotype is not None: - sample_id = next(iter(sample['sample_ids_by_type'].values())) sample_gt_map.append(f"'{sample_id}', {self.genotype_lookup[genotype]}") if genotype in self.nullable_genotypes: null_gt_samples.append(sample_id) + elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: + any_affected_samples.append(sample_id) if quality_filter_conditions and quality_filter.get('affected_only') and affected != AFFECTED: - no_quality_samples += list(sample['sample_ids_by_type'].values()) - # TODO any affected + no_quality_samples.append(sample_id) if sample_gt_map: gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") if null_gt_samples: null_gt_map.append(f"'{s['family_guid']}', {null_gt_samples}") + if any_affected_samples: + affected_sample_map.append(f"'{s['family_guid']}', {any_affected_samples}") if no_quality_samples: no_quality_map.append(f"'{s['family_guid']}', {no_quality_samples}") @@ -812,6 +817,11 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if null_gt_map: gt_conditions.append((', '.join(null_gt_map), 'and(isNull({field}), has(map({value})[family_guid], x.sampleId))')) entries = entries.filter(calls__array_all={'OR': [{'gt': condition} for condition in gt_conditions]}) + elif affected_sample_map: + entries = entries.filter(calls__array_exists={ + 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), + 'sampleId': (', '.join(affected_sample_map), 'has(map({value})[family_guid], {field})'), + }) if no_quality_map: quality_filter_conditions = {'OR': [ From 52700a9fac2811d707177416ca7449e9568f3ac3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 18:30:18 -0400 Subject: [PATCH 012/109] clean up --- clickhouse_search/managers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index bfbd40ec1e..8fcfba45bc 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -811,12 +811,15 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if gt_map: gt_map = ', '.join(gt_map) gt_conditions = [ - (gt_map, 'has(map({value})[family_guid][x.sampleId], {field})'), - (gt_map, 'not mapContains(map({value})[family_guid], x.sampleId)'), + {'gt': (gt_map, 'has(map({value})[family_guid][x.sampleId], {field})')}, + {'sampleId': (gt_map, 'not mapContains(map({value})[family_guid], {field})')}, ] if null_gt_map: - gt_conditions.append((', '.join(null_gt_map), 'and(isNull({field}), has(map({value})[family_guid], x.sampleId))')) - entries = entries.filter(calls__array_all={'OR': [{'gt': condition} for condition in gt_conditions]}) + gt_conditions.append({ + 'gt': (None, 'isNull({field})'), + 'sampleId': (', '.join(null_gt_map), 'has(map({value})[family_guid], {field})'), + }) + entries = entries.filter(calls__array_all={'OR': gt_conditions}) elif affected_sample_map: entries = entries.filter(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), From 6307bb55dd2537dc29458e85f70b087ab9d73cfd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 12 Aug 2025 18:45:57 -0400 Subject: [PATCH 013/109] handle unfiltered quality samples --- clickhouse_search/managers.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8fcfba45bc..6cbbb2b47a 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -779,13 +779,13 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit gt_map = [] null_gt_map = [] affected_sample_map = [] - no_quality_map = [] + quality_samples_map = [] quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: sample_gt_map = [] null_gt_samples = [] any_affected_samples = [] - no_quality_samples = [] + quality_samples = [] for sample in s['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) @@ -796,8 +796,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit null_gt_samples.append(sample_id) elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: any_affected_samples.append(sample_id) - if quality_filter_conditions and quality_filter.get('affected_only') and affected != AFFECTED: - no_quality_samples.append(sample_id) + if (not quality_filter.get('affected_only')) or affected == AFFECTED: + quality_samples.append(sample_id) if sample_gt_map: gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") @@ -805,8 +805,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit null_gt_map.append(f"'{s['family_guid']}', {null_gt_samples}") if any_affected_samples: affected_sample_map.append(f"'{s['family_guid']}', {any_affected_samples}") - if no_quality_samples: - no_quality_map.append(f"'{s['family_guid']}', {no_quality_samples}") + if quality_samples: + quality_samples_map.append(f"'{s['family_guid']}', {quality_samples}") if gt_map: gt_map = ', '.join(gt_map) @@ -826,12 +826,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit 'sampleId': (', '.join(affected_sample_map), 'has(map({value})[family_guid], {field})'), }) - if no_quality_map: - quality_filter_conditions = {'OR': [ - quality_filter_conditions, - {'sampleId': (', '.join(no_quality_map), 'has(map({value})[family_guid], {field})')}, - ]} if quality_filter_conditions: + if quality_samples_map: + quality_filter_conditions = {'OR': [ + quality_filter_conditions, + {'sampleId': (', '.join(quality_samples_map), 'not has(map({value})[family_guid], {field})')} + ]} quality_q = Q(calls__array_all=quality_filter_conditions) if clinvar_override_q: quality_q |= clinvar_override_q From e9c71e0e0bc9059d573c2e917c6d3bdf1d40c700 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 10:53:51 -0400 Subject: [PATCH 014/109] include sample type in inheritance filter --- clickhouse_search/managers.py | 60 ++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 6cbbb2b47a..d447377097 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -782,55 +782,79 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_samples_map = [] quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: - sample_gt_map = [] - null_gt_samples = [] - any_affected_samples = [] - quality_samples = [] + sample_gt_map = defaultdict(list) + null_gt_samples = defaultdict(list) + any_affected_samples = defaultdict(list) + quality_samples = defaultdict(list) for sample in s['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) - sample_id = next(iter(sample['sample_ids_by_type'].values())) - if genotype is not None: - sample_gt_map.append(f"'{sample_id}', {self.genotype_lookup[genotype]}") - if genotype in self.nullable_genotypes: - null_gt_samples.append(sample_id) - elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: - any_affected_samples.append(sample_id) - if (not quality_filter.get('affected_only')) or affected == AFFECTED: - quality_samples.append(sample_id) + for sample_type, sample_id in sample['sample_ids_by_type'].items(): + if genotype is not None: + sample_gt_map[sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") + if genotype in self.nullable_genotypes: + null_gt_samples[sample_type].append(sample_id) + elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: + any_affected_samples[sample_type].append(sample_id) + if (not quality_filter.get('affected_only')) or affected == AFFECTED: + quality_samples[sample_type].append(sample_id) if sample_gt_map: + if self.single_sample_type: + sample_gt_map = list(next(iter(sample_gt_map.values()))) + else: + sample_gt_map = [ + f"'{sample_type}', map({', '.join(sample_gts)})" for sample_type, sample_gts in sample_gt_map.items() + ] gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") if null_gt_samples: + if self.single_sample_type: + null_gt_samples = list(next(iter(null_gt_samples.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in null_gt_samples.items()] + null_gt_samples = f"map({', '.join(sample_map)})" null_gt_map.append(f"'{s['family_guid']}', {null_gt_samples}") if any_affected_samples: + if self.single_sample_type: + any_affected_samples = list(next(iter(any_affected_samples.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in any_affected_samples.items()] + any_affected_samples = f"map({', '.join(sample_map)})" affected_sample_map.append(f"'{s['family_guid']}', {any_affected_samples}") if quality_samples: + if self.single_sample_type: + quality_samples = list(next(iter(quality_samples.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in quality_samples.items()] + quality_samples = f"map({', '.join(sample_map)})" quality_samples_map.append(f"'{s['family_guid']}', {quality_samples}") + map_template = 'map({value})[family_guid]' + if not self.single_sample_type: + map_template += '[sample_type::String]' if gt_map: gt_map = ', '.join(gt_map) gt_conditions = [ - {'gt': (gt_map, 'has(map({value})[family_guid][x.sampleId], {field})')}, - {'sampleId': (gt_map, 'not mapContains(map({value})[family_guid], {field})')}, + {'gt': (gt_map, 'has(' + map_template + '[x.sampleId], {field})')}, + {'sampleId': (gt_map, 'not mapContains(' + map_template + ', {field})')}, ] if null_gt_map: gt_conditions.append({ 'gt': (None, 'isNull({field})'), - 'sampleId': (', '.join(null_gt_map), 'has(map({value})[family_guid], {field})'), + 'sampleId': (', '.join(null_gt_map), 'has(' + map_template + ', {field})'), }) entries = entries.filter(calls__array_all={'OR': gt_conditions}) elif affected_sample_map: entries = entries.filter(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (', '.join(affected_sample_map), 'has(map({value})[family_guid], {field})'), + 'sampleId': (', '.join(affected_sample_map), 'has(' + map_template + ', {field})'), }) if quality_filter_conditions: if quality_samples_map: quality_filter_conditions = {'OR': [ quality_filter_conditions, - {'sampleId': (', '.join(quality_samples_map), 'not has(map({value})[family_guid], {field})')} + {'sampleId': (', '.join(quality_samples_map), 'not has(' + map_template + ', {field})')} ]} quality_q = Q(calls__array_all=quality_filter_conditions) if clinvar_override_q: From 709e43a0705847f7dc5f796419004284fbf14c0e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 11:09:28 -0400 Subject: [PATCH 015/109] cleaner map creation --- clickhouse_search/managers.py | 87 +++++++++++++++++------------------ 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index d447377097..e883e4bd18 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -776,81 +776,80 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None - gt_map = [] - null_gt_map = [] - affected_sample_map = [] - quality_samples_map = [] + family_sample_gts = defaultdict(lambda: defaultdict(list)) + family_sample_null_gts = defaultdict(lambda: defaultdict(list)) + family_affected_samples = defaultdict(lambda: defaultdict(list)) + family_quality_samples = defaultdict(lambda: defaultdict(list)) quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: - sample_gt_map = defaultdict(list) - null_gt_samples = defaultdict(list) - any_affected_samples = defaultdict(list) - quality_samples = defaultdict(list) for sample in s['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) for sample_type, sample_id in sample['sample_ids_by_type'].items(): if genotype is not None: - sample_gt_map[sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") + family_sample_gts[s['family_guid']][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") if genotype in self.nullable_genotypes: - null_gt_samples[sample_type].append(sample_id) + family_sample_null_gts[s['family_guid']][sample_type].append(sample_id) elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: - any_affected_samples[sample_type].append(sample_id) + family_affected_samples[s['family_guid']][sample_type].append(sample_id) if (not quality_filter.get('affected_only')) or affected == AFFECTED: - quality_samples[sample_type].append(sample_id) + family_quality_samples[s['family_guid']][sample_type].append(sample_id) - if sample_gt_map: + map_template = 'map({value})[family_guid]' + if not self.single_sample_type: + map_template += '[sample_type::String]' + if family_sample_gts: + gt_map = [] + for family_guid, sample_type_map in family_sample_gts.items(): if self.single_sample_type: - sample_gt_map = list(next(iter(sample_gt_map.values()))) + sample_gt_map = list(next(iter(sample_type_map.values()))) else: sample_gt_map = [ - f"'{sample_type}', map({', '.join(sample_gts)})" for sample_type, sample_gts in sample_gt_map.items() + f"'{sample_type}', map({', '.join(sample_gts)})" for sample_type, sample_gts in sample_type_map.items() ] - gt_map.append(f"'{s['family_guid']}', map({', '.join(sample_gt_map)})") - if null_gt_samples: - if self.single_sample_type: - null_gt_samples = list(next(iter(null_gt_samples.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in null_gt_samples.items()] - null_gt_samples = f"map({', '.join(sample_map)})" - null_gt_map.append(f"'{s['family_guid']}', {null_gt_samples}") - if any_affected_samples: - if self.single_sample_type: - any_affected_samples = list(next(iter(any_affected_samples.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in any_affected_samples.items()] - any_affected_samples = f"map({', '.join(sample_map)})" - affected_sample_map.append(f"'{s['family_guid']}', {any_affected_samples}") - if quality_samples: - if self.single_sample_type: - quality_samples = list(next(iter(quality_samples.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in quality_samples.items()] - quality_samples = f"map({', '.join(sample_map)})" - quality_samples_map.append(f"'{s['family_guid']}', {quality_samples}") - - map_template = 'map({value})[family_guid]' - if not self.single_sample_type: - map_template += '[sample_type::String]' - if gt_map: + gt_map.append(f"'{family_guid}', map({', '.join(sample_gt_map)})") gt_map = ', '.join(gt_map) gt_conditions = [ {'gt': (gt_map, 'has(' + map_template + '[x.sampleId], {field})')}, {'sampleId': (gt_map, 'not mapContains(' + map_template + ', {field})')}, ] - if null_gt_map: + if family_sample_null_gts: + null_gt_map = [] + for family_guid, sample_type_map in family_sample_null_gts.items(): + if self.single_sample_type: + null_gt_samples = list(next(iter(sample_type_map.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] + null_gt_samples = f"map({', '.join(sample_map)})" + null_gt_map.append(f"'{family_guid}', {null_gt_samples}") gt_conditions.append({ 'gt': (None, 'isNull({field})'), 'sampleId': (', '.join(null_gt_map), 'has(' + map_template + ', {field})'), }) entries = entries.filter(calls__array_all={'OR': gt_conditions}) - elif affected_sample_map: + elif family_affected_samples: + affected_sample_map = [] + for family_guid, sample_type_map in family_affected_samples.items(): + if self.single_sample_type: + any_affected_samples = list(next(iter(sample_type_map.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] + any_affected_samples = f"map({', '.join(sample_map)})" + affected_sample_map.append(f"'{family_guid}', {any_affected_samples}") entries = entries.filter(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), 'sampleId': (', '.join(affected_sample_map), 'has(' + map_template + ', {field})'), }) if quality_filter_conditions: + quality_samples_map = [] + for family_guid, sample_type_map in family_quality_samples.items(): + if self.single_sample_type: + quality_samples = list(next(iter(sample_type_map.values()))) + else: + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] + quality_samples = f"map({', '.join(sample_map)})" + quality_samples_map.append(f"'{family_guid}', {quality_samples}") if quality_samples_map: quality_filter_conditions = {'OR': [ quality_filter_conditions, From cb96f15a8a49cfe1ca71dbd7acc60e4ddcb47cac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 12:16:25 -0400 Subject: [PATCH 016/109] multi sample type --- clickhouse_search/managers.py | 101 ++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index e883e4bd18..145f8d2efa 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -753,10 +753,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) - sample_type_families, multi_sample_type_families = self._get_family_sample_types(sample_data) + sample_type_families, multi_sample_type_families = self._get_family_sample_types(sample_data) # TODO multi_sample_type_families can be a set family_q = None + multi_sample_type_family_q = None if multi_sample_type_families: family_q = Q(family_guid__in=multi_sample_type_families.keys()) + multi_sample_type_family_q = family_q for sample_type, families in sample_type_families.items(): sample_family_q = Q(family_guid__in=families) if not self.single_sample_type: @@ -768,6 +770,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit entries = entries.filter(family_q) + inheritance_q = None + quality_q = None quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') custom_affected = (inheritance_filter or {}).get('affected') or {} @@ -780,6 +784,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit family_sample_null_gts = defaultdict(lambda: defaultdict(list)) family_affected_samples = defaultdict(lambda: defaultdict(list)) family_quality_samples = defaultdict(lambda: defaultdict(list)) + family_missing_type_samples = defaultdict(lambda: defaultdict(list)) quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: for sample in s['samples']: @@ -794,6 +799,10 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit family_affected_samples[s['family_guid']][sample_type].append(sample_id) if (not quality_filter.get('affected_only')) or affected == AFFECTED: family_quality_samples[s['family_guid']][sample_type].append(sample_id) + if s['family_guid'] in multi_sample_type_families and len(sample['sample_ids_by_type']) == 1: + sample_type, sample_id = next(iter(sample['sample_ids_by_type'].items())) + missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + family_missing_type_samples[s['family_guid']][missing_type].append(sample_id) map_template = 'map({value})[family_guid]' if not self.single_sample_type: @@ -826,7 +835,10 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit 'gt': (None, 'isNull({field})'), 'sampleId': (', '.join(null_gt_map), 'has(' + map_template + ', {field})'), }) - entries = entries.filter(calls__array_all={'OR': gt_conditions}) + inheritance_q = Q(calls__array_all={'OR': gt_conditions}) + if multi_sample_type_families: + inheritance_q |= multi_sample_type_family_q + entries = self._annotate_failed_family_samples(entries, gt_map, family_missing_type_samples) elif family_affected_samples: affected_sample_map = [] for family_guid, sample_type_map in family_affected_samples.items(): @@ -836,10 +848,13 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] any_affected_samples = f"map({', '.join(sample_map)})" affected_sample_map.append(f"'{family_guid}', {any_affected_samples}") - entries = entries.filter(calls__array_exists={ + inheritance_q = Q(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), 'sampleId': (', '.join(affected_sample_map), 'has(' + map_template + ', {field})'), }) + if multi_sample_type_families: + entries = entries.annotate(passes_inheritance=inheritance_q) + inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q if quality_filter_conditions: quality_samples_map = [] @@ -858,7 +873,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_q = Q(calls__array_all=quality_filter_conditions) if clinvar_override_q: quality_q |= clinvar_override_q - entries = entries.filter(quality_q) + if multi_sample_type_families: + entries = entries.annotate(passes_quality=quality_q) + quality_q = Q(passes_quality=True) | multi_sample_type_family_q # call_q = None # multi_sample_type_quality_q = None @@ -879,26 +896,22 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit # any_affected_samples=any_affected_samples, filter_sample_type=not self.single_sample_type # ) - # With families with multiple sample types, can only filter rows after aggregating - filtered_multi_sample_type_families = { - family_guid: filters for family_guid, filters in multi_sample_type_families.items() if filters - } - # TODO multi sample type - if filtered_multi_sample_type_families: - multi_type_q = Q(family_guid__in=filtered_multi_sample_type_families.keys()) - call_q = (call_q | multi_type_q) if call_q else multi_type_q - entries = entries.annotate(passes_quality=~multi_type_q | (multi_sample_type_quality_q or Value(True))) - if multi_sample_type_any_affected_q: - entries = entries.annotate(passes_inheritance=~multi_type_q | multi_sample_type_any_affected_q) - else: - entries = self._annotate_failed_family_samples(entries, filtered_multi_sample_type_families) - if quality_filter.get('vcf_filter'): q = Q(filters__len=0) if clinvar_override_q: q |= clinvar_override_q entries = entries.filter(q) + if inheritance_q is not None: + entries = entries.filter(inheritance_q) + elif multi_sample_type_families: + entries = entries.annotate(passes_inheritance=Value(True)) + + if quality_q is not None: + entries = entries.filter(quality_q) + elif multi_sample_type_families: + entries = entries.annotate(passes_quality=Value(True)) + return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) @staticmethod @@ -1013,34 +1026,40 @@ def _sample_quality_filter(self, affected, quality_filter): return sample_filter @classmethod - def _annotate_failed_family_samples(cls, entries, family_sample_gt_filters): - gt_map = [] - missing_sample_map = [] - for family_guid, sample_filters in family_sample_gt_filters.items(): - sample_type_filters = defaultdict(list) - missing_type_samples = defaultdict(list) - for sample_ids_by_type, gt_filter in sample_filters: - for sample_type, sample_id in sample_ids_by_type.items(): - sample_type_filters[sample_type].append(f"'{sample_id}', {cls.INVALID_NUM_ALT_LOOKUP[gt_filter]}") - if len(sample_ids_by_type) == 1: - missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS - missing_type_samples[missing_type].append(sample_id) - sample_type_map = [ - f"'{sample_type}', map({', '.join(type_filters)})" - for sample_type, type_filters in sample_type_filters.items() - ] - gt_map.append(f"'{family_guid}', map({', '.join(sample_type_map)})") - if missing_type_samples: - missing_type_map = [f"'{sample_type}', {samples}" for sample_type, samples in missing_type_samples.items()] - missing_sample_map.append(f"'{family_guid}', map({', '.join(missing_type_map)})") + def _annotate_failed_family_samples(cls, entries, gt_map, family_missing_type_samples): + # gt_map = [] + # missing_sample_map = [] + # for family_guid, sample_filters in family_sample_gt_filters.items(): + # sample_type_filters = defaultdict(list) + # missing_type_samples = defaultdict(list) + # for sample_ids_by_type, gt_filter in sample_filters: + # for sample_type, sample_id in sample_ids_by_type.items(): + # sample_type_filters[sample_type].append(f"'{sample_id}', {cls.INVALID_NUM_ALT_LOOKUP[gt_filter]}") + # if len(sample_ids_by_type) == 1: + # missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + # missing_type_samples[missing_type].append(sample_id) + # sample_type_map = [ + # f"'{sample_type}', map({', '.join(type_filters)})" + # for sample_type, type_filters in sample_type_filters.items() + # ] + # gt_map.append(f"'{family_guid}', map({', '.join(sample_type_map)})") + # if missing_type_samples: + # missing_type_map = [f"'{sample_type}', {samples}" for sample_type, samples in missing_type_samples.items()] + # missing_sample_map.append(f"'{family_guid}', map({', '.join(missing_type_map)})") entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ - 'gt': (', '.join(gt_map), 'has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), + 'gt': (gt_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), + 'sampleId': (gt_map, 'mapContains(map({value})[family_guid][sample_type::String], {field})'), }]), mapped_expression='tuple(family_guid, x.sampleId)', )) - if missing_sample_map: + if family_missing_type_samples: + missing_sample_map = [] + for family_guid, sample_type_map in family_missing_type_samples.items(): + sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] + missing_sample_map.append(f"'{family_guid}', map({', '.join(sample_map)})") + entries = entries.annotate( missing_family_samples=ArrayMap( MapLookup('family_guid', Cast('sample_type', models.StringField()), map_values=', '.join(missing_sample_map)), @@ -1095,7 +1114,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), )) - if any((multi_sample_type_families or {}).values()): + if multi_sample_type_families: entries = self._multi_sample_type_filtered_entries(entries) else: if carriers_expression: From 1ddfa825beea858306d50865fae1ef2486a7660d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 12:27:48 -0400 Subject: [PATCH 017/109] clean up --- clickhouse_search/managers.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 145f8d2efa..55a106386d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -877,18 +877,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit entries = entries.annotate(passes_quality=quality_q) quality_q = Q(passes_quality=True) | multi_sample_type_family_q - # call_q = None - # multi_sample_type_quality_q = None - # multi_sample_type_any_affected_q = None # for s in sample_data: # sample_filters, any_affected_samples = self._family_sample_filters(s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected) - # if not (sample_filters or any_affected_samples): - # continue - # if s['family_guid'] in multi_sample_type_families: - # multi_sample_type_quality_q, multi_sample_type_any_affected_q = self._multi_sample_type_family_calls_q( - # multi_sample_type_quality_q, multi_sample_type_any_affected_q, s, sample_filters, any_affected_samples, - # clinvar_override_q, multi_sample_type_families, - # ) # else: # sample_type = s['sample_types'][0] # call_q = self._family_calls_q( @@ -968,30 +958,6 @@ def _family_calls_q(self, call_q, family_sample_data, sample_filters, sample_typ call_q |= family_sample_q return call_q or family_sample_q - def _multi_sample_type_family_calls_q(self, call_q, any_affected_q, family_sample_data, sample_filters, any_affected_samples, clinvar_override_q, multi_sample_type_families): - sample_quality_filters = [] - for sample_ids_by_type, sample_inheritance_filter, sample_quality_filter in sample_filters: - if sample_inheritance_filter.get('gt'): - multi_sample_type_families[family_sample_data['family_guid']].append( - (sample_ids_by_type, sample_inheritance_filter['gt']) - ) - if sample_quality_filter: - sample_quality_filters.append((sample_ids_by_type, {}, sample_quality_filter)) - if sample_quality_filters: - for sample_type in family_sample_data['sample_types']: - call_q = self._family_calls_q( - call_q, family_sample_data, sample_quality_filters, sample_type, clinvar_override_q, - ) - if any_affected_samples: - multi_sample_type_families[family_sample_data['family_guid']] = True - for sample_type in family_sample_data['sample_types']: - any_affected_q = self._family_calls_q( - any_affected_q, family_sample_data, [], sample_type, clinvar_override_q, - any_affected_samples= any_affected_samples - ) - - return call_q, any_affected_q - def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None if individual_genotype_filter: From ec3d6858adb90e9ca75f0d2d95fff3bd4551cb62 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 13:04:15 -0400 Subject: [PATCH 018/109] clean up --- clickhouse_search/managers.py | 150 ++++++++++++---------------------- 1 file changed, 50 insertions(+), 100 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 55a106386d..442d22f815 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1,3 +1,5 @@ +from email.policy import default + from clickhouse_backend import models from collections import OrderedDict, defaultdict @@ -5,6 +7,7 @@ from django.db.models.expressions import Col from django.db.models.functions import Cast from django.db.models.sql.constants import INNER +from fastparquet.converted_types import nullable from clickhouse_search.backend.fields import NestedField, NamedTupleField from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ @@ -622,14 +625,6 @@ def has_annotation(self, field): class EntriesManager(SearchQuerySet): - # # TODO used for something else? - # GENOTYPE_LOOKUP = { - # REF_REF: (0,), - # REF_ALT: (1,), - # ALT_ALT: (2,), - # HAS_ALT: (0, '{field} > {value}'), - # HAS_REF: (2, '{field} < {value}'), - # } GENOTYPE_LOOKUP = { REF_REF: [0], REF_ALT: [1], @@ -641,19 +636,9 @@ class EntriesManager(SearchQuerySet): GENOTYPE_LOOKUP[COMP_HET_ALT] = GENOTYPE_LOOKUP[REF_ALT] NULLABLE_GENOTYPE_LOOKUP = { **GENOTYPE_LOOKUP, - # REF_REF: (0, 'or(isNull({field}), {field} = {value})'), - # HAS_REF: (2, 'or(isNull({field}), {field} < {value})'), - # HAS_ALT: (0, 'and(isNotNull({field}), {field} > {value})'), + COMP_HET_ALT: GENOTYPE_LOOKUP[HAS_ALT], } - NULLABLE_GENOTYPE_LOOKUP[COMP_HET_ALT] = NULLABLE_GENOTYPE_LOOKUP[HAS_ALT] NULLABLE_GENOTYPES = {REF_REF, HAS_REF} - INVALID_NUM_ALT_LOOKUP = { - (0,): [1, 2], - (1,): [0, 2], - (2,): [0, 1], - (0, '{field} > {value}'): [0], - (2, '{field} < {value}'): [2], - } INHERITANCE_FILTERS = { **INHERITANCE_FILTERS, @@ -808,67 +793,37 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if not self.single_sample_type: map_template += '[sample_type::String]' if family_sample_gts: - gt_map = [] - for family_guid, sample_type_map in family_sample_gts.items(): - if self.single_sample_type: - sample_gt_map = list(next(iter(sample_type_map.values()))) - else: - sample_gt_map = [ - f"'{sample_type}', map({', '.join(sample_gts)})" for sample_type, sample_gts in sample_type_map.items() - ] - gt_map.append(f"'{family_guid}', map({', '.join(sample_gt_map)})") - gt_map = ', '.join(gt_map) + gt_map = self._get_family_sample_map(family_sample_gts, is_nested_map=True) gt_conditions = [ {'gt': (gt_map, 'has(' + map_template + '[x.sampleId], {field})')}, {'sampleId': (gt_map, 'not mapContains(' + map_template + ', {field})')}, ] if family_sample_null_gts: - null_gt_map = [] - for family_guid, sample_type_map in family_sample_null_gts.items(): - if self.single_sample_type: - null_gt_samples = list(next(iter(sample_type_map.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] - null_gt_samples = f"map({', '.join(sample_map)})" - null_gt_map.append(f"'{family_guid}', {null_gt_samples}") + null_gt_map = self._get_family_sample_map(family_sample_null_gts) gt_conditions.append({ 'gt': (None, 'isNull({field})'), - 'sampleId': (', '.join(null_gt_map), 'has(' + map_template + ', {field})'), + 'sampleId': (null_gt_map, 'has(' + map_template + ', {field})'), }) inheritance_q = Q(calls__array_all={'OR': gt_conditions}) if multi_sample_type_families: inheritance_q |= multi_sample_type_family_q entries = self._annotate_failed_family_samples(entries, gt_map, family_missing_type_samples) elif family_affected_samples: - affected_sample_map = [] - for family_guid, sample_type_map in family_affected_samples.items(): - if self.single_sample_type: - any_affected_samples = list(next(iter(sample_type_map.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] - any_affected_samples = f"map({', '.join(sample_map)})" - affected_sample_map.append(f"'{family_guid}', {any_affected_samples}") + affected_sample_map = self._get_family_sample_map(family_affected_samples) inheritance_q = Q(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (', '.join(affected_sample_map), 'has(' + map_template + ', {field})'), + 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), }) if multi_sample_type_families: entries = entries.annotate(passes_inheritance=inheritance_q) inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q if quality_filter_conditions: - quality_samples_map = [] - for family_guid, sample_type_map in family_quality_samples.items(): - if self.single_sample_type: - quality_samples = list(next(iter(sample_type_map.values()))) - else: - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] - quality_samples = f"map({', '.join(sample_map)})" - quality_samples_map.append(f"'{family_guid}', {quality_samples}") + quality_samples_map = self._get_family_sample_map(family_quality_samples) if quality_samples_map: quality_filter_conditions = {'OR': [ quality_filter_conditions, - {'sampleId': (', '.join(quality_samples_map), 'not has(' + map_template + ', {field})')} + {'sampleId': (quality_samples_map, 'not has(' + map_template + ', {field})')} ]} quality_q = Q(calls__array_all=quality_filter_conditions) if clinvar_override_q: @@ -915,18 +870,30 @@ def _get_family_sample_types(sample_data): multi_sample_type_families[s['family_guid']] = [] return sample_type_families, multi_sample_type_families - def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected): - sample_filters = [] - any_affected_samples = [] + def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, is_multi_sample_family): + sample_filters = defaultdict(list) + any_affected_samples = defaultdict(list) + nullable_gt_samples = defaultdict(list) + quality_samples = defaultdict(list) + missing_type_samples = defaultdict(list) for sample in family_sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] - sample_inheritance_filter = self._sample_genotype_filter(sample, affected, inheritance_mode, individual_genotype_filter) - sample_quality_filter = self._sample_quality_filter(affected, quality_filter) - if sample_inheritance_filter or sample_quality_filter: - sample_filters.append((sample['sample_ids_by_type'], sample_inheritance_filter, sample_quality_filter)) - if inheritance_mode == ANY_AFFECTED and affected == AFFECTED: - any_affected_samples.append(sample['sample_ids_by_type']) - return sample_filters, any_affected_samples + genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) + for sample_type, sample_id in sample['sample_ids_by_type'].items(): + if genotype is not None: + sample_filters[sample_type].append( + f"'{sample_id}', {self.genotype_lookup[genotype]}") + if genotype in self.nullable_genotypes: + nullable_gt_samples[sample_type].append(sample_id) + elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: + any_affected_samples[sample_type].append(sample_id) + if (not quality_filter.get('affected_only')) or affected == AFFECTED: + quality_samples[sample_type].append(sample_id) + if is_multi_sample_family and len(sample['sample_ids_by_type']) == 1: + missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + missing_type_samples[missing_type].append(sample_id) + + return sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples def _family_calls_q(self, call_q, family_sample_data, sample_filters, sample_type, clinvar_override_q, any_affected_samples=None, filter_sample_type=True): family_sample_q = None @@ -968,13 +935,6 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _sample_genotype_filter(self, sample, affected, inheritance_mode, individual_genotype_filter): - sample_filter = {} - genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) - if genotype: - sample_filter['gt'] = self.genotype_lookup[genotype] - return sample_filter - def _sample_quality_filter(self, affected, quality_filter): sample_filter = {} if quality_filter.get('affected_only') and affected != AFFECTED: @@ -991,28 +951,22 @@ def _sample_quality_filter(self, affected, quality_filter): return sample_filter - @classmethod - def _annotate_failed_family_samples(cls, entries, gt_map, family_missing_type_samples): - # gt_map = [] - # missing_sample_map = [] - # for family_guid, sample_filters in family_sample_gt_filters.items(): - # sample_type_filters = defaultdict(list) - # missing_type_samples = defaultdict(list) - # for sample_ids_by_type, gt_filter in sample_filters: - # for sample_type, sample_id in sample_ids_by_type.items(): - # sample_type_filters[sample_type].append(f"'{sample_id}', {cls.INVALID_NUM_ALT_LOOKUP[gt_filter]}") - # if len(sample_ids_by_type) == 1: - # missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS - # missing_type_samples[missing_type].append(sample_id) - # sample_type_map = [ - # f"'{sample_type}', map({', '.join(type_filters)})" - # for sample_type, type_filters in sample_type_filters.items() - # ] - # gt_map.append(f"'{family_guid}', map({', '.join(sample_type_map)})") - # if missing_type_samples: - # missing_type_map = [f"'{sample_type}', {samples}" for sample_type, samples in missing_type_samples.items()] - # missing_sample_map.append(f"'{family_guid}', map({', '.join(missing_type_map)})") + def _get_family_sample_map(self, family_samples, is_nested_map=False): + family_map = [] + for family_guid, sample_types in family_samples.items(): + if self.single_sample_type: + samples = list(next(iter(sample_types.values()))) + else: + if is_nested_map: + sample_types = {sample_type: f"map({', '.join(samples)})" for sample_type, samples in sample_types.items()} + samples = [f"'{sample_type}', {samples}" for sample_type, samples in sample_types.items()] + if is_nested_map or not self.single_sample_type: + samples = f"map({', '.join(samples)})" + family_map.append(f"'{family_guid}', {samples}") + return ', '.join(family_map) + + def _annotate_failed_family_samples(self, entries, gt_map, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ 'gt': (gt_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), @@ -1021,14 +975,10 @@ def _annotate_failed_family_samples(cls, entries, gt_map, family_missing_type_sa mapped_expression='tuple(family_guid, x.sampleId)', )) if family_missing_type_samples: - missing_sample_map = [] - for family_guid, sample_type_map in family_missing_type_samples.items(): - sample_map = [f"'{sample_type}', {samples}" for sample_type, samples in sample_type_map.items()] - missing_sample_map.append(f"'{family_guid}', map({', '.join(sample_map)})") - + missing_sample_map = self._get_family_sample_map(family_missing_type_samples) entries = entries.annotate( missing_family_samples=ArrayMap( - MapLookup('family_guid', Cast('sample_type', models.StringField()), map_values=', '.join(missing_sample_map)), + MapLookup('family_guid', Cast('sample_type', models.StringField()), map_values=missing_sample_map), mapped_expression='tuple(family_guid, x)', ) ) From 31219277d8c969d64456a4a387702e2e8ef47a9c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 13:12:14 -0400 Subject: [PATCH 019/109] clean up --- clickhouse_search/managers.py | 46 +++++++++++++++-------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 442d22f815..31384b3828 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1,5 +1,3 @@ -from email.policy import default - from clickhouse_backend import models from collections import OrderedDict, defaultdict @@ -7,7 +5,6 @@ from django.db.models.expressions import Col from django.db.models.functions import Cast from django.db.models.sql.constants import INNER -from fastparquet.converted_types import nullable from clickhouse_search.backend.fields import NestedField, NamedTupleField from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ @@ -765,29 +762,27 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None - family_sample_gts = defaultdict(lambda: defaultdict(list)) - family_sample_null_gts = defaultdict(lambda: defaultdict(list)) - family_affected_samples = defaultdict(lambda: defaultdict(list)) - family_quality_samples = defaultdict(lambda: defaultdict(list)) - family_missing_type_samples = defaultdict(lambda: defaultdict(list)) + family_sample_gts = {} + family_sample_null_gts = {} + family_affected_samples = {} + family_quality_samples = {} + family_missing_type_samples = {} quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: - for sample in s['samples']: - affected = custom_affected.get(sample['individual_guid']) or sample['affected'] - genotype = self._sample_genotype(sample, affected, inheritance_mode,individual_genotype_filter) - for sample_type, sample_id in sample['sample_ids_by_type'].items(): - if genotype is not None: - family_sample_gts[s['family_guid']][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") - if genotype in self.nullable_genotypes: - family_sample_null_gts[s['family_guid']][sample_type].append(sample_id) - elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: - family_affected_samples[s['family_guid']][sample_type].append(sample_id) - if (not quality_filter.get('affected_only')) or affected == AFFECTED: - family_quality_samples[s['family_guid']][sample_type].append(sample_id) - if s['family_guid'] in multi_sample_type_families and len(sample['sample_ids_by_type']) == 1: - sample_type, sample_id = next(iter(sample['sample_ids_by_type'].items())) - missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS - family_missing_type_samples[s['family_guid']][missing_type].append(sample_id) + sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( + s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, + is_multi_sample_family=s['family_guid'] in multi_sample_type_families, + ) + if sample_filters: + family_sample_gts[s['family_guid']] = sample_filters + if any_affected_samples: + family_affected_samples[s['family_guid']] = any_affected_samples + if nullable_gt_samples: + family_sample_null_gts[s['family_guid']] = nullable_gt_samples + if quality_samples: + family_quality_samples[s['family_guid']] = quality_samples + if missing_type_samples: + family_missing_type_samples[s['family_guid']] = missing_type_samples map_template = 'map({value})[family_guid]' if not self.single_sample_type: @@ -833,9 +828,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_q = Q(passes_quality=True) | multi_sample_type_family_q # for s in sample_data: - # sample_filters, any_affected_samples = self._family_sample_filters(s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected) # else: - # sample_type = s['sample_types'][0] + # TODO # call_q = self._family_calls_q( # call_q, s, sample_filters, sample_type, clinvar_override_q, # any_affected_samples=any_affected_samples, filter_sample_type=not self.single_sample_type From 043aca704b9e71b963b5652797266a1cc949c0ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 13:41:23 -0400 Subject: [PATCH 020/109] clean up --- clickhouse_search/managers.py | 114 +++++++++++++++++----------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 31384b3828..fc39c08b43 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -754,6 +754,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q = None quality_q = None + gt_filter_map = None quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') custom_affected = (inheritance_filter or {}).get('affected') or {} @@ -767,7 +768,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit family_affected_samples = {} family_quality_samples = {} family_missing_type_samples = {} - quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper for s in sample_data: sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, @@ -784,35 +784,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples + inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples) + map_template = 'map({value})[family_guid]' if not self.single_sample_type: map_template += '[sample_type::String]' - if family_sample_gts: - gt_map = self._get_family_sample_map(family_sample_gts, is_nested_map=True) - gt_conditions = [ - {'gt': (gt_map, 'has(' + map_template + '[x.sampleId], {field})')}, - {'sampleId': (gt_map, 'not mapContains(' + map_template + ', {field})')}, - ] - if family_sample_null_gts: - null_gt_map = self._get_family_sample_map(family_sample_null_gts) - gt_conditions.append({ - 'gt': (None, 'isNull({field})'), - 'sampleId': (null_gt_map, 'has(' + map_template + ', {field})'), - }) - inheritance_q = Q(calls__array_all={'OR': gt_conditions}) - if multi_sample_type_families: - inheritance_q |= multi_sample_type_family_q - entries = self._annotate_failed_family_samples(entries, gt_map, family_missing_type_samples) - elif family_affected_samples: - affected_sample_map = self._get_family_sample_map(family_affected_samples) - inheritance_q = Q(calls__array_exists={ - 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), - }) - if multi_sample_type_families: - entries = entries.annotate(passes_inheritance=inheritance_q) - inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q - + quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper if quality_filter_conditions: quality_samples_map = self._get_family_sample_map(family_quality_samples) if quality_samples_map: @@ -823,17 +800,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_q = Q(calls__array_all=quality_filter_conditions) if clinvar_override_q: quality_q |= clinvar_override_q - if multi_sample_type_families: - entries = entries.annotate(passes_quality=quality_q) - quality_q = Q(passes_quality=True) | multi_sample_type_family_q - - # for s in sample_data: - # else: - # TODO - # call_q = self._family_calls_q( - # call_q, s, sample_filters, sample_type, clinvar_override_q, - # any_affected_samples=any_affected_samples, filter_sample_type=not self.single_sample_type - # ) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) @@ -841,15 +807,26 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit q |= clinvar_override_q entries = entries.filter(q) + if multi_sample_type_families: + if gt_filter_map: + inheritance_q |= multi_sample_type_family_q + entries = self._annotate_failed_family_samples(entries, gt_filter_map, family_missing_type_samples) + elif inheritance_q is not None: + entries = entries.annotate(passes_inheritance=inheritance_q) + inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q + else: + entries = entries.annotate(passes_inheritance=Value(True)) + + if quality_q is None: + entries = entries.annotate(passes_quality=Value(True)) + else: + entries = entries.annotate(passes_quality=quality_q) + quality_q = Q(passes_quality=True) | multi_sample_type_family_q + if inheritance_q is not None: entries = entries.filter(inheritance_q) - elif multi_sample_type_families: - entries = entries.annotate(passes_inheritance=Value(True)) - if quality_q is not None: entries = entries.filter(quality_q) - elif multi_sample_type_families: - entries = entries.annotate(passes_quality=Value(True)) return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) @@ -889,16 +866,39 @@ def _family_sample_filters(self, family_sample_data, inheritance_mode, individua return sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples - def _family_calls_q(self, call_q, family_sample_data, sample_filters, sample_type, clinvar_override_q, any_affected_samples=None, filter_sample_type=True): - family_sample_q = None - if any_affected_samples: - affected_sample_ids = [f"'{sample_ids_by_type[sample_type]}'" for sample_ids_by_type in any_affected_samples] - family_sample_q = Q(calls__array_exists={ - 'gt': self.genotype_lookup[HAS_ALT], - 'sampleId': (', '.join(affected_sample_ids), 'has([{value}], {field})'), - }) + def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples): + map_template = 'map({value})[family_guid]' + if not self.single_sample_type: + map_template += '[sample_type::String]' + + gt_filter_map = None + inheritance_q = None + if any_affected_samples: + affected_sample_map = self._get_family_sample_map(any_affected_samples) + inheritance_q = Q(calls__array_exists={ + 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), + 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), + }) + + elif sample_filters: + gt_filter_map = self._get_family_sample_map(sample_filters, is_nested_map=True) + gt_conditions = [ + {'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})')}, + {'sampleId': (gt_filter_map, 'not mapContains(' + map_template + ', {field})')}, + ] + if sample_null_gts: + null_gt_map = self._get_family_sample_map(sample_null_gts) + gt_conditions.append({ + 'gt': (None, 'isNull({field})'), + 'sampleId': (null_gt_map, 'has(' + map_template + ', {field})'), + }) + inheritance_q = Q(calls__array_all={'OR': gt_conditions}) + + return inheritance_q, gt_filter_map - for sample_ids_by_type, sample_inheritance_filter, sample_quality_filter in sample_filters: + def _quality_q(self): + # TODO + for sample_ids_by_type, sample_inheritance_filter, sample_quality_filter in sample_filters: sample_inheritance_filter['sampleId'] = (f"'{sample_ids_by_type[sample_type]}'",) sample_q = Q( calls__array_exists={**sample_inheritance_filter, **sample_quality_filter}, @@ -915,9 +915,9 @@ def _family_calls_q(self, call_q, family_sample_data, sample_filters, sample_typ else: family_sample_q &= sample_q - if family_sample_q and call_q: + if family_sample_q and call_q: call_q |= family_sample_q - return call_q or family_sample_q + return call_q or family_sample_q def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None @@ -960,11 +960,11 @@ def _get_family_sample_map(self, family_samples, is_nested_map=False): return ', '.join(family_map) - def _annotate_failed_family_samples(self, entries, gt_map, family_missing_type_samples): + def _annotate_failed_family_samples(self, entries, gt_filter_map, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ - 'gt': (gt_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), - 'sampleId': (gt_map, 'mapContains(map({value})[family_guid][sample_type::String], {field})'), + 'gt': (gt_filter_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), + 'sampleId': (gt_filter_map, 'mapContains(map({value})[family_guid][sample_type::String], {field})'), }]), mapped_expression='tuple(family_guid, x.sampleId)', )) From 6c973ac9769459761ba27139ffb12f092a24cda6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 13:50:30 -0400 Subject: [PATCH 021/109] clean up --- clickhouse_search/managers.py | 72 ++++++++++++----------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index fc39c08b43..832eb24e4f 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -735,11 +735,11 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) - sample_type_families, multi_sample_type_families = self._get_family_sample_types(sample_data) # TODO multi_sample_type_families can be a set + sample_type_families, multi_sample_type_families = self._get_family_sample_types(sample_data) family_q = None multi_sample_type_family_q = None if multi_sample_type_families: - family_q = Q(family_guid__in=multi_sample_type_families.keys()) + family_q = Q(family_guid__in=multi_sample_type_families) multi_sample_type_family_q = family_q for sample_type, families in sample_type_families.items(): sample_family_q = Q(family_guid__in=families) @@ -786,20 +786,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples) - map_template = 'map({value})[family_guid]' - if not self.single_sample_type: - map_template += '[sample_type::String]' - quality_filter_conditions = self._sample_quality_filter(AFFECTED, quality_filter) # TODO clean up helper - if quality_filter_conditions: - quality_samples_map = self._get_family_sample_map(family_quality_samples) - if quality_samples_map: - quality_filter_conditions = {'OR': [ - quality_filter_conditions, - {'sampleId': (quality_samples_map, 'not has(' + map_template + ', {field})')} - ]} - quality_q = Q(calls__array_all=quality_filter_conditions) - if clinvar_override_q: - quality_q |= clinvar_override_q + quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) @@ -833,12 +820,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit @staticmethod def _get_family_sample_types(sample_data): sample_type_families = defaultdict(list) - multi_sample_type_families = {} + multi_sample_type_families = set() for s in sample_data: if len(s['sample_types']) == 1: sample_type_families[s['sample_types'][0]].append(s['family_guid']) else: - multi_sample_type_families[s['family_guid']] = [] + multi_sample_type_families.add(s['family_guid']) return sample_type_families, multi_sample_type_families def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, is_multi_sample_family): @@ -896,29 +883,6 @@ def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples): return inheritance_q, gt_filter_map - def _quality_q(self): - # TODO - for sample_ids_by_type, sample_inheritance_filter, sample_quality_filter in sample_filters: - sample_inheritance_filter['sampleId'] = (f"'{sample_ids_by_type[sample_type]}'",) - sample_q = Q( - calls__array_exists={**sample_inheritance_filter, **sample_quality_filter}, - family_guid=family_sample_data['family_guid'], - project_guid= family_sample_data['project_guid'], - ) - if filter_sample_type: - sample_q &= Q(sample_type=sample_type) - if clinvar_override_q and sample_quality_filter: - sample_q |= clinvar_override_q & Q(calls__array_exists=sample_inheritance_filter) - - if family_sample_q is None: - family_sample_q = sample_q - else: - family_sample_q &= sample_q - - if family_sample_q and call_q: - call_q |= family_sample_q - return call_q or family_sample_q - def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None if individual_genotype_filter: @@ -929,10 +893,8 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _sample_quality_filter(self, affected, quality_filter): - sample_filter = {} - if quality_filter.get('affected_only') and affected != AFFECTED: - return sample_filter + def _quality_q(self, quality_filter, quality_samples, clinvar_override_q): + quality_filter_conditions = {} for field, scale, *filters in self.quality_filters: filter_key = f'min_{field}' @@ -941,9 +903,25 @@ def _sample_quality_filter(self, affected, quality_filter): value = quality_filter.get(filter_key) if value: or_filters = ['isNull({field})', '{field} >= {value}'] + filters - sample_filter[field] = (value / scale, f'or({", ".join(or_filters)})') + quality_filter_conditions[field] = (value / scale, f'or({", ".join(or_filters)})') + + if not quality_filter_conditions: + return None - return sample_filter + template = 'not has(map({value})[family_guid]' + if not self.single_sample_type: + template += '[sample_type::String]' + quality_samples_map = self._get_family_sample_map(quality_samples) + quality_filter_conditions = {'OR': [ + quality_filter_conditions, + {'sampleId': (quality_samples_map, template + ', {field})')} + ]} + + quality_q = Q(calls__array_all=quality_filter_conditions) + if clinvar_override_q: + quality_q |= clinvar_override_q + + return quality_q def _get_family_sample_map(self, family_samples, is_nested_map=False): family_map = [] From 6495031e2aa234f62ead6f239b9f5c9048a05c36 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 13 Aug 2025 16:45:06 -0400 Subject: [PATCH 022/109] gnomad genomes dict (#4931) * gnomad genomes dict * fix comma --- ...mesgrch37snvindel_gnomadgenomessnvindel.py | 85 +++++++++++++++++++ clickhouse_search/models.py | 22 +++++ 2 files changed, 107 insertions(+) create mode 100644 clickhouse_search/migrations/0014_gnomadgenomesgrch37snvindel_gnomadgenomessnvindel.py diff --git a/clickhouse_search/migrations/0014_gnomadgenomesgrch37snvindel_gnomadgenomessnvindel.py b/clickhouse_search/migrations/0014_gnomadgenomesgrch37snvindel_gnomadgenomessnvindel.py new file mode 100644 index 0000000000..5b621c8477 --- /dev/null +++ b/clickhouse_search/migrations/0014_gnomadgenomesgrch37snvindel_gnomadgenomessnvindel.py @@ -0,0 +1,85 @@ +# Generated by Django 4.2.21 on 2025-08-13 19:35 and editted manually by the seqr team + +import os +from string import Template + +import clickhouse_backend.models +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.manager + +CLICKHOUSE_WRITER_PASSWORD = os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test') +CLICKHOUSE_WRITER_USER = os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse') + +GNOMAD_GENOMES_DICT = Template(Template(""" +CREATE DICTIONARY `$reference_genome/$dataset_type/gnomad_genomes_dict` +( + key UInt32, + filter_af Decimal(9, 5) +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(USER $clickhouse_writer_user PASSWORD $clickhouse_writer_password TABLE `$reference_genome/$dataset_type/gnomad_genomes`)) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT(MAX_ARRAY_SIZE $size)) +""").safe_substitute( + # Note the nested Template-ing that allows + # double substitution these shared values + clickhouse_writer_user=CLICKHOUSE_WRITER_USER, + clickhouse_writer_password=CLICKHOUSE_WRITER_PASSWORD, +)) + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0013_annotationsdiskgcnv_annotationsdiskmito_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='GnomadGenomesGRCh37SnvIndel', + fields=[ + ('key', models.OneToOneField(db_column='key', on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='clickhouse_search.annotationsgrch37snvindel')), + ('filter_af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), + ], + options={ + 'db_table': 'GRCh37/SNV_INDEL/gnomad_genomes', + 'engine': clickhouse_backend.models.ReplacingMergeTree(order_by='key', primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='GnomadGenomesSnvIndel', + fields=[ + ('key', models.OneToOneField(db_column='key', on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='clickhouse_search.annotationssnvindel')), + ('filter_af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), + ], + options={ + 'db_table': 'GRCh38/SNV_INDEL/gnomad_genomes', + 'engine': clickhouse_backend.models.ReplacingMergeTree(order_by='key', primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.RunSQL( + GNOMAD_GENOMES_DICT.substitute( + reference_genome='GRCh37', + dataset_type='SNV_INDEL', + size=int(1e8), + ), + hints={'clickhouse': True}, + ), + migrations.RunSQL( + GNOMAD_GENOMES_DICT.substitute( + reference_genome='GRCh38', + dataset_type='SNV_INDEL', + size=int(5e8), + ), + hints={'clickhouse': True}, + ), + ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index c23171abcf..d742852f73 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -883,6 +883,28 @@ class GtStatsSv(models.ClickhouseModel): class Meta(BaseGtStats.Meta): db_table = 'GRCh38/SV/gt_stats' +class GnomadGenomesGRCh37SnvIndel(models.ClickhouseModel): + key = OneToOneField('AnnotationsGRCh37SnvIndel', db_column='key', primary_key=True, on_delete=CASCADE) + filter_af = models.DecimalField(max_digits=9, decimal_places=5) + + class Meta: + db_table = 'GRCh37/SNV_INDEL/gnomad_genomes' + engine = models.ReplacingMergeTree( + primary_key='key', + order_by='key' + ) + +class GnomadGenomesSnvIndel(models.ClickhouseModel): + key = OneToOneField('AnnotationsSnvIndel', db_column='key', primary_key=True, on_delete=CASCADE) + filter_af = models.DecimalField(max_digits=9, decimal_places=5) + + class Meta: + db_table = 'GRCh38/SNV_INDEL/gnomad_genomes' + engine = models.ReplacingMergeTree( + primary_key='key', + order_by='key' + ) + ENTRY_CLASS_MAP = { GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, From 623ca4dd4c5ddc95b17144591cfc3f21ffdde690 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 13 Aug 2025 16:58:45 -0400 Subject: [PATCH 023/109] minimize maps for single family search --- clickhouse_search/managers.py | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 832eb24e4f..0c0a8379a4 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -784,9 +784,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples - inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples) - - quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q) + is_single_family = len(sample_data) == 1 + inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples, is_single_family) + quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) @@ -853,7 +853,7 @@ def _family_sample_filters(self, family_sample_data, inheritance_mode, individua return sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples - def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples): + def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples, is_single_family): map_template = 'map({value})[family_guid]' if not self.single_sample_type: map_template += '[sample_type::String]' @@ -861,20 +861,32 @@ def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples): gt_filter_map = None inheritance_q = None if any_affected_samples: - affected_sample_map = self._get_family_sample_map(any_affected_samples) + if is_single_family: + affected_sample_map = next(iter(next(iter(any_affected_samples.values())).values())) + map_template = '{value}' + else: + affected_sample_map = self._get_family_sample_map(any_affected_samples) inheritance_q = Q(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), }) elif sample_filters: - gt_filter_map = self._get_family_sample_map(sample_filters, is_nested_map=True) + if is_single_family: + gt_filter_map = ', '.join(next(iter(next(iter(sample_filters.values())).values()))) + map_template = 'map({value})' + else: + gt_filter_map = self._get_family_sample_map(sample_filters, is_nested_map=True) gt_conditions = [ {'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})')}, {'sampleId': (gt_filter_map, 'not mapContains(' + map_template + ', {field})')}, ] if sample_null_gts: - null_gt_map = self._get_family_sample_map(sample_null_gts) + if is_single_family: + null_gt_map = next(iter(next(iter(sample_null_gts.values())).values())) + map_template = '{value}' + else: + null_gt_map = self._get_family_sample_map(sample_null_gts) gt_conditions.append({ 'gt': (None, 'isNull({field})'), 'sampleId': (null_gt_map, 'has(' + map_template + ', {field})'), @@ -893,7 +905,7 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _quality_q(self, quality_filter, quality_samples, clinvar_override_q): + def _quality_q(self, quality_filter, quality_samples, clinvar_override_q, is_single_family): quality_filter_conditions = {} for field, scale, *filters in self.quality_filters: @@ -908,10 +920,15 @@ def _quality_q(self, quality_filter, quality_samples, clinvar_override_q): if not quality_filter_conditions: return None - template = 'not has(map({value})[family_guid]' - if not self.single_sample_type: - template += '[sample_type::String]' - quality_samples_map = self._get_family_sample_map(quality_samples) + template = 'not has(' + if is_single_family: + quality_samples_map = next(iter(next(iter(quality_samples.values())).values())) + template += '{value}' + else: + quality_samples_map = self._get_family_sample_map(quality_samples) + template += 'map({value})[family_guid]' + if not self.single_sample_type: + template += '[sample_type::String]' quality_filter_conditions = {'OR': [ quality_filter_conditions, {'sampleId': (quality_samples_map, template + ', {field})')} From 0babf4f476e9b36a788366a477adf5b5cf7b55b7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 11:01:41 -0400 Subject: [PATCH 024/109] fix multi sample type --- clickhouse_search/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 0c0a8379a4..3fc527ff4b 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -784,7 +784,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples - is_single_family = len(sample_data) == 1 + is_single_family = len(sample_data) == 1 and not multi_sample_type_families inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples, is_single_family) quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) From fb11f923b04662081f134f2234ca4007e6237e98 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 11:53:38 -0400 Subject: [PATCH 025/109] use dict to prefilter gnomad --- clickhouse_search/backend/functions.py | 5 +++++ clickhouse_search/managers.py | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 1a3300a220..6cdf3552c2 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -125,6 +125,11 @@ class DictGet(Func): template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' +class DictGetOrDefault(DictGet): + function = 'dictGetOrDefault' + template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' + + class If(Func): function = 'if' template = '%(function)s(%(condition)s%(expressions)s)' diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 832eb24e4f..60390cab53 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -9,7 +9,7 @@ from clickhouse_search.backend.fields import NestedField, NamedTupleField from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ - GroupArrayIntersect, DictGet, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat + GroupArrayIntersect, DictGet, DictGetOrDefault, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat from seqr.models import Sample from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ @@ -683,7 +683,7 @@ def genotype_fields(self): def clinvar_model(self): return self.model.clinvar_join.rel.related_model - def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): + def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, pathogenicity=None, **kwargs): entries = self.filter_intervals(**(parsed_locus or {})) entries = self._join_annotations(entries) @@ -693,14 +693,26 @@ def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, * if (freqs or {}).get(callset_filter_field) and self.annotations_model.SEQR_POPULATIONS: entries = self._filter_seqr_frequency(entries, **freqs[callset_filter_field]) + clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( + pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), + ) if self._has_clinvar() else None gnomad_filter = (freqs or {}).get('gnomad_genomes') or {} - if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= 0.05 or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): + if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= PATH_FREQ_OVERRIDE_CUTOFF or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): entries = entries.filter(is_gnomad_gt_5_percent=False) + if (gnomad_filter.get('af') or 1) < PATH_FREQ_OVERRIDE_CUTOFF: + entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( + 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", + output_field=models.DecimalField(), + )) + af_q = Q(gnomad_genomes_af__lte=gnomad_filter['af']) + if clinvar_override_q is not None: + af_q |= clinvar_override_q + entries = entries.filter(af_q) if (annotations or {}).get(NEW_SV_FIELD) and 'newCall' in self.call_fields: entries = entries.filter(calls__array_exists={'newCall': (None, '{field}')}) - return self._search_call_data(entries, sample_data, **kwargs) + return self._search_call_data(entries, sample_data, clinvar_override_q=clinvar_override_q, **kwargs) def _join_annotations(self, entries): if self._has_clinvar(): @@ -730,7 +742,7 @@ def result_values(self, sample_data): def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') - def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, clinvar_override_q=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): project_guids = {s['project_guid'] for s in sample_data} project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) @@ -759,10 +771,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit individual_genotype_filter = (inheritance_filter or {}).get('genotype') custom_affected = (inheritance_filter or {}).get('affected') or {} if inheritance_mode or individual_genotype_filter or quality_filter: - clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( - pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), - ) if self._has_clinvar() else None - family_sample_gts = {} family_sample_null_gts = {} family_affected_samples = {} From 2322f2e2cb73ccee247e220646ea75c0fa85f1cb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 12:27:59 -0400 Subject: [PATCH 026/109] test fixtures --- .../fixtures/clickhouse_search.json | 49 +++++++++++++++++++ clickhouse_search/search_tests.py | 2 + 2 files changed, 51 insertions(+) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index c2f05942bc..ff019d9ee5 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -264,6 +264,48 @@ ], "pathogenicity": "Conflicting_classifications_of_pathogenicity" } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 1, + "fields": { + "key": 1, + "filter_af": 0.04028 + } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 3, + "fields": { + "key": 3, + "filter_af": 0.47978 + } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 4, + "fields": { + "key": 4, + "filter_af": 0.0015 + } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 5, + "fields": { + "key": 5, + "filter_af": 0.00233 + } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 6, + "fields": { + "key": 6, + "filter_af": 0.00011 + } +}, { + "model": "clickhouse_search.gnomadgenomessnvindel", + "pk": 7, + "fields": { + "key": 7, + "filter_af": 0.0002 + } }, { "model": "clickhouse_search.entriessnvindel", "pk": 1, @@ -613,6 +655,13 @@ "key": 21, "variant_id": "1-10439-AC-A" } +}, { + "model": "clickhouse_search.gnomadgenomesgrch37snvindel", + "pk": 11, + "fields": { + "key": 11, + "filter_af": 0.82844 + } }, { "model": "clickhouse_search.annotationsmito", "pk": 6, diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index d3c466f762..aca721fd73 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -39,6 +39,8 @@ def setUpTestData(cls): cursor.execute(f'SYSTEM REFRESH VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') cursor.execute(f'SYSTEM WAIT VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') + if table_base.endswith('SNV_INDEL'): + cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gnomad_genomes_dict"') Project.objects.update(genome_version='38') def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', results_model=None, **search_kwargs): From cad2e8441fe8114f78cd48c654995c8307389ea7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 12:34:22 -0400 Subject: [PATCH 027/109] do not use gnomad af entries column --- clickhouse_search/managers.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 60390cab53..38f2c694ed 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -697,17 +697,18 @@ def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, p pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None gnomad_filter = (freqs or {}).get('gnomad_genomes') or {} - if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= PATH_FREQ_OVERRIDE_CUTOFF or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): - entries = entries.filter(is_gnomad_gt_5_percent=False) - if (gnomad_filter.get('af') or 1) < PATH_FREQ_OVERRIDE_CUTOFF: - entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( - 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", - output_field=models.DecimalField(), - )) - af_q = Q(gnomad_genomes_af__lte=gnomad_filter['af']) - if clinvar_override_q is not None: - af_q |= clinvar_override_q - entries = entries.filter(af_q) + af_cutoff = gnomad_filter.get('af') + if af_cutoff is None and any(gnomad_filter.get(field) is not None for field in ['ac', 'hh']): + af_cutoff = PATH_FREQ_OVERRIDE_CUTOFF + if af_cutoff is not None and any(obj.name.startswith('gnomadgenomes') for obj in self.annotations_model._meta.related_objects): + entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( + 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", + output_field=models.DecimalField(), + )) + af_q = Q(gnomad_genomes_af__lte=af_cutoff) + if clinvar_override_q is not None: + af_q |= (clinvar_override_q & Q(gnomad_genomes_af__lte=PATH_FREQ_OVERRIDE_CUTOFF)) + entries = entries.filter(af_q) if (annotations or {}).get(NEW_SV_FIELD) and 'newCall' in self.call_fields: entries = entries.filter(calls__array_exists={'newCall': (None, '{field}')}) From 850dec609d86da4fed710d54901676eb01fbb5f9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 12:37:16 -0400 Subject: [PATCH 028/109] clean up --- clickhouse_search/backend/functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 6cdf3552c2..6dc98c2286 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -127,7 +127,6 @@ class DictGet(Func): class DictGetOrDefault(DictGet): function = 'dictGetOrDefault' - template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' class If(Func): From 9f3c9a01d1c03999cf32bdca444d1ae6616e2793 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 12:53:14 -0400 Subject: [PATCH 029/109] codacy cleanup --- clickhouse_search/managers.py | 61 ++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 5ba9aefdc7..ceba844491 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -768,35 +768,14 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q = None quality_q = None gt_filter_map = None + family_missing_type_samples = None quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') - custom_affected = (inheritance_filter or {}).get('affected') or {} if inheritance_mode or individual_genotype_filter or quality_filter: - family_sample_gts = {} - family_sample_null_gts = {} - family_affected_samples = {} - family_quality_samples = {} - family_missing_type_samples = {} - for s in sample_data: - sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( - s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, - is_multi_sample_family=s['family_guid'] in multi_sample_type_families, - ) - if sample_filters: - family_sample_gts[s['family_guid']] = sample_filters - if any_affected_samples: - family_affected_samples[s['family_guid']] = any_affected_samples - if nullable_gt_samples: - family_sample_null_gts[s['family_guid']] = nullable_gt_samples - if quality_samples: - family_quality_samples[s['family_guid']] = quality_samples - if missing_type_samples: - family_missing_type_samples[s['family_guid']] = missing_type_samples - - is_single_family = len(sample_data) == 1 and not multi_sample_type_families - inheritance_q, gt_filter_map = self._inheritance_q(family_sample_gts, family_sample_null_gts, family_affected_samples, is_single_family) - quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) - + inheritance_q, quality_q, gt_filter_map, family_missing_type_samples = self._get_inheritance_quality_qs( + sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, + custom_affected=(inheritance_filter or {}).get('affected') or {}, + ) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) if clinvar_override_q: @@ -826,6 +805,36 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) + def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): + family_sample_gts = {} + family_sample_null_gts = {} + family_affected_samples = {} + family_quality_samples = {} + family_missing_type_samples = {} + for s in sample_data: + sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( + s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, + is_multi_sample_family=s['family_guid'] in multi_sample_type_families, + ) + if sample_filters: + family_sample_gts[s['family_guid']] = sample_filters + if any_affected_samples: + family_affected_samples[s['family_guid']] = any_affected_samples + if nullable_gt_samples: + family_sample_null_gts[s['family_guid']] = nullable_gt_samples + if quality_samples: + family_quality_samples[s['family_guid']] = quality_samples + if missing_type_samples: + family_missing_type_samples[s['family_guid']] = missing_type_samples + + is_single_family = len(sample_data) == 1 and not multi_sample_type_families + inheritance_q, gt_filter_map = self._inheritance_q( + family_sample_gts, family_sample_null_gts, family_affected_samples, is_single_family, + ) + quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) + + return inheritance_q, quality_q, gt_filter_map, family_missing_type_samples + @staticmethod def _get_family_sample_types(sample_data): sample_type_families = defaultdict(list) From 6ab511a7fdfd4f4f6b65dec302c90c9c453174b3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 16:10:39 -0400 Subject: [PATCH 030/109] imporve any affected performance --- clickhouse_search/managers.py | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index ceba844491..aed761cea7 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -808,18 +808,16 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): family_sample_gts = {} family_sample_null_gts = {} - family_affected_samples = {} + family_affected_samples = defaultdict(lambda: defaultdict(list)) family_quality_samples = {} family_missing_type_samples = {} for s in sample_data: - sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( - s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, + sample_filters, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( + s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_affected_samples, s['family_guid'], is_multi_sample_family=s['family_guid'] in multi_sample_type_families, ) if sample_filters: family_sample_gts[s['family_guid']] = sample_filters - if any_affected_samples: - family_affected_samples[s['family_guid']] = any_affected_samples if nullable_gt_samples: family_sample_null_gts[s['family_guid']] = nullable_gt_samples if quality_samples: @@ -846,9 +844,8 @@ def _get_family_sample_types(sample_data): multi_sample_type_families.add(s['family_guid']) return sample_type_families, multi_sample_type_families - def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, is_multi_sample_family): + def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_affected_samples, family_guid, is_multi_sample_family): sample_filters = defaultdict(list) - any_affected_samples = defaultdict(list) nullable_gt_samples = defaultdict(list) quality_samples = defaultdict(list) missing_type_samples = defaultdict(list) @@ -861,15 +858,15 @@ def _family_sample_filters(self, family_sample_data, inheritance_mode, individua f"'{sample_id}', {self.genotype_lookup[genotype]}") if genotype in self.nullable_genotypes: nullable_gt_samples[sample_type].append(sample_id) - elif inheritance_mode == ANY_AFFECTED and affected == AFFECTED: - any_affected_samples[sample_type].append(sample_id) + elif inheritance_mode == ANY_AFFECTED: + family_affected_samples[sample_id][affected].append((family_guid, sample_type)) if (not quality_filter.get('affected_only')) or affected == AFFECTED: quality_samples[sample_type].append(sample_id) if is_multi_sample_family and len(sample['sample_ids_by_type']) == 1: missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS missing_type_samples[missing_type].append(sample_id) - return sample_filters, any_affected_samples, nullable_gt_samples, quality_samples, missing_type_samples + return sample_filters, nullable_gt_samples, quality_samples, missing_type_samples def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples, is_single_family): map_template = 'map({value})[family_guid]' @@ -879,15 +876,28 @@ def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples, gt_filter_map = None inheritance_q = None if any_affected_samples: - if is_single_family: - affected_sample_map = next(iter(next(iter(any_affected_samples.values())).values())) - map_template = '{value}' - else: - affected_sample_map = self._get_family_sample_map(any_affected_samples) + mismatched_affected_samples = defaultdict(lambda: defaultdict(list)) + affected_samples = [] + for sample_id, affected_families in any_affected_samples.items(): + if AFFECTED not in affected_families: + continue + if len(affected_families) > 1: + for family_guid, sample_type in affected_families[AFFECTED]: + mismatched_affected_samples[family_guid][sample_type].append(sample_id) + else: + affected_samples.append(sample_id) + inheritance_q = Q(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), + 'sampleId': (affected_samples, 'has({value}, {field})'), }) + if mismatched_affected_samples: + affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) + inheritance_q |= Q(calls__array_exists={ + 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), + 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), + 'family': (list(mismatched_affected_samples.keys()), 'has({value}, family_guid)'), + }) elif sample_filters: if is_single_family: From b7548626ee1733a34c3ee3cb1a6cdf5d3ae84bb8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 14 Aug 2025 17:11:33 -0400 Subject: [PATCH 031/109] map gentoypes to allowed sample ids --- clickhouse_search/managers.py | 88 ++++++++++++++++------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index aed761cea7..324b677ff4 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -635,7 +635,6 @@ class EntriesManager(SearchQuerySet): **GENOTYPE_LOOKUP, COMP_HET_ALT: GENOTYPE_LOOKUP[HAS_ALT], } - NULLABLE_GENOTYPES = {REF_REF, HAS_REF} INHERITANCE_FILTERS = { **INHERITANCE_FILTERS, @@ -651,14 +650,13 @@ def annotations_model(self): def call_fields(self): return dict(self.model.CALL_FIELDS) - @property - def genotype_lookup(self): - return self.NULLABLE_GENOTYPE_LOOKUP if self.annotations_model.GENOTYPE_OVERRIDE_FIELDS else self.GENOTYPE_LOOKUP + def has_null_ref_genotypes(self): + return self.annotations_model.GENOTYPE_OVERRIDE_FIELDS @property - def nullable_genotypes(self): - return self.NULLABLE_GENOTYPES if self.annotations_model.GENOTYPE_OVERRIDE_FIELDS else [] + def genotype_lookup(self): + return self.NULLABLE_GENOTYPE_LOOKUP if self.has_null_ref_genotypes else self.GENOTYPE_LOOKUP @property def quality_filters(self): @@ -806,29 +804,23 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): - family_sample_gts = {} - family_sample_null_gts = {} + family_sample_genotypes = defaultdict(lambda: defaultdict(list)) family_affected_samples = defaultdict(lambda: defaultdict(list)) family_quality_samples = {} family_missing_type_samples = {} for s in sample_data: - sample_filters, nullable_gt_samples, quality_samples, missing_type_samples = self._family_sample_filters( - s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_affected_samples, s['family_guid'], + quality_samples, missing_type_samples = self._family_sample_filters( + s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, + family_sample_genotypes, family_affected_samples, s['family_guid'], is_multi_sample_family=s['family_guid'] in multi_sample_type_families, ) - if sample_filters: - family_sample_gts[s['family_guid']] = sample_filters - if nullable_gt_samples: - family_sample_null_gts[s['family_guid']] = nullable_gt_samples if quality_samples: family_quality_samples[s['family_guid']] = quality_samples if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples is_single_family = len(sample_data) == 1 and not multi_sample_type_families - inheritance_q, gt_filter_map = self._inheritance_q( - family_sample_gts, family_sample_null_gts, family_affected_samples, is_single_family, - ) + inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes, family_affected_samples) quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) return inheritance_q, quality_q, gt_filter_map, family_missing_type_samples @@ -844,20 +836,16 @@ def _get_family_sample_types(sample_data): multi_sample_type_families.add(s['family_guid']) return sample_type_families, multi_sample_type_families - def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_affected_samples, family_guid, is_multi_sample_family): - sample_filters = defaultdict(list) - nullable_gt_samples = defaultdict(list) + def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, family_guid, is_multi_sample_family): quality_samples = defaultdict(list) missing_type_samples = defaultdict(list) for sample in family_sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) for sample_type, sample_id in sample['sample_ids_by_type'].items(): + # TODO handle null genotype for known sample (i.e. manual genotype filter) if genotype is not None: - sample_filters[sample_type].append( - f"'{sample_id}', {self.genotype_lookup[genotype]}") - if genotype in self.nullable_genotypes: - nullable_gt_samples[sample_type].append(sample_id) + family_sample_genotypes[sample_id][genotype].append((family_guid, sample_type)) elif inheritance_mode == ANY_AFFECTED: family_affected_samples[sample_id][affected].append((family_guid, sample_type)) if (not quality_filter.get('affected_only')) or affected == AFFECTED: @@ -866,14 +854,14 @@ def _family_sample_filters(self, family_sample_data, inheritance_mode, individua missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS missing_type_samples[missing_type].append(sample_id) - return sample_filters, nullable_gt_samples, quality_samples, missing_type_samples + return quality_samples, missing_type_samples - def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples, is_single_family): + def _inheritance_q(self, sample_filters, any_affected_samples): map_template = 'map({value})[family_guid]' if not self.single_sample_type: map_template += '[sample_type::String]' - gt_filter_map = None + gt_map = None inheritance_q = None if any_affected_samples: mismatched_affected_samples = defaultdict(lambda: defaultdict(list)) @@ -893,35 +881,40 @@ def _inheritance_q(self, sample_filters, sample_null_gts, any_affected_samples, }) if mismatched_affected_samples: affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) - inheritance_q |= Q(calls__array_exists={ + inheritance_q |= Q(family_guid___in=list(mismatched_affected_samples.keys()), calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), - 'family': (list(mismatched_affected_samples.keys()), 'has({value}, family_guid)'), }) elif sample_filters: - if is_single_family: - gt_filter_map = ', '.join(next(iter(next(iter(sample_filters.values())).values()))) - map_template = 'map({value})' - else: - gt_filter_map = self._get_family_sample_map(sample_filters, is_nested_map=True) - gt_conditions = [ - {'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})')}, - {'sampleId': (gt_filter_map, 'not mapContains(' + map_template + ', {field})')}, - ] - if sample_null_gts: - if is_single_family: - null_gt_map = next(iter(next(iter(sample_null_gts.values())).values())) - map_template = '{value}' + mismatched_genotype_samples = defaultdict(lambda: defaultdict(list)) + samples_by_gt = defaultdict(list) + for sample_id, genotype_families in sample_genotypes.items(): + if len(genotype_families) > 1: + for genotype, family_data in genotype_families.items(): + for family_guid, sample_type in family_data: + mismatched_genotype_samples[family_guid][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") else: - null_gt_map = self._get_family_sample_map(sample_null_gts) - gt_conditions.append({ + for gt in self.genotype_lookup[next(iter(genotype_families))]: + samples_by_gt[gt].append(sample_id) + + gt_map = [f"{gt}, {samples}" for gt, samples in samples_by_gt.items()] + gt_conditions = {'gt': (gt_map, 'has(map({value})[{field}], x.sampleId)')} + if self.has_null_ref_genotypes and 0 in samples_by_gt: + gt_conditions = {'OR': [gt_conditions, { 'gt': (None, 'isNull({field})'), - 'sampleId': (null_gt_map, 'has(' + map_template + ', {field})'), + 'sampleId': (samples_by_gt[0], 'has({value}, {field})'), + }]} + + inheritance_q = Q(calls__array_all=gt_conditions) + + if mismatched_genotype_samples: + gt_filter_map = self._get_family_sample_map(mismatched_genotype_samples, is_nested_map=True) + inheritance_q |= Q(family_guid___in=list(mismatched_genotype_samples.keys()), calls__array_all={ + 'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})'), }) - inheritance_q = Q(calls__array_all={'OR': gt_conditions}) - return inheritance_q, gt_filter_map + return inheritance_q, gt_map def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None @@ -984,6 +977,7 @@ def _get_family_sample_map(self, family_samples, is_nested_map=False): return ', '.join(family_map) def _annotate_failed_family_samples(self, entries, gt_filter_map, family_missing_type_samples): + # TODO update entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ 'gt': (gt_filter_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), From 0d6e2833392c7c194ff334306823d09893631331 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 11:15:17 -0400 Subject: [PATCH 032/109] quality filter --- clickhouse_search/managers.py | 165 +++++++++++++------------- seqr/views/utils/permissions_utils.py | 3 +- 2 files changed, 85 insertions(+), 83 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 324b677ff4..49844250ab 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -806,22 +806,39 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): family_sample_genotypes = defaultdict(lambda: defaultdict(list)) family_affected_samples = defaultdict(lambda: defaultdict(list)) - family_quality_samples = {} family_missing_type_samples = {} for s in sample_data: - quality_samples, missing_type_samples = self._family_sample_filters( + missing_type_samples = self._family_sample_filters( s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, s['family_guid'], is_multi_sample_family=s['family_guid'] in multi_sample_type_families, ) - if quality_samples: - family_quality_samples[s['family_guid']] = quality_samples if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples - is_single_family = len(sample_data) == 1 and not multi_sample_type_families - inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes, family_affected_samples) - quality_q = self._quality_q(quality_filter, family_quality_samples, clinvar_override_q, is_single_family) + mismatched_affected_samples = defaultdict(lambda: defaultdict(list)) + affected_samples = [] + for sample_id, affected_families in family_affected_samples.items(): + if AFFECTED not in affected_families: + continue + if len(affected_families) > 1: + for family_guid, sample_type in affected_families[AFFECTED]: + mismatched_affected_samples[family_guid][sample_type].append(sample_id) + else: + affected_samples.append(sample_id) + + map_template = 'map({value})[family_guid]' + if not self.single_sample_type: + map_template += '[sample_type::String]' + + inheritance_q = None + gt_filter_map = None + if inheritance_mode == ANY_AFFECTED: + inheritance_q = self._any_affected_q(affected_samples, mismatched_affected_samples, map_template) + elif family_sample_genotypes: + inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes, map_template) + + quality_q = self._quality_q(quality_filter, affected_samples, mismatched_affected_samples, clinvar_override_q, map_template) return inheritance_q, quality_q, gt_filter_map, family_missing_type_samples @@ -837,82 +854,61 @@ def _get_family_sample_types(sample_data): return sample_type_families, multi_sample_type_families def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, family_guid, is_multi_sample_family): - quality_samples = defaultdict(list) missing_type_samples = defaultdict(list) for sample in family_sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) for sample_type, sample_id in sample['sample_ids_by_type'].items(): - # TODO handle null genotype for known sample (i.e. manual genotype filter) + # TODO handle null genotype for known sample (i.e. manual genotype filter, unknown affected) if genotype is not None: family_sample_genotypes[sample_id][genotype].append((family_guid, sample_type)) - elif inheritance_mode == ANY_AFFECTED: + if inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): family_affected_samples[sample_id][affected].append((family_guid, sample_type)) - if (not quality_filter.get('affected_only')) or affected == AFFECTED: - quality_samples[sample_type].append(sample_id) if is_multi_sample_family and len(sample['sample_ids_by_type']) == 1: missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS missing_type_samples[missing_type].append(sample_id) - return quality_samples, missing_type_samples + return missing_type_samples - def _inheritance_q(self, sample_filters, any_affected_samples): - map_template = 'map({value})[family_guid]' - if not self.single_sample_type: - map_template += '[sample_type::String]' - - gt_map = None - inheritance_q = None - if any_affected_samples: - mismatched_affected_samples = defaultdict(lambda: defaultdict(list)) - affected_samples = [] - for sample_id, affected_families in any_affected_samples.items(): - if AFFECTED not in affected_families: - continue - if len(affected_families) > 1: - for family_guid, sample_type in affected_families[AFFECTED]: - mismatched_affected_samples[family_guid][sample_type].append(sample_id) - else: - affected_samples.append(sample_id) - - inheritance_q = Q(calls__array_exists={ + def _any_affected_q(self, affected_samples, mismatched_affected_samples, map_template): + inheritance_q = Q(calls__array_exists={ + 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), + 'sampleId': (affected_samples, 'has({value}, {field})'), + }) + if mismatched_affected_samples: + affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) + inheritance_q |= Q(family_guid___in=list(mismatched_affected_samples.keys()), calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_samples, 'has({value}, {field})'), + 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), + }) + return inheritance_q + + def _inheritance_q(self, sample_genotypes, map_template): + mismatched_genotype_samples = defaultdict(lambda: defaultdict(list)) + samples_by_gt = defaultdict(list) + for sample_id, genotype_families in sample_genotypes.items(): + if len(genotype_families) > 1: + for genotype, family_data in genotype_families.items(): + for family_guid, sample_type in family_data: + mismatched_genotype_samples[family_guid][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") + else: + for gt in self.genotype_lookup[next(iter(genotype_families))]: + samples_by_gt[gt].append(sample_id) + + gt_map = [f"{gt}, {samples}" for gt, samples in samples_by_gt.items()] + gt_conditions = {'gt': (gt_map, 'has(map({value})[{field}], x.sampleId)')} + if self.has_null_ref_genotypes and 0 in samples_by_gt: + gt_conditions = {'OR': [gt_conditions, { + 'gt': (None, 'isNull({field})'), + 'sampleId': (samples_by_gt[0], 'has({value}, {field})'), + }]} + inheritance_q = Q(calls__array_all=gt_conditions) + + if mismatched_genotype_samples: + gt_filter_map = self._get_family_sample_map(mismatched_genotype_samples, is_nested_map=True) + inheritance_q |= Q(family_guid___in=list(mismatched_genotype_samples.keys()), calls__array_all={ + 'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})'), }) - if mismatched_affected_samples: - affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) - inheritance_q |= Q(family_guid___in=list(mismatched_affected_samples.keys()), calls__array_exists={ - 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), - }) - - elif sample_filters: - mismatched_genotype_samples = defaultdict(lambda: defaultdict(list)) - samples_by_gt = defaultdict(list) - for sample_id, genotype_families in sample_genotypes.items(): - if len(genotype_families) > 1: - for genotype, family_data in genotype_families.items(): - for family_guid, sample_type in family_data: - mismatched_genotype_samples[family_guid][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") - else: - for gt in self.genotype_lookup[next(iter(genotype_families))]: - samples_by_gt[gt].append(sample_id) - - gt_map = [f"{gt}, {samples}" for gt, samples in samples_by_gt.items()] - gt_conditions = {'gt': (gt_map, 'has(map({value})[{field}], x.sampleId)')} - if self.has_null_ref_genotypes and 0 in samples_by_gt: - gt_conditions = {'OR': [gt_conditions, { - 'gt': (None, 'isNull({field})'), - 'sampleId': (samples_by_gt[0], 'has({value}, {field})'), - }]} - - inheritance_q = Q(calls__array_all=gt_conditions) - - if mismatched_genotype_samples: - gt_filter_map = self._get_family_sample_map(mismatched_genotype_samples, is_nested_map=True) - inheritance_q |= Q(family_guid___in=list(mismatched_genotype_samples.keys()), calls__array_all={ - 'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})'), - }) return inheritance_q, gt_map @@ -926,7 +922,7 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _quality_q(self, quality_filter, quality_samples, clinvar_override_q, is_single_family): + def _quality_q(self, quality_filter, affected_samples, mismatched_affected_samples, clinvar_override_q, map_template): quality_filter_conditions = {} for field, scale, *filters in self.quality_filters: @@ -941,21 +937,26 @@ def _quality_q(self, quality_filter, quality_samples, clinvar_override_q, is_sin if not quality_filter_conditions: return None - template = 'not has(' - if is_single_family: - quality_samples_map = next(iter(next(iter(quality_samples.values())).values())) - template += '{value}' - else: - quality_samples_map = self._get_family_sample_map(quality_samples) - template += 'map({value})[family_guid]' - if not self.single_sample_type: - template += '[sample_type::String]' - quality_filter_conditions = {'OR': [ - quality_filter_conditions, - {'sampleId': (quality_samples_map, template + ', {field})')} - ]} + affected_only = quality_filter.get('affected_only') + if affected_only: + quality_filter_conditions = {'OR': [ + quality_filter_conditions, + {'sampleId': (affected_samples, 'not has({value}, {field})')} + ]} quality_q = Q(calls__array_all=quality_filter_conditions) + + if affected_only and mismatched_affected_samples: + affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) + mismatch_families = list(mismatched_affected_samples.keys()) + quality_q = (quality_q & ~Q(family_guid___in=mismatch_families)) | Q( + family_guid___in=mismatch_families, + calls__array_all={'OR': [ + quality_filter_conditions, + {'sampleId': (affected_sample_map, 'not has(' + map_template + ', {field})')} + ]}, + ) + if clinvar_override_q: quality_q |= clinvar_override_q diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index 9617f8734b..d0142090fe 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -258,7 +258,7 @@ def get_project_guids_user_can_view(user, limit_data_manager=True): return list(Project.objects.values_list('guid', flat=True)) cache_key = 'projects__{}'.format(user) - project_guids = safe_redis_get_json(cache_key) + project_guids = None #safe_redis_get_json(cache_key) if project_guids is not None: return project_guids @@ -270,6 +270,7 @@ def get_project_guids_user_can_view(user, limit_data_manager=True): ).filter(workspace__in=workspaces) else: projects = get_objects_for_user(user, CAN_VIEW, Project) + projects = projects.filter(workspace_namespace__in=['gregor-consortium','seqr-access']) projects = projects | Project.objects.filter(all_user_demo=True, is_demo=True) From 5834f9d1991f5016203aa5f308628974536283af Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 12:01:25 -0400 Subject: [PATCH 033/109] fix map construction --- clickhouse_search/managers.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 49844250ab..236ab92ba3 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -895,14 +895,10 @@ def _inheritance_q(self, sample_genotypes, map_template): for gt in self.genotype_lookup[next(iter(genotype_families))]: samples_by_gt[gt].append(sample_id) - gt_map = [f"{gt}, {samples}" for gt, samples in samples_by_gt.items()] - gt_conditions = {'gt': (gt_map, 'has(map({value})[{field}], x.sampleId)')} - if self.has_null_ref_genotypes and 0 in samples_by_gt: - gt_conditions = {'OR': [gt_conditions, { - 'gt': (None, 'isNull({field})'), - 'sampleId': (samples_by_gt[0], 'has({value}, {field})'), - }]} - inheritance_q = Q(calls__array_all=gt_conditions) + if self.has_null_ref_genotypes: + samples_by_gt[-1] = samples_by_gt[0] + gt_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) + inheritance_q = Q(calls__array_all={'gt': (gt_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) if mismatched_genotype_samples: gt_filter_map = self._get_family_sample_map(mismatched_genotype_samples, is_nested_map=True) From d457969dfc7785778e7d6d29e9df81fa50b7be3c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 12:10:25 -0400 Subject: [PATCH 034/109] update fixture data --- clickhouse_search/fixtures/clickhouse_search.json | 1 - clickhouse_search/test_utils.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index ff019d9ee5..78e8cac1ff 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -393,7 +393,6 @@ "filters": [], "sign": 1, "calls": [ - ["NA20872", 1, 99, 0.34285714285714286, 35], ["NA20870", 1, 99, 0.6785714285714286, 28] ] } diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 2f3758e368..3313ef981b 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -22,7 +22,7 @@ VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 8, 'hom': 3}}} VARIANT2 = {**deepcopy(HAIL_VARIANT2), 'key': 2, 'populations': {**deepcopy(HAIL_VARIANT2)['populations'], 'seqr': {'ac': 7, 'hom': 2}}} -VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 6, 'hom': 0}}} +VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 5, 'hom': 0}}} VARIANT4 = {**deepcopy(HAIL_VARIANT4), 'key': 4, 'populations': {**deepcopy(HAIL_VARIANT4)['populations'], 'seqr': {'ac': 4, 'hom': 1}}} PROJECT_2_VARIANT = {**deepcopy(HAIL_PROJECT_2_VARIANT), 'key': 5, 'populations': {**deepcopy(HAIL_PROJECT_2_VARIANT)['populations'], 'seqr': {'ac': 2, 'hom': 0}}} MITO_VARIANT1 = {**deepcopy(HAIL_MITO_VARIANT1), 'key': 6, 'populations': {**deepcopy(HAIL_MITO_VARIANT1)['populations'], 'seqr': {'ac': 0}, 'seqr_heteroplasmy': {'ac': 1}}} From 7774096cc21ce38d4a6385bd055f285787ed9676 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 12:35:46 -0400 Subject: [PATCH 035/109] handle any allowed genotypes --- clickhouse_search/managers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 236ab92ba3..baef9592c0 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -628,6 +628,7 @@ class EntriesManager(SearchQuerySet): ALT_ALT: [2], HAS_ALT: [1, 2], HAS_REF: [0, 1], + None: [-1, 0, 1, 2], } COMP_HET_ALT = 'COMP_HET_ALT' GENOTYPE_LOOKUP[COMP_HET_ALT] = GENOTYPE_LOOKUP[REF_ALT] @@ -859,8 +860,7 @@ def _family_sample_filters(self, family_sample_data, inheritance_mode, individua affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) for sample_type, sample_id in sample['sample_ids_by_type'].items(): - # TODO handle null genotype for known sample (i.e. manual genotype filter, unknown affected) - if genotype is not None: + if inheritance_mode or individual_genotype_filter: family_sample_genotypes[sample_id][genotype].append((family_guid, sample_type)) if inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): family_affected_samples[sample_id][affected].append((family_guid, sample_type)) @@ -896,7 +896,7 @@ def _inheritance_q(self, sample_genotypes, map_template): samples_by_gt[gt].append(sample_id) if self.has_null_ref_genotypes: - samples_by_gt[-1] = samples_by_gt[0] + samples_by_gt[-1] += samples_by_gt[0] gt_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) inheritance_q = Q(calls__array_all={'gt': (gt_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) From 83150a85735ffa042c17d869ee9b57d39e6b31d8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 12:39:06 -0400 Subject: [PATCH 036/109] cleaner nullabel genotypes --- clickhouse_search/managers.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index baef9592c0..1c475852a8 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -635,6 +635,8 @@ class EntriesManager(SearchQuerySet): NULLABLE_GENOTYPE_LOOKUP = { **GENOTYPE_LOOKUP, COMP_HET_ALT: GENOTYPE_LOOKUP[HAS_ALT], + REF_REF: [-1] + GENOTYPE_LOOKUP[REF_REF], + HAS_REF: [-1] + GENOTYPE_LOOKUP[HAS_REF], } INHERITANCE_FILTERS = { @@ -651,13 +653,9 @@ def annotations_model(self): def call_fields(self): return dict(self.model.CALL_FIELDS) - @property - def has_null_ref_genotypes(self): - return self.annotations_model.GENOTYPE_OVERRIDE_FIELDS - @property def genotype_lookup(self): - return self.NULLABLE_GENOTYPE_LOOKUP if self.has_null_ref_genotypes else self.GENOTYPE_LOOKUP + return self.NULLABLE_GENOTYPE_LOOKUP if self.annotations_model.GENOTYPE_OVERRIDE_FIELDS else self.GENOTYPE_LOOKUP @property def quality_filters(self): @@ -895,8 +893,6 @@ def _inheritance_q(self, sample_genotypes, map_template): for gt in self.genotype_lookup[next(iter(genotype_families))]: samples_by_gt[gt].append(sample_id) - if self.has_null_ref_genotypes: - samples_by_gt[-1] += samples_by_gt[0] gt_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) inheritance_q = Q(calls__array_all={'gt': (gt_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) From 2c61836cfabec79c7a2d5267584d23877e670186 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 13:13:21 -0400 Subject: [PATCH 037/109] do not allow mismatches in affected status --- clickhouse_search/managers.py | 100 +++++++++++++--------------------- 1 file changed, 37 insertions(+), 63 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1c475852a8..f5e80583c4 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -10,7 +10,7 @@ from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ GroupArrayIntersect, DictGet, DictGetOrDefault, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat -from seqr.models import Sample +from seqr.models import Sample, Project from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ @@ -815,29 +815,39 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i if missing_type_samples: family_missing_type_samples[s['family_guid']] = missing_type_samples - mismatched_affected_samples = defaultdict(lambda: defaultdict(list)) - affected_samples = [] - for sample_id, affected_families in family_affected_samples.items(): - if AFFECTED not in affected_families: - continue - if len(affected_families) > 1: - for family_guid, sample_type in affected_families[AFFECTED]: - mismatched_affected_samples[family_guid][sample_type].append(sample_id) - else: - affected_samples.append(sample_id) + mismatched_samples = { + sample_id: {project for projects in projects_map.values() for project in projects} + for sample_id, projects_map in {**family_affected_samples, **family_sample_genotypes}.items() if len(projects_map) > 1 + } + if mismatched_samples: + from seqr.utils.search.utils import InvalidSearchException + project_name_map = dict(Project.objects.filter( + guid__in={p for projects in mismatched_samples.values() for p in projects} + ).values_list('guid', 'name')) + sample_summaries = [ + f'{sample_id} ({"/ ".join([project_name_map[p] for p in projects])})' + for sample_id, projects in mismatched_samples.items() + ] + raise InvalidSearchException( + 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + + ', '.join(sample_summaries), + ) - map_template = 'map({value})[family_guid]' - if not self.single_sample_type: - map_template += '[sample_type::String]' + affected_samples = [ + sample_id for sample_id, affected_families in family_affected_samples.items() if AFFECTED in affected_families + ] inheritance_q = None gt_filter_map = None if inheritance_mode == ANY_AFFECTED: - inheritance_q = self._any_affected_q(affected_samples, mismatched_affected_samples, map_template) + inheritance_q = Q(calls__array_exists={ + 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), + 'sampleId': (affected_samples, 'has({value}, {field})'), + }) elif family_sample_genotypes: - inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes, map_template) + inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes) - quality_q = self._quality_q(quality_filter, affected_samples, mismatched_affected_samples, clinvar_override_q, map_template) + quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) return inheritance_q, quality_q, gt_filter_map, family_missing_type_samples @@ -852,57 +862,33 @@ def _get_family_sample_types(sample_data): multi_sample_type_families.add(s['family_guid']) return sample_type_families, multi_sample_type_families - def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, family_guid, is_multi_sample_family): + def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, project_guid, is_multi_sample_family): missing_type_samples = defaultdict(list) for sample in family_sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) for sample_type, sample_id in sample['sample_ids_by_type'].items(): if inheritance_mode or individual_genotype_filter: - family_sample_genotypes[sample_id][genotype].append((family_guid, sample_type)) + family_sample_genotypes[sample_id][genotype].append(project_guid) if inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): - family_affected_samples[sample_id][affected].append((family_guid, sample_type)) + family_affected_samples[sample_id][affected].append(project_guid) if is_multi_sample_family and len(sample['sample_ids_by_type']) == 1: missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS missing_type_samples[missing_type].append(sample_id) return missing_type_samples - def _any_affected_q(self, affected_samples, mismatched_affected_samples, map_template): - inheritance_q = Q(calls__array_exists={ - 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_samples, 'has({value}, {field})'), - }) - if mismatched_affected_samples: - affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) - inheritance_q |= Q(family_guid___in=list(mismatched_affected_samples.keys()), calls__array_exists={ - 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), - 'sampleId': (affected_sample_map, 'has(' + map_template + ', {field})'), - }) - return inheritance_q - def _inheritance_q(self, sample_genotypes, map_template): - mismatched_genotype_samples = defaultdict(lambda: defaultdict(list)) + def _inheritance_q(self, sample_genotypes): samples_by_gt = defaultdict(list) for sample_id, genotype_families in sample_genotypes.items(): - if len(genotype_families) > 1: - for genotype, family_data in genotype_families.items(): - for family_guid, sample_type in family_data: - mismatched_genotype_samples[family_guid][sample_type].append(f"'{sample_id}', {self.genotype_lookup[genotype]}") - else: - for gt in self.genotype_lookup[next(iter(genotype_families))]: - samples_by_gt[gt].append(sample_id) - - gt_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) - inheritance_q = Q(calls__array_all={'gt': (gt_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) + for gt in self.genotype_lookup[next(iter(genotype_families))]: + samples_by_gt[gt].append(sample_id) - if mismatched_genotype_samples: - gt_filter_map = self._get_family_sample_map(mismatched_genotype_samples, is_nested_map=True) - inheritance_q |= Q(family_guid___in=list(mismatched_genotype_samples.keys()), calls__array_all={ - 'gt': (gt_filter_map, 'has(' + map_template + '[x.sampleId], {field})'), - }) + gt_filter_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) + inheritance_q = Q(calls__array_all={'gt': (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) - return inheritance_q, gt_map + return inheritance_q, gt_filter_map def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None @@ -914,7 +900,7 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _quality_q(self, quality_filter, affected_samples, mismatched_affected_samples, clinvar_override_q, map_template): + def _quality_q(self, quality_filter, affected_samples, clinvar_override_q): quality_filter_conditions = {} for field, scale, *filters in self.quality_filters: @@ -937,18 +923,6 @@ def _quality_q(self, quality_filter, affected_samples, mismatched_affected_sampl ]} quality_q = Q(calls__array_all=quality_filter_conditions) - - if affected_only and mismatched_affected_samples: - affected_sample_map = self._get_family_sample_map(mismatched_affected_samples) - mismatch_families = list(mismatched_affected_samples.keys()) - quality_q = (quality_q & ~Q(family_guid___in=mismatch_families)) | Q( - family_guid___in=mismatch_families, - calls__array_all={'OR': [ - quality_filter_conditions, - {'sampleId': (affected_sample_map, 'not has(' + map_template + ', {field})')} - ]}, - ) - if clinvar_override_q: quality_q |= clinvar_override_q From 00f5074f2b0e0f0838ea191d24a95e4255d3e7e7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 14:51:12 -0400 Subject: [PATCH 038/109] cleaner logic --- clickhouse_search/managers.py | 102 ++++++++++------------------------ clickhouse_search/search.py | 12 +++- 2 files changed, 40 insertions(+), 74 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index f5e80583c4..9745c86a7d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -10,7 +10,7 @@ from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ GroupArrayIntersect, DictGet, DictGetOrDefault, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat -from seqr.models import Sample, Project +from seqr.models import Sample from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ @@ -764,12 +764,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q = None quality_q = None - gt_filter_map = None + gt_filter = None family_missing_type_samples = None quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') if inheritance_mode or individual_genotype_filter or quality_filter: - inheritance_q, quality_q, gt_filter_map, family_missing_type_samples = self._get_inheritance_quality_qs( + inheritance_q, quality_q, gt_filter, family_missing_type_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected=(inheritance_filter or {}).get('affected') or {}, ) @@ -780,9 +780,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit entries = entries.filter(q) if multi_sample_type_families: - if gt_filter_map: + if gt_filter: inheritance_q |= multi_sample_type_family_q - entries = self._annotate_failed_family_samples(entries, gt_filter_map, family_missing_type_samples) + entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples) elif inheritance_q is not None: entries = entries.annotate(passes_inheritance=inheritance_q) inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q @@ -803,53 +803,39 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): - family_sample_genotypes = defaultdict(lambda: defaultdict(list)) - family_affected_samples = defaultdict(lambda: defaultdict(list)) - family_missing_type_samples = {} + samples_by_gt = defaultdict(list) + affected_samples = [] + family_missing_type_samples = defaultdict(lambda: defaultdict(list)) for s in sample_data: - missing_type_samples = self._family_sample_filters( - s, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, - family_sample_genotypes, family_affected_samples, s['family_guid'], - is_multi_sample_family=s['family_guid'] in multi_sample_type_families, - ) - if missing_type_samples: - family_missing_type_samples[s['family_guid']] = missing_type_samples - - mismatched_samples = { - sample_id: {project for projects in projects_map.values() for project in projects} - for sample_id, projects_map in {**family_affected_samples, **family_sample_genotypes}.items() if len(projects_map) > 1 - } - if mismatched_samples: - from seqr.utils.search.utils import InvalidSearchException - project_name_map = dict(Project.objects.filter( - guid__in={p for projects in mismatched_samples.values() for p in projects} - ).values_list('guid', 'name')) - sample_summaries = [ - f'{sample_id} ({"/ ".join([project_name_map[p] for p in projects])})' - for sample_id, projects in mismatched_samples.items() - ] - raise InvalidSearchException( - 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + - ', '.join(sample_summaries), - ) - - affected_samples = [ - sample_id for sample_id, affected_families in family_affected_samples.items() if AFFECTED in affected_families - ] + for sample in s['samples']: + affected = custom_affected.get(sample['individual_guid']) or sample['affected'] + genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) + sample_ids = list(sample['sample_ids_by_type'].values()) + if affected == AFFECTED and inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): + affected_samples += sample_ids + if inheritance_mode or individual_genotype_filter: + for gt in self.genotype_lookup[genotype]: + samples_by_gt[gt] += sample_ids + if s['family_guid'] in multi_sample_type_families and len(sample_ids) == 1: + sample_type = next(iter(sample['sample_ids_by_type'])) + missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + family_missing_type_samples[s['family_guid']][missing_type].append(sample_ids[0]) inheritance_q = None - gt_filter_map = None + gt_filter = None if inheritance_mode == ANY_AFFECTED: inheritance_q = Q(calls__array_exists={ 'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'), 'sampleId': (affected_samples, 'has({value}, {field})'), }) - elif family_sample_genotypes: - inheritance_q, gt_filter_map = self._inheritance_q(family_sample_genotypes) + elif samples_by_gt: + gt_filter_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) + gt_filter = (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)') + inheritance_q = Q(calls__array_all={'gt': gt_filter}) quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) - return inheritance_q, quality_q, gt_filter_map, family_missing_type_samples + return inheritance_q, quality_q, gt_filter, family_missing_type_samples @staticmethod def _get_family_sample_types(sample_data): @@ -862,34 +848,6 @@ def _get_family_sample_types(sample_data): multi_sample_type_families.add(s['family_guid']) return sample_type_families, multi_sample_type_families - def _family_sample_filters(self, family_sample_data, inheritance_mode, individual_genotype_filter, quality_filter, custom_affected, family_sample_genotypes, family_affected_samples, project_guid, is_multi_sample_family): - missing_type_samples = defaultdict(list) - for sample in family_sample_data['samples']: - affected = custom_affected.get(sample['individual_guid']) or sample['affected'] - genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) - for sample_type, sample_id in sample['sample_ids_by_type'].items(): - if inheritance_mode or individual_genotype_filter: - family_sample_genotypes[sample_id][genotype].append(project_guid) - if inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): - family_affected_samples[sample_id][affected].append(project_guid) - if is_multi_sample_family and len(sample['sample_ids_by_type']) == 1: - missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS - missing_type_samples[missing_type].append(sample_id) - - return missing_type_samples - - - def _inheritance_q(self, sample_genotypes): - samples_by_gt = defaultdict(list) - for sample_id, genotype_families in sample_genotypes.items(): - for gt in self.genotype_lookup[next(iter(genotype_families))]: - samples_by_gt[gt].append(sample_id) - - gt_filter_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]]) - inheritance_q = Q(calls__array_all={'gt': (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')}) - - return inheritance_q, gt_filter_map - def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None if individual_genotype_filter: @@ -943,12 +901,10 @@ def _get_family_sample_map(self, family_samples, is_nested_map=False): return ', '.join(family_map) - def _annotate_failed_family_samples(self, entries, gt_filter_map, family_missing_type_samples): - # TODO update + def _annotate_failed_family_samples(self, entries, gt_filter, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ - 'gt': (gt_filter_map, 'not has(map({value})[family_guid][sample_type::String][x.sampleId], {field})'), - 'sampleId': (gt_filter_map, 'mapContains(map({value})[family_guid][sample_type::String], {field})'), + 'gt': (gt_filter[0], f'not {gt_filter[1]}'), }]), mapped_expression='tuple(family_guid, x.sampleId)', )) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index e6145a40a3..0a44053529 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -305,8 +305,18 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): and transcript.get('spliceregion', {}).get('extended_intronic_splice_region_variant') == minimal_transcript.get('extendedIntronicSpliceRegionVariant')) - def _get_sample_data(samples): + mismatch_affected_samples = samples.values('sample_id').annotate( + projects=ArrayAgg('individual__family__project__name', distinct=True), + affected=ArrayAgg('individual__affected', distinct=True), + ).filter(affected__len__gt=1) + if mismatch_affected_samples: + from seqr.utils.search.utils import InvalidSearchException + raise InvalidSearchException( + 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + + ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), + ) + sample_data = samples.values( 'dataset_type', family_guid=F('individual__family__guid'), project_guid=F('individual__family__project__guid'), ).annotate( From 15fddb80c2e0852fef3a696a5dc9260facc4429a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:10:12 -0400 Subject: [PATCH 039/109] fix test --- clickhouse_search/search_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index aca721fd73..59bbdf4fe9 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -15,6 +15,7 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant +from clickhouse_search.models import EntriesGcnv from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper @@ -1096,6 +1097,9 @@ def test_secondary_annotations_filter(self): **GCNV_VARIANT4, 'genotypes': {k: v for k, v in GCNV_VARIANT4['genotypes'].items() if k != 'I000005_hg00732'} } Sample.objects.filter(guid='S000146_hg00732').update(is_active=False) + for entry in EntriesGcnv.objects.filter(family_guid='F000002_2'): + entry.calls = [list(call.values()) for call in [entry.calls[0], entry.calls[2]]] + entry.save(using='clickhouse_write') self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, missing_gt_gcnv_variant]], inheritance_mode='compound_het', pathogenicity=pathogenicity, locus=None, From afbd83c82bf53a041e03b342b82439a4cabd03ac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:13:36 -0400 Subject: [PATCH 040/109] undo test code --- seqr/views/utils/permissions_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index d0142090fe..9617f8734b 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -258,7 +258,7 @@ def get_project_guids_user_can_view(user, limit_data_manager=True): return list(Project.objects.values_list('guid', flat=True)) cache_key = 'projects__{}'.format(user) - project_guids = None #safe_redis_get_json(cache_key) + project_guids = safe_redis_get_json(cache_key) if project_guids is not None: return project_guids @@ -270,7 +270,6 @@ def get_project_guids_user_can_view(user, limit_data_manager=True): ).filter(workspace__in=workspaces) else: projects = get_objects_for_user(user, CAN_VIEW, Project) - projects = projects.filter(workspace_namespace__in=['gregor-consortium','seqr-access']) projects = projects | Project.objects.filter(all_user_demo=True, is_demo=True) From 482a6573597c9580194e1de949c43c0cb64f1758 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:35:06 -0400 Subject: [PATCH 041/109] Revert "do not use gnomad af entries column" This reverts commit cad2e8441fe8114f78cd48c654995c8307389ea7. --- clickhouse_search/managers.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index aed761cea7..5a9aaedbda 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -697,18 +697,17 @@ def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, p pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None gnomad_filter = (freqs or {}).get('gnomad_genomes') or {} - af_cutoff = gnomad_filter.get('af') - if af_cutoff is None and any(gnomad_filter.get(field) is not None for field in ['ac', 'hh']): - af_cutoff = PATH_FREQ_OVERRIDE_CUTOFF - if af_cutoff is not None and any(obj.name.startswith('gnomadgenomes') for obj in self.annotations_model._meta.related_objects): - entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( - 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", - output_field=models.DecimalField(), - )) - af_q = Q(gnomad_genomes_af__lte=af_cutoff) - if clinvar_override_q is not None: - af_q |= (clinvar_override_q & Q(gnomad_genomes_af__lte=PATH_FREQ_OVERRIDE_CUTOFF)) - entries = entries.filter(af_q) + if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= PATH_FREQ_OVERRIDE_CUTOFF or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): + entries = entries.filter(is_gnomad_gt_5_percent=False) + if (gnomad_filter.get('af') or 1) < PATH_FREQ_OVERRIDE_CUTOFF: + entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( + 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", + output_field=models.DecimalField(), + )) + af_q = Q(gnomad_genomes_af__lte=gnomad_filter['af']) + if clinvar_override_q is not None: + af_q |= clinvar_override_q + entries = entries.filter(af_q) if (annotations or {}).get(NEW_SV_FIELD) and 'newCall' in self.call_fields: entries = entries.filter(calls__array_exists={'newCall': (None, '{field}')}) From 7ca44964fa6f499b8a8b9aa04ee00649d33321e4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:35:19 -0400 Subject: [PATCH 042/109] Revert "test fixtures" This reverts commit 2322f2e2cb73ccee247e220646ea75c0fa85f1cb. --- .../fixtures/clickhouse_search.json | 49 ------------------- clickhouse_search/search_tests.py | 2 - 2 files changed, 51 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index ff019d9ee5..c2f05942bc 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -264,48 +264,6 @@ ], "pathogenicity": "Conflicting_classifications_of_pathogenicity" } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 1, - "fields": { - "key": 1, - "filter_af": 0.04028 - } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 3, - "fields": { - "key": 3, - "filter_af": 0.47978 - } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 4, - "fields": { - "key": 4, - "filter_af": 0.0015 - } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 5, - "fields": { - "key": 5, - "filter_af": 0.00233 - } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 6, - "fields": { - "key": 6, - "filter_af": 0.00011 - } -}, { - "model": "clickhouse_search.gnomadgenomessnvindel", - "pk": 7, - "fields": { - "key": 7, - "filter_af": 0.0002 - } }, { "model": "clickhouse_search.entriessnvindel", "pk": 1, @@ -655,13 +613,6 @@ "key": 21, "variant_id": "1-10439-AC-A" } -}, { - "model": "clickhouse_search.gnomadgenomesgrch37snvindel", - "pk": 11, - "fields": { - "key": 11, - "filter_af": 0.82844 - } }, { "model": "clickhouse_search.annotationsmito", "pk": 6, diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index aca721fd73..d3c466f762 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -39,8 +39,6 @@ def setUpTestData(cls): cursor.execute(f'SYSTEM REFRESH VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') cursor.execute(f'SYSTEM WAIT VIEW "{table_base}/project_gt_stats_to_gt_stats_mv"') cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') - if table_base.endswith('SNV_INDEL'): - cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gnomad_genomes_dict"') Project.objects.update(genome_version='38') def _assert_expected_search(self, expected_results, gene_counts=None, inheritance_mode=None, inheritance_filter=None, quality_filter=None, cached_variant_fields=None, sort='xpos', results_model=None, **search_kwargs): From d6e0f4862b150a11dbe99598309175bc0ecba6eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:35:36 -0400 Subject: [PATCH 043/109] Revert "clean up" This reverts commit 850dec609d86da4fed710d54901676eb01fbb5f9. --- clickhouse_search/backend/functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 6dc98c2286..6cdf3552c2 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -127,6 +127,7 @@ class DictGet(Func): class DictGetOrDefault(DictGet): function = 'dictGetOrDefault' + template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' class If(Func): From 5a99a3662eb55624ca929a0f749b5eb02e4844a6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:37:54 -0400 Subject: [PATCH 044/109] Revert "use dict to prefilter gnomad" This reverts commit fb11f923b04662081f134f2234ca4007e6237e98. --- clickhouse_search/backend/functions.py | 5 ----- clickhouse_search/managers.py | 22 +++++----------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 6cdf3552c2..1a3300a220 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -125,11 +125,6 @@ class DictGet(Func): template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' -class DictGetOrDefault(DictGet): - function = 'dictGetOrDefault' - template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' - - class If(Func): function = 'if' template = '%(function)s(%(condition)s%(expressions)s)' diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 5a9aaedbda..60689fa89d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -9,7 +9,7 @@ from clickhouse_search.backend.fields import NestedField, NamedTupleField from clickhouse_search.backend.functions import Array, ArrayConcat, ArrayDistinct, ArrayFilter, ArrayFold, \ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ - GroupArrayIntersect, DictGet, DictGetOrDefault, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat + GroupArrayIntersect, DictGet, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat from seqr.models import Sample from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ @@ -683,7 +683,7 @@ def genotype_fields(self): def clinvar_model(self): return self.model.clinvar_join.rel.related_model - def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, pathogenicity=None, **kwargs): + def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): entries = self.filter_intervals(**(parsed_locus or {})) entries = self._join_annotations(entries) @@ -693,26 +693,14 @@ def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, p if (freqs or {}).get(callset_filter_field) and self.annotations_model.SEQR_POPULATIONS: entries = self._filter_seqr_frequency(entries, **freqs[callset_filter_field]) - clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( - pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), - ) if self._has_clinvar() else None gnomad_filter = (freqs or {}).get('gnomad_genomes') or {} - if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= PATH_FREQ_OVERRIDE_CUTOFF or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): + if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= 0.05 or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): entries = entries.filter(is_gnomad_gt_5_percent=False) - if (gnomad_filter.get('af') or 1) < PATH_FREQ_OVERRIDE_CUTOFF: - entries = entries.annotate(gnomad_genomes_af=DictGetOrDefault( - 'key', Value(0), dict_name=f"{self.table_basename}/gnomad_genomes_dict", fields="'filter_af'", - output_field=models.DecimalField(), - )) - af_q = Q(gnomad_genomes_af__lte=gnomad_filter['af']) - if clinvar_override_q is not None: - af_q |= clinvar_override_q - entries = entries.filter(af_q) if (annotations or {}).get(NEW_SV_FIELD) and 'newCall' in self.call_fields: entries = entries.filter(calls__array_exists={'newCall': (None, '{field}')}) - return self._search_call_data(entries, sample_data, clinvar_override_q=clinvar_override_q, **kwargs) + return self._search_call_data(entries, sample_data, **kwargs) def _join_annotations(self, entries): if self._has_clinvar(): @@ -742,7 +730,7 @@ def result_values(self, sample_data): def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') - def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, clinvar_override_q=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): project_guids = {s['project_guid'] for s in sample_data} project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) entries = entries.filter(project_filter) From a04a258ee5e31a92605eab68ef9ba0e1691a70eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:39:40 -0400 Subject: [PATCH 045/109] fix merge --- clickhouse_search/managers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 60689fa89d..2c55feb326 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -759,6 +759,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') if inheritance_mode or individual_genotype_filter or quality_filter: + clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( + pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), + ) if self._has_clinvar() else None inheritance_q, quality_q, gt_filter_map, family_missing_type_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected=(inheritance_filter or {}).get('affected') or {}, From d6f79dacc8b59f597082fe2f41ef19b2ff61b383 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:42:30 -0400 Subject: [PATCH 046/109] correctl use pruning" --- clickhouse_search/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 2c55feb326..6b8b348cce 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -695,7 +695,10 @@ def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, * gnomad_filter = (freqs or {}).get('gnomad_genomes') or {} if hasattr(self.model, 'is_gnomad_gt_5_percent') and ((gnomad_filter.get('af') or 1) <= 0.05 or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh'])): - entries = entries.filter(is_gnomad_gt_5_percent=False) + # Passing field=Value(False) to the django filter causes the SQL to evaluate to "field = false", + # while passing field=False evaluates to "NOT field". + # For fields used for pruning the table based on the order_by for the table, the former is needed + entries = entries.filter(is_gnomad_gt_5_percent=Value(False)) if (annotations or {}).get(NEW_SV_FIELD) and 'newCall' in self.call_fields: entries = entries.filter(calls__array_exists={'newCall': (None, '{field}')}) From 5e8bee30498a4324e0b0754bc4a92c9496bc32e1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 15:51:40 -0400 Subject: [PATCH 047/109] drop full gnamd tables and dicts --- ...move_gnomadgenomessnvindel_key_and_more.py | 27 +++++++++++++++++++ clickhouse_search/models.py | 22 --------------- 2 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 clickhouse_search/migrations/0015_remove_gnomadgenomessnvindel_key_and_more.py diff --git a/clickhouse_search/migrations/0015_remove_gnomadgenomessnvindel_key_and_more.py b/clickhouse_search/migrations/0015_remove_gnomadgenomessnvindel_key_and_more.py new file mode 100644 index 0000000000..1c82f6d549 --- /dev/null +++ b/clickhouse_search/migrations/0015_remove_gnomadgenomessnvindel_key_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.22 on 2025-08-15 19:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0014_gnomadgenomesgrch37snvindel_gnomadgenomessnvindel'), + ] + + operations = [ + migrations.RunSQL( + 'DROP DICTIONARY `GRCh37/SNV_INDEL/gnomad_genomes_dict`', + hints={'clickhouse': True}, + ), + migrations.RunSQL( + 'DROP DICTIONARY `GRCh38/SNV_INDEL/gnomad_genomes_dict`', + hints={'clickhouse': True}, + ), + migrations.DeleteModel( + name='GnomadGenomesGRCh37SnvIndel', + ), + migrations.DeleteModel( + name='GnomadGenomesSnvIndel', + ), + ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index d742852f73..c23171abcf 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -883,28 +883,6 @@ class GtStatsSv(models.ClickhouseModel): class Meta(BaseGtStats.Meta): db_table = 'GRCh38/SV/gt_stats' -class GnomadGenomesGRCh37SnvIndel(models.ClickhouseModel): - key = OneToOneField('AnnotationsGRCh37SnvIndel', db_column='key', primary_key=True, on_delete=CASCADE) - filter_af = models.DecimalField(max_digits=9, decimal_places=5) - - class Meta: - db_table = 'GRCh37/SNV_INDEL/gnomad_genomes' - engine = models.ReplacingMergeTree( - primary_key='key', - order_by='key' - ) - -class GnomadGenomesSnvIndel(models.ClickhouseModel): - key = OneToOneField('AnnotationsSnvIndel', db_column='key', primary_key=True, on_delete=CASCADE) - filter_af = models.DecimalField(max_digits=9, decimal_places=5) - - class Meta: - db_table = 'GRCh38/SNV_INDEL/gnomad_genomes' - engine = models.ReplacingMergeTree( - primary_key='key', - order_by='key' - ) - ENTRY_CLASS_MAP = { GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, From 8fe7fc39a7d9e5032c73693719858a81b9809702 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 16:01:29 -0400 Subject: [PATCH 048/109] fix conditional --- clickhouse_search/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 9745c86a7d..960795effe 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -811,7 +811,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) sample_ids = list(sample['sample_ids_by_type'].values()) - if affected == AFFECTED and inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only'): + if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')): affected_samples += sample_ids if inheritance_mode or individual_genotype_filter: for gt in self.genotype_lookup[genotype]: From 7c2489c82dbf9434a06334d2f9f6daa0ea880347 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 16:23:38 -0400 Subject: [PATCH 049/109] correctly set up fixture data --- .../fixtures/clickhouse_search.json | 134 ++++++++++++++++++ clickhouse_search/search_tests.py | 17 ++- clickhouse_search/test_utils.py | 8 +- 3 files changed, 149 insertions(+), 10 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 78e8cac1ff..265b3f7601 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -380,6 +380,80 @@ ["HG00731", 1, 58, 0.17241, 29] ] } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 51, + "fields": { + "key": 1, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "sample_type": "WES", + "xpos": 1000010439, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 0, 20, 0, 60], + ["HG00732", 0, 0, 0, 24], + ["HG00731", 1, 99, 0.5, 10] + ] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 52, + "fields": { + "key": 2, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "sample_type": "WES", + "xpos": 1038724419, + "is_gnomad_gt_5_percent": false, + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 0, 40, 0, 33], + ["HG00732", 1, 99, 0.625, 32], + ["HG00731", 2, 99, 1, 36] + ] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 53, + "fields": { + "key": 3, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "sample_type": "WES", + "xpos": 1091502721, + "is_gnomad_gt_5_percent": true, + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 1, 99, 0.40741, 27], + ["HG00732", 0, 99, 0.45946, 37], + ["HG00731", 1, 99, 1, 40] + ] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 54, + "fields": { + "key": 4, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "sample_type": "WES", + "xpos": 1091511686, + "is_gnomad_gt_5_percent": false, + "filters": [ + "VQSRTrancheSNP99.95to100.00" + ], + "sign": 1, + "calls": [ + ["HG00733", 0, 0, 0, 45], + ["HG00732", 0, 0, 0, 24], + ["HG00731", 1, 58, 0.17241, 29] + ] + } }, { "model": "clickhouse_search.entriessnvindel", "pk": 5, @@ -1164,6 +1238,66 @@ ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] ] } +}, { + "model": "clickhouse_search.entriesgcnv", + "pk": 48, + "fields": { + "key": 16, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "xpos": 14022417556, + "filters": [], + "sign": 1, + "calls": [ + ["HG00731", 1, 3, 38, false, 22438910, 22469796, 0, [], false, true, false], + ["HG00733", null, null, null, null, null, null, null, [], null, null, null] + ] + } +}, { + "model": "clickhouse_search.entriesgcnv", + "pk": 49, + "fields": { + "key": 17, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "xpos": 16029802672, + "filters": [], + "sign": 1, + "calls": [ + ["HG00731", 1, 3, 29, false, 29809156, 29815990, 8, ["ENSG00000103495", "ENSG00000167371", "ENSG00000280893"], false, true, false], + ["HG00733", 1, 3, 37, false, 29809156, 29815990, 8, ["ENSG00000103495", "ENSG00000167371", "ENSG00000280893"], false, true, false] + ] + } +}, { + "model": "clickhouse_search.entriesgcnv", + "pk": 40, + "fields": { + "key": 18, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "xpos": 17038717327, + "filters": [], + "sign": 1, + "calls": [ + ["HG00731", 2, 4, 13, true, 38717327, 38719636, 3, ["ENSG00000275023"], true, false, false], + ["HG00733", null, null, null, null, null, null, null, [], null, null, null] + ] + } +}, { + "model": "clickhouse_search.entriesgcnv", + "pk": 41, + "fields": { + "key": 19, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2_x", + "xpos": 17038721781, + "filters": [], + "sign": 1, + "calls": [ + ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false], + ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false] + ] + } }, { "model": "clickhouse_search.entriesgcnv", "pk": 32, diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 59bbdf4fe9..1a46fa510c 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -15,7 +15,7 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant -from clickhouse_search.models import EntriesGcnv +from clickhouse_search.models import EntriesSnvIndel from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper @@ -1094,14 +1094,19 @@ def test_secondary_annotations_filter(self): # Search works with a different number of samples within the family self._reset_search_families() missing_gt_gcnv_variant = { - **GCNV_VARIANT4, 'genotypes': {k: v for k, v in GCNV_VARIANT4['genotypes'].items() if k != 'I000005_hg00732'} + **GCNV_VARIANT4, + 'familyGuids': ['F000002_2_x'], + 'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in GCNV_VARIANT4['genotypes'].items() if k != 'I000005_hg00732'} + } + missing_gt_comp_het_variant = { + **MULTI_DATA_TYPE_COMP_HET_VARIANT2, + 'familyGuids': ['F000002_2_x'], + 'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in MULTI_DATA_TYPE_COMP_HET_VARIANT2['genotypes'].items()} } Sample.objects.filter(guid='S000146_hg00732').update(is_active=False) - for entry in EntriesGcnv.objects.filter(family_guid='F000002_2'): - entry.calls = [list(call.values()) for call in [entry.calls[0], entry.calls[2]]] - entry.save(using='clickhouse_write') + Family.objects.filter(guid='F000002_2').update(guid='F000002_2_x') self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, missing_gt_gcnv_variant]], + [[missing_gt_comp_het_variant, missing_gt_gcnv_variant]], inheritance_mode='compound_het', pathogenicity=pathogenicity, locus=None, annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, cached_variant_fields=[[ {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': None}, diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 3313ef981b..1dfecffb28 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -20,10 +20,10 @@ GCNV_VARIANT4 as HAIL_GCNV_VARIANT4, ) -VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 8, 'hom': 3}}} -VARIANT2 = {**deepcopy(HAIL_VARIANT2), 'key': 2, 'populations': {**deepcopy(HAIL_VARIANT2)['populations'], 'seqr': {'ac': 7, 'hom': 2}}} -VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 5, 'hom': 0}}} -VARIANT4 = {**deepcopy(HAIL_VARIANT4), 'key': 4, 'populations': {**deepcopy(HAIL_VARIANT4)['populations'], 'seqr': {'ac': 4, 'hom': 1}}} +VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 9, 'hom': 3}}} +VARIANT2 = {**deepcopy(HAIL_VARIANT2), 'key': 2, 'populations': {**deepcopy(HAIL_VARIANT2)['populations'], 'seqr': {'ac': 10, 'hom': 3}}} +VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 7, 'hom': 0}}} +VARIANT4 = {**deepcopy(HAIL_VARIANT4), 'key': 4, 'populations': {**deepcopy(HAIL_VARIANT4)['populations'], 'seqr': {'ac': 5, 'hom': 1}}} PROJECT_2_VARIANT = {**deepcopy(HAIL_PROJECT_2_VARIANT), 'key': 5, 'populations': {**deepcopy(HAIL_PROJECT_2_VARIANT)['populations'], 'seqr': {'ac': 2, 'hom': 0}}} MITO_VARIANT1 = {**deepcopy(HAIL_MITO_VARIANT1), 'key': 6, 'populations': {**deepcopy(HAIL_MITO_VARIANT1)['populations'], 'seqr': {'ac': 0}, 'seqr_heteroplasmy': {'ac': 1}}} MITO_VARIANT2 = {**deepcopy(HAIL_MITO_VARIANT2), 'key': 7, 'populations': {**deepcopy(HAIL_MITO_VARIANT2)['populations'], 'seqr': {'ac': 0}, 'seqr_heteroplasmy': {'ac': 1}}} From 801973452872aea4a6ad9a4db705539b07e4974b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 15 Aug 2025 16:37:52 -0400 Subject: [PATCH 050/109] update tests for new fixture data --- .../fixtures/clickhouse_search.json | 18 ------------------ clickhouse_search/search_tests.py | 7 +++---- clickhouse_search/test_utils.py | 2 +- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 265b3f7601..5d3e32f4ae 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -380,24 +380,6 @@ ["HG00731", 1, 58, 0.17241, 29] ] } -}, { - "model": "clickhouse_search.entriessnvindel", - "pk": 51, - "fields": { - "key": 1, - "project_guid": "R0001_1kg", - "family_guid": "F000002_2_x", - "sample_type": "WES", - "xpos": 1000010439, - "is_gnomad_gt_5_percent": false, - "filters": [], - "sign": 1, - "calls": [ - ["HG00733", 0, 20, 0, 60], - ["HG00732", 0, 0, 0, 24], - ["HG00731", 1, 99, 0.5, 10] - ] - } }, { "model": "clickhouse_search.entriessnvindel", "pk": 52, diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 1a46fa510c..ce2625c6c3 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -15,7 +15,6 @@ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant -from clickhouse_search.models import EntriesSnvIndel from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper @@ -690,7 +689,7 @@ def test_frequency_filter(self): self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - freqs={'callset': {'ac': 6}, **sv_callset_filter}, + freqs={'callset': {'ac': 7}, **sv_callset_filter}, ) self._assert_expected_search( @@ -698,7 +697,7 @@ def test_frequency_filter(self): ) self._assert_expected_search( - [MULTI_FAMILY_VARIANT, GCNV_VARIANT3, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], freqs={'callset': {'ac': 6, 'hh': 0}, 'sv_callset': {'ac': 50}}, + [MULTI_FAMILY_VARIANT, GCNV_VARIANT3, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], freqs={'callset': {'ac': 7, 'hh': 0}, 'sv_callset': {'ac': 50}}, ) self._set_sv_family_search() @@ -1191,7 +1190,7 @@ def test_sort(self): ) self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, VARIANT4, MULTI_FAMILY_VARIANT, VARIANT2, VARIANT1, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1], + [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, VARIANT4, MULTI_FAMILY_VARIANT, VARIANT1, VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1], sort='callset_af', ) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 1dfecffb28..0395005abf 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -20,7 +20,7 @@ GCNV_VARIANT4 as HAIL_GCNV_VARIANT4, ) -VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 9, 'hom': 3}}} +VARIANT1 = {**deepcopy(HAIL_VARIANT1), 'key': 1, 'populations': {**deepcopy(HAIL_VARIANT1)['populations'], 'seqr': {'ac': 8, 'hom': 3}}} VARIANT2 = {**deepcopy(HAIL_VARIANT2), 'key': 2, 'populations': {**deepcopy(HAIL_VARIANT2)['populations'], 'seqr': {'ac': 10, 'hom': 3}}} VARIANT3 = {**deepcopy(HAIL_VARIANT3), 'key': 3, 'populations': {**deepcopy(HAIL_VARIANT3)['populations'], 'seqr': {'ac': 7, 'hom': 0}}} VARIANT4 = {**deepcopy(HAIL_VARIANT4), 'key': 4, 'populations': {**deepcopy(HAIL_VARIANT4)['populations'], 'seqr': {'ac': 5, 'hom': 1}}} From 01ccb453a9ec922f126f508a95f7287cfcabab0e Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 18 Aug 2025 10:13:21 -0400 Subject: [PATCH 051/109] implement new fields --- clickhouse_search/backend/fields.py | 28 ++++++++ .../fixtures/clickhouse_saved_variants.json | 6 ++ .../fixtures/clickhouse_search.json | 66 +++++++++++++++++++ clickhouse_search/migrations/0001_initial.py | 6 +- ..._annotationsdiskgrch37snvindel_and_more.py | 6 +- clickhouse_search/models.py | 8 ++- 6 files changed, 113 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/backend/fields.py b/clickhouse_search/backend/fields.py index a43519218c..d729c4f3cf 100644 --- a/clickhouse_search/backend/fields.py +++ b/clickhouse_search/backend/fields.py @@ -93,6 +93,34 @@ class UInt64FieldDeltaCodecField(models.UInt64Field): def db_type(self, connection): return f'{super().db_type(connection)} CODEC(Delta(8), ZSTD(1))' +class BitmapField(models.ArrayField): + + def __init__(self, *args, null_when_empty=False, **kwargs): + self.null_when_empty = null_when_empty + super().__init__(*args, **kwargs) + + def clone(self): + clone = super().clone() + clone.null_when_empty = self.null_when_empty + return clone + + def get_internal_type(self): + return "BitmapField" + + @property + def description(self): + return "Bitmap of %s" % self.base_field.description + + def db_type(self, connection): + return "AggregateFunction(groupBitmap, %s)" % self.base_field.db_type(connection) + + def cast_db_type(self, connection): + return "bitmapBuild(%s)" % self.base_field.cast_db_type(connection) + + def from_db_value(self, value, expression, connection): + if self.null_when_empty and not value: + return None + return super().from_db_value(value, expression, connection) class NamedTupleField(models.TupleField): diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 8ad858a65f..6a7e7706e7 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -66,6 +66,8 @@ "sample_type": "WGS", "xpos": 1248367227, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [6, 48], "filters": [], "sign": 1, "calls": [ @@ -118,6 +120,8 @@ "sample_type": "WES", "xpos": 2103343353, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [48, 45], "filters": [], "sign": 1, "calls": [ @@ -196,6 +200,8 @@ "sample_type": "WGS", "xpos": 1248367227, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index c2f05942bc..719977bf48 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -274,6 +274,8 @@ "sample_type": "WES", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -292,6 +294,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -310,6 +314,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -328,6 +334,8 @@ "sample_type": "WES", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], @@ -348,6 +356,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -365,6 +375,8 @@ "sample_type": "WES", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -381,6 +393,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -397,6 +411,8 @@ "sample_type": "WES", "xpos": 1000010146, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -413,6 +429,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -429,6 +447,8 @@ "sample_type": "WGS", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -445,6 +465,8 @@ "sample_type": "WGS", "xpos": 1000010146, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -461,6 +483,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -479,6 +503,8 @@ "sample_type": "WGS", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -497,6 +523,8 @@ "sample_type": "WGS", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -515,6 +543,8 @@ "sample_type": "WGS", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], @@ -535,6 +565,8 @@ "sample_type": "WGS", "xpos": 1009310123, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [71], "filters": [], "sign": 1, "calls": [ @@ -581,6 +613,8 @@ "sample_type": "WES", "xpos": 7143270172, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [73, 74], "filters": [ "VQSRTrancheSNP99.90to99.95" ], @@ -600,6 +634,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -1339,5 +1375,35 @@ "gencode_gene_type": "protein_coding", "gencode_release": 27 } +}, +{ + "model": "reference_data.geneinfo", + "pk": 73, + "fields": { + "gene_id": "ENSG00000271079", + "gene_symbol": "CTAGE15", + "chrom_grch38": "7", + "start_grch38": 143571801, + "end_grch38": 143574387, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } +}, +{ + "model": "reference_data.geneinfo", + "pk": 74, + "fields": { + "gene_id": "ENSG00000176227", + "gene_symbol": "CTAGE6", + "chrom_grch38": "7", + "start_grch38": 91500851, + "end_grch38": 91525764, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } } ] \ No newline at end of file diff --git a/clickhouse_search/migrations/0001_initial.py b/clickhouse_search/migrations/0001_initial.py index 05d5815bc7..c308cca9cd 100644 --- a/clickhouse_search/migrations/0001_initial.py +++ b/clickhouse_search/migrations/0001_initial.py @@ -89,14 +89,16 @@ class Migration(migrations.Migration): ('sample_type', clickhouse_backend.models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')])), ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), + ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), + ('geneId_ids', clickhouse_search.backend.fields.BitmapField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), ], options={ 'db_table': 'GRCh38/SNV_INDEL/entries', - 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), partition_by='project_guid'), - 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent'), + 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'), + 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py index ef42cbe289..aec53d173d 100644 --- a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py +++ b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py @@ -82,6 +82,8 @@ class Migration(migrations.Migration): ('sample_type', clickhouse_backend.models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')])), ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), + ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), + ('geneId_ids', clickhouse_search.backend.fields.BitmapField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), @@ -89,8 +91,8 @@ class Migration(migrations.Migration): options={ 'db_table': 'GRCh37/SNV_INDEL/entries', 'abstract': False, - 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), partition_by='project_guid'), - 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent'), + 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'), + 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index c23171abcf..054cc117ee 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -4,7 +4,7 @@ from django.db.models import options, ForeignKey, OneToOneField, Func, CASCADE, PROTECT from clickhouse_search.backend.engines import CollapsingMergeTree, EmbeddedRocksDB, Join -from clickhouse_search.backend.fields import Enum8Field, NestedField, UInt32FieldDeltaCodecField, UInt64FieldDeltaCodecField, NamedTupleField +from clickhouse_search.backend.fields import BitmapField, Enum8Field, NestedField, UInt32FieldDeltaCodecField, UInt64FieldDeltaCodecField, NamedTupleField from clickhouse_search.backend.functions import ArrayDistinct, ArrayFlatten, ArrayMin, ArrayMax from clickhouse_search.managers import EntriesManager, AnnotationsQuerySet from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 @@ -588,18 +588,20 @@ class BaseEntriesSnvIndel(BaseEntries): sample_type = models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')]) is_gnomad_gt_5_percent = models.BoolField() + is_annotated_in_any_gene = models.BoolField() + geneId_ids = BitmapField(models.UInt32Field(), null_when_empty=True) calls = models.ArrayField(NamedTupleField(CALL_FIELDS)) class Meta: abstract = True engine = CollapsingMergeTree( 'sign', - order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), + order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, ) - projection = Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent') + projection = Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos') class EntriesGRCh37SnvIndel(BaseEntriesSnvIndel): From a88dd00005d2701f33adc08903ae6aacd0e45bf5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 11:03:00 -0400 Subject: [PATCH 052/109] bacth bulk updates for saved variant models --- .../commands/set_saved_variant_key.py | 20 ++++++++++++++----- .../tests/set_saved_variant_key_tests.py | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index bc3ea807d2..2a2c5649a8 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) +BATCH_SIZE = 10000 GCNV_CALLSET_PATH = 'gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz' SV_ID_UPDATE_MAP = { @@ -2264,16 +2265,25 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ saved_variants = SavedVariant.objects.filter( family__project__genome_version=genome_version, variant_id__in=mapped_variant_ids, ) + update_fields = ['key', 'dataset_type'] + if variant_id_updates: + update_fields.append('variant_id') + update_variants = [] + total_num_updated = 0 for variant in saved_variants: if variant_id_updates: variant.variant_id = variant_id_updates[variant.variant_id] variant.key = variant_key_map[variant.variant_id] variant.dataset_type = dataset_type - update_fields = ['key', 'dataset_type'] - if variant_id_updates: - update_fields.append('variant_id') - num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields, batch_size=10000) - logger.info(f'Updated keys for {num_updated} {dataset_type} (GRCh{genome_version}) variants') + update_variants.append(variant) + if len(update_variants) >= BATCH_SIZE: + num_updated = SavedVariant.objects.bulk_update(update_variants, update_fields) + logger.info(f'Updated batch of {num_updated}') + total_num_updated += num_updated + update_variants = [] + if update_variants: + total_num_updated += SavedVariant.objects.bulk_update(update_variants, update_fields) + logger.info(f'Updated keys for {total_num_updated} {dataset_type} (GRCh{genome_version}) variants') no_key = set(variants_ids) - set(variant_key_map.keys()) if no_key: diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 56029887ee..e8f1d7a79d 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -22,6 +22,7 @@ def setUpTestData(cls): Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) SavedVariant.objects.update(key=None) + @mock.patch('clickhouse_search.management.commands.set_saved_variant_key.BATCH_SIZE', 2) @mock.patch('seqr.utils.file_utils.subprocess.Popen') def test_command(self, mock_subprocess): mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA @@ -34,6 +35,7 @@ def test_command(self, mock_subprocess): ('Updated keys for 1 MITO (GRCh38) variants', None), ('Finding keys for 1 SNV_INDEL (GRCh38) variant ids', None), ('Found 1 keys', None), + ('Updated batch of 2', None), ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None), ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), ('Found 0 keys', None), From bbd4c0f365f2ad85bca6494bb186f769475c32c0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 11:41:40 -0400 Subject: [PATCH 053/109] do not allow moving families still associated with analysis groups --- .../commands/transfer_families_to_different_project.py | 7 ++++++- .../transfer_families_to_different_project_tests.py | 10 ++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 95ea20879d..e5eb13b838 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -49,9 +49,14 @@ def handle(self, *args, **options): num_found = len(families) num_expected = len(set(family_ids)) - missing_id_message = '' if num_found == num_expected else f' No match for: {", ".join(set(family_ids) - set([f.family_id for f in families]))}.' + found_families = set([f.family_id for f in families]) + missing_id_message = '' if num_found == num_expected else f' No match for: {", ".join(set(family_ids) - found_families)}.' logger.info(f'Found {num_found} out of {num_expected} families.{missing_id_message}') + families = families.filter(analysisgroup__isnull=True) + if len(families) < num_found: + logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(found_families - set([f.family_id for f in families]))}') + backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index d2c6c78f2c..88ea77a140 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -10,11 +10,13 @@ class TransferFamiliesTest(object): def _test_command(self, additional_family, logs): call_command( - 'transfer_families_to_different_project', '--from-project=R0001_1kg', '--to-project=R0003_test', additional_family, '2', + 'transfer_families_to_different_project', '--from-project=R0001_1kg', '--to-project=R0003_test', additional_family, '2', '5', ) self.assert_json_logs(user=None, expected=[ - *logs, + logs[0], + ('Skipping 1 families with analysis groups in the project: 5', None), + *logs[1:], ('Updating "Excluded" tags', None), ('Updating families', None), ('Done.', None), @@ -43,7 +45,7 @@ class TransferFamiliesLocalTest(TransferFamiliesTest, AuthenticationTestCase): def test_es_command(self): self._test_command( - additional_family='12', logs=[('Found 1 out of 2 families. No match for: 12.', None)] + additional_family='12', logs=[('Found 2 out of 3 families. No match for: 12.', None)] ) @@ -90,7 +92,7 @@ def _assert_update_check_airflow_calls(self, call_count, offset, update_check_pa @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', '') def test_hail_backend_command(self): searchable_family = self._test_command(additional_family='4', logs=[ - ('Found 2 out of 2 families.', None), + ('Found 3 out of 3 families.', None), ('Disabled search for 7 samples in the following 1 families: 2', None), ('Successfully triggered DELETE_FAMILIES DAG for 1 MITO families', None), ('Successfully triggered DELETE_FAMILIES DAG for 1 SNV_INDEL families', None), From edb24e5e625957d79cc54715b5329f8f9a616118 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 11:48:00 -0400 Subject: [PATCH 054/109] include group names in error message --- .../transfer_families_to_different_project.py | 11 ++++++++--- .../transfer_families_to_different_project_tests.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index e5eb13b838..80bd0fdb2a 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -49,13 +49,18 @@ def handle(self, *args, **options): num_found = len(families) num_expected = len(set(family_ids)) - found_families = set([f.family_id for f in families]) - missing_id_message = '' if num_found == num_expected else f' No match for: {", ".join(set(family_ids) - found_families)}.' + missing_id_message = '' if num_found == num_expected else f' No match for: {", ".join(set(family_ids) - set([f.family_id for f in families]))}.' logger.info(f'Found {num_found} out of {num_expected} families.{missing_id_message}') + found_families = families families = families.filter(analysisgroup__isnull=True) if len(families) < num_found: - logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(found_families - set([f.family_id for f in families]))}') + update_family_ids = set([f.family_id for f in families]) + group_families = [ + f'{f.family_id} ({", ".join(f.analysisgroup_set.values_list("name", flat=True))})' + for f in found_families if f.family_id not in update_family_ids + ] + logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 88ea77a140..634cc6b6f7 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -15,7 +15,7 @@ def _test_command(self, additional_family, logs): self.assert_json_logs(user=None, expected=[ logs[0], - ('Skipping 1 families with analysis groups in the project: 5', None), + ('Skipping 1 families with analysis groups in the project: 5 (Test Group 1)', None), *logs[1:], ('Updating "Excluded" tags', None), ('Updating families', None), From 4d94dc05b799318e9608332087bd432aa1648c32 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 12:52:06 -0400 Subject: [PATCH 055/109] first pass restructure sample data --- clickhouse_search/managers.py | 74 ++++++++++--------------- clickhouse_search/search.py | 100 ++++++++++++++++++---------------- 2 files changed, 81 insertions(+), 93 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 003a51ca55..a849416473 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -731,17 +731,19 @@ def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): - project_guids = {s['project_guid'] for s in sample_data} - project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=sample_data[0]['project_guid']) + project_guids = sample_data['project_guids'] + project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=project_guids[0]) entries = entries.filter(project_filter) - sample_type_families, multi_sample_type_families = self._get_family_sample_types(sample_data) + multi_sample_type_families = sample_data['sample_type_families'].get('multi', []) family_q = None multi_sample_type_family_q = None if multi_sample_type_families: family_q = Q(family_guid__in=multi_sample_type_families) multi_sample_type_family_q = family_q - for sample_type, families in sample_type_families.items(): + for sample_type, families in sample_data['sample_type_families'].items(): + if sample_type == 'multi': + continue sample_family_q = Q(family_guid__in=families) if not self.single_sample_type: sample_family_q &= Q(sample_type=sample_type) @@ -799,20 +801,19 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i samples_by_gt = defaultdict(list) affected_samples = [] family_missing_type_samples = defaultdict(lambda: defaultdict(list)) - for s in sample_data: - for sample in s['samples']: - affected = custom_affected.get(sample['individual_guid']) or sample['affected'] - genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) - sample_ids = list(sample['sample_ids_by_type'].values()) - if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')): - affected_samples += sample_ids - if inheritance_mode or individual_genotype_filter: - for gt in self.genotype_lookup[genotype]: - samples_by_gt[gt] += sample_ids - if s['family_guid'] in multi_sample_type_families and len(sample_ids) == 1: - sample_type = next(iter(sample['sample_ids_by_type'])) + for sample in sample_data['samples']: + affected = custom_affected.get(sample['individual_guid']) or sample['affected'] + genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) + if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')): + affected_samples.append(sample['sample_id']) + if (inheritance_mode and inheritance_mode != ANY_AFFECTED) or individual_genotype_filter: + for gt in self.genotype_lookup[genotype]: + samples_by_gt[gt].append(sample['sample_id']) + if sample['family_guid'] in multi_sample_type_families: + sample_type = sample['sample_type'] missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS - family_missing_type_samples[s['family_guid']][missing_type].append(sample_ids[0]) + if not any(s for s in sample_data['samples'] if s['individual_guid'] == sample['individual_guid'] and s['sample_type'] == missing_type): + family_missing_type_samples[sample['family_guid']][missing_type].append(sample['sample_id']) inheritance_q = None gt_filter = None @@ -830,17 +831,6 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i return inheritance_q, quality_q, gt_filter, family_missing_type_samples - @staticmethod - def _get_family_sample_types(sample_data): - sample_type_families = defaultdict(list) - multi_sample_type_families = set() - for s in sample_data: - if len(s['sample_types']) == 1: - sample_type_families[s['sample_types'][0]].append(s['family_guid']) - else: - multi_sample_type_families.add(s['family_guid']) - return sample_type_families, multi_sample_type_families - def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None if individual_genotype_filter: @@ -932,7 +922,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an fields.append('seqrPop') if self._has_clinvar(): fields += ['clinvar', 'clinvar_key'] - if multi_sample_type_families or sample_data is None or len(sample_data) > 1: + if multi_sample_type_families or sample_data is None or len(set(sample_data['family_guids'])) > 1: genotype_sample_data = None if skip_individual_guid else sample_data entries = entries.values(*fields).annotate( familyGuids=ArraySort(ArrayDistinct(GroupArray('family_guid'))), @@ -979,14 +969,12 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an return entries def genotype_expression(self, sample_data=None): - sample_map = [] - for data in sample_data or []: - family_samples = [] - for s in data['samples']: - family_samples += [ - f"'{sample_id}', '{s['individual_guid']}'" for sample_id in set(s['sample_ids_by_type'].values()) - ] - sample_map.append(f"'{data['family_guid']}', map({', '.join(family_samples)})") + family_samples = defaultdict(list) + for s in (sample_data or {}).get('samples', []): + family_samples[s['family_guid']].append(f"'{s['sample_id']}', '{s['individual_guid']}'") + sample_map = [ + f"'{family_guid}', map({', '.join(samples)})" for family_guid, samples in family_samples.items() + ] genotype_expressions = list(self.genotype_fields.keys()) output_base_fields = list(self.genotype_fields.values()) output_field_kwargs = {'group_by_key': 'familyGuid'} @@ -1004,14 +992,10 @@ def genotype_expression(self, sample_data=None): ) def _carriers_expression(self, sample_data): - family_carriers = {} - for family_sample_data in sample_data: - family_carriers[family_sample_data['family_guid']] = set() - for s in family_sample_data['samples']: - if s['affected'] == UNAFFECTED: - family_carriers[family_sample_data['family_guid']].update([ - f"'{sample_id}'" for sample_id in s['sample_ids_by_type'].values() - ]) + family_carriers = defaultdict(set) + for s in sample_data['samples']: + if s['affected'] == UNAFFECTED: + family_carriers[s['family_guid']].add(f"'{s['sample_id']}'") if not any(family_carriers.values()): return None diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 0a44053529..b390ea46db 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -44,8 +44,8 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] - family_guid = sample_data[0]['family_guid'] - is_multi_project = any(s['project_guid'] != sample_data[0]['project_guid'] for s in sample_data) + family_guid = sample_data['family_guids'][0] + is_multi_project = len(sample_data['project_guids']) > 1 skip_individual_guid = is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS dataset_results = [] @@ -88,36 +88,45 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - snv_indel_families = {s['family_guid'] for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS]} + snv_indel_sample_data = sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + snv_indel_families = set(snv_indel_sample_data['family_guids']) + skip_individual_guid = len(snv_indel_sample_data['project_guids']) > 1 results = [] for sample_type in [Sample.SAMPLE_TYPE_WES, Sample.SAMPLE_TYPE_WGS]: sv_dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - sv_families = {s['family_guid'] for s in sample_data_by_dataset_type.get(sv_dataset_type, [])} + sv_sample_data = sample_data_by_dataset_type.get(sv_dataset_type, {}) + sv_families = set(sv_sample_data.get('family_guids', [])) families = snv_indel_families.intersection(sv_families) if not families: continue - sample_data = [ - s for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] if s['family_guid'] in families - ] - skip_individual_guid = any(s['project_guid'] != sample_data[0]['project_guid'] for s in sample_data) + snv_indel_sample_data = { + **snv_indel_sample_data, + 'family_guids': list(families), + 'samples': [s for s in snv_indel_sample_data['samples'] if s['family_guid'] in families] + } - entries = entry_cls.objects.search([ - s for s in sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] if s['family_guid'] in families - ], skip_individual_guid=skip_individual_guid, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True) + entries = entry_cls.objects.search( + snv_indel_sample_data, skip_individual_guid=skip_individual_guid, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True, + ) snv_indel_q = annotations_cls.objects.subquery_join(entries).search(**search_kwargs, annotations=annotations) - sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search([ - s for s in sample_data_by_dataset_type[sv_dataset_type] if s['family_guid'] in families - ], **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET, annotate_carriers=True) + sv_sample_data = { + **sv_sample_data, + 'family_guids': list(families), + 'samples': [s for s in sv_sample_data['samples'] if s['family_guid'] in families] + } + sv_entries = ENTRY_CLASS_MAP[genome_version][sv_dataset_type].objects.search( + sv_sample_data, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET, annotate_carriers=True, + ) sv_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][sv_dataset_type] sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations) result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) dataset_results = [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] if skip_individual_guid: - _add_individual_guids(dataset_results, sample_data, single_data_type=False) + _add_individual_guids(dataset_results, sv_sample_data, single_data_type=False) results += dataset_results return results @@ -133,7 +142,7 @@ def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_ annotations=annotations_secondary or annotations, **search_kwargs, ) - return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(sample_data)) + return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(set(sample_data['family_guids']))) def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_families): @@ -209,12 +218,7 @@ def _result_as_tuple(results, field_prefix): def _add_individual_guids(results, sample_data, single_data_type=True): - sample_map = {} - for data in sample_data: - for s in data['samples']: - sample_map.update({ - (data['family_guid'], sample_id): s['individual_guid'] for sample_id in set(s['sample_ids_by_type'].values()) - }) + sample_map = {(s['family_guid'], s['sample_id']): s['individual_guid'] for s in sample_data['samples']} for result in results: if isinstance(result, list): for variant in result: @@ -318,37 +322,37 @@ def _get_sample_data(samples): ) sample_data = samples.values( - 'dataset_type', family_guid=F('individual__family__guid'), project_guid=F('individual__family__project__guid'), + 'dataset_type', 'sample_type', ).annotate( - samples=ArrayAgg(JSONObject(affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', individual_guid=F('individual__guid'))), - sample_types=ArrayAgg('sample_type', distinct=True), + project_guids=ArrayAgg('individual__family__project__name', distinct=True), + family_guids=ArrayAgg('individual__family__project__name', distinct=True), + samples=ArrayAgg(JSONObject(affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'))), ) - samples_by_dataset_type = defaultdict(list) + samples_by_dataset_type = {} for data in sample_data: - samples = _group_by_sample_type(data['samples']) - if data['dataset_type'] == Sample.DATASET_TYPE_SV_CALLS: - samples_by_type = defaultdict(list) - for sample in samples: - for sample_type in sample['sample_ids_by_type']: - samples_by_type[sample_type].append( - {**sample, 'sample_ids_by_type': {sample_type: sample['sample_ids_by_type'][sample_type]}} - ) - for sample_type, type_samples in samples_by_type.items(): - samples_by_dataset_type[f"{data['dataset_type']}_{sample_type}"].append({**data, 'samples': type_samples, 'sample_types': [sample_type]}) + dataset_type = data.pop('dataset_type') + sample_type = data.pop('sample_type') + if dataset_type == Sample.DATASET_TYPE_SV_CALLS: + dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + + if dataset_type in samples_by_dataset_type: + other_type_data = samples_by_dataset_type[dataset_type] + other_sample_type = next(iter(other_type_data['sample_type_families'].keys())) + family_guids = set(data['family_guids']) + other_type_family_guids = set(other_type_data['family_guids']) + sample_type_families = { + other_sample_type: other_type_family_guids - family_guids, + sample_type: family_guids - other_type_family_guids, + 'multi': family_guids.intersection(other_type_family_guids), + } + data['sample_type_families'] = {k: v for k, v in sample_type_families.items() if v} + for key in ['project_guids', 'family_guids', 'samples']: + data[key] += other_type_data[key] else: - samples_by_dataset_type[data['dataset_type']].append({**data, 'samples': samples}) - return samples_by_dataset_type + data['sample_type_families'] = {sample_type: set(data['family_guids'])} - -def _group_by_sample_type(samples): - samples_by_individual_type = {} - for sample in samples: - sample_type = sample.pop('sample_type') - sample_id = sample.pop('sample_id') - if sample['individual_guid'] not in samples_by_individual_type: - samples_by_individual_type[sample['individual_guid']] = {'sample_ids_by_type': {}, **sample} - samples_by_individual_type[sample['individual_guid']]['sample_ids_by_type'][sample_type] = sample_id - return list(samples_by_individual_type.values()) + samples_by_dataset_type[dataset_type] = data + return samples_by_dataset_type OMIM_SORT = 'in_omim' From 612e15e8853a3fecd3aba366c4ba33d995fb968a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 13:37:15 -0400 Subject: [PATCH 056/109] fix sample data agg --- clickhouse_search/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index b390ea46db..1f04b63aec 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -324,8 +324,8 @@ def _get_sample_data(samples): sample_data = samples.values( 'dataset_type', 'sample_type', ).annotate( - project_guids=ArrayAgg('individual__family__project__name', distinct=True), - family_guids=ArrayAgg('individual__family__project__name', distinct=True), + project_guids=ArrayAgg('individual__family__project__guid', distinct=True), + family_guids=ArrayAgg('individual__family__guid', distinct=True), samples=ArrayAgg(JSONObject(affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'))), ) samples_by_dataset_type = {} From eee2be3afdbf82bd80ac0d2be300d394082fabbc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 14:44:25 -0400 Subject: [PATCH 057/109] remove underused helper fnc --- clickhouse_search/managers.py | 31 ++++++++++++++----------------- clickhouse_search/search.py | 3 +++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index a849416473..1d57313751 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -159,6 +159,7 @@ def subquery_join(self, subquery, join_key='key'): # Add key to intermediate select if not already present join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) if join_key not in subquery.query.values_select: + # TODO unused subquery.query.values_select = tuple([join_key, *subquery.query.values_select]) subquery.query.select = tuple([Col(subquery.model._meta.db_table, join_field), *subquery.query.select]) @@ -371,6 +372,7 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): if pop_filter.get('hh') is not None: for subfield in ['hom', 'hemi']: if subfield not in pop_subfields: + # TODO continue hh_q = Q(**{ f'populations__{population}__{subfield}__lte': pop_filter['hh'], @@ -475,6 +477,7 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl def _interval_query(self, chrom, start, end): q = Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) if hasattr(self.model, 'endChrom'): + # TODO q |= Q(endChrom__isnull=True, chrom=chrom, end__range=(start, end)) q |= Q(endChrom=chrom, end__range=(start, end)) elif hasattr(self.model, 'end'): @@ -522,6 +525,7 @@ def _parse_annotation_filters(self, annotations, pathogenicity): for field, value in (pathogenicity or {}).items(): if not value: + # TODO continue elif field == HGMD_KEY: filters_by_field[HGMD_KEY] = self._hgmd_filter(value) @@ -570,6 +574,7 @@ def _hgmd_filter(hgmd): if min_class == max_class: return ('{field}__classification', min_class) elif min_class and max_class: + # TODO return ('{field}__classification__range', (min_class, max_class)) return ('{field}__classification__gt', min_class) @@ -779,6 +784,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q |= multi_sample_type_family_q entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples) elif inheritance_q is not None: + # TODO entries = entries.annotate(passes_inheritance=inheritance_q) inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q else: @@ -787,6 +793,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if quality_q is None: entries = entries.annotate(passes_quality=Value(True)) else: + # TODO entries = entries.annotate(passes_quality=quality_q) quality_q = Q(passes_quality=True) | multi_sample_type_family_q @@ -869,21 +876,6 @@ def _quality_q(self, quality_filter, affected_samples, clinvar_override_q): return quality_q - def _get_family_sample_map(self, family_samples, is_nested_map=False): - family_map = [] - for family_guid, sample_types in family_samples.items(): - if self.single_sample_type: - samples = list(next(iter(sample_types.values()))) - else: - if is_nested_map: - sample_types = {sample_type: f"map({', '.join(samples)})" for sample_type, samples in sample_types.items()} - samples = [f"'{sample_type}', {samples}" for sample_type, samples in sample_types.items()] - if is_nested_map or not self.single_sample_type: - samples = f"map({', '.join(samples)})" - family_map.append(f"'{family_guid}', {samples}") - - return ', '.join(family_map) - def _annotate_failed_family_samples(self, entries, gt_filter, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ @@ -892,10 +884,13 @@ def _annotate_failed_family_samples(self, entries, gt_filter, family_missing_typ mapped_expression='tuple(family_guid, x.sampleId)', )) if family_missing_type_samples: - missing_sample_map = self._get_family_sample_map(family_missing_type_samples) + missing_sample_map = [] + for family_guid, sample_types in family_missing_type_samples.items(): + samples = [f"'{sample_type}', {samples}" for sample_type, samples in sample_types.items()] + missing_sample_map.append(f"'{family_guid}', map({', '.join(samples)})") entries = entries.annotate( missing_family_samples=ArrayMap( - MapLookup('family_guid', Cast('sample_type', models.StringField()), map_values=missing_sample_map), + MapLookup('family_guid', Cast('sample_type', models.StringField()), map_values=', '.join(missing_sample_map)), mapped_expression='tuple(family_guid, x)', ) ) @@ -942,6 +937,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an family_carriers = Cast(Tuple('familyGuids', GroupArray('carriers')), map_field) entries = entries.annotate(family_carriers=family_carriers) if annotate_hom_alts: + # TODO entries = entries.annotate(no_hom_alt_families=ArrayMap( ArrayFilter(GroupArray(Tuple('family_guid', 'has_hom_alt')), conditions=[{2: (None, 'NOT {field}')}]), mapped_expression='x.1', @@ -1060,6 +1056,7 @@ def filter_intervals(self, exclude_intervals=False, intervals=None, gene_interva return entries if padded_interval: + # TODO pos = padded_interval['start'] padding = int((padded_interval['end'] - pos) * padded_interval['padding']) intervals = [(padded_interval['chrom'], max(pos - padding, MIN_POS), min(pos + padding, MAX_POS))] diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1f04b63aec..47b6fc35e4 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -126,6 +126,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) dataset_results = [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] if skip_individual_guid: + # TODO _add_individual_guids(dataset_results, sv_sample_data, single_data_type=False) results += dataset_results @@ -164,6 +165,7 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ if results.has_annotation('primary_has_hom_alt'): results = results.filter(is_overlapped_del | Q(primary_has_hom_alt=False)) else: + # TODO results = results.annotate(primary_familyGuids=If( is_overlapped_del, F('primary_familyGuids'), @@ -315,6 +317,7 @@ def _get_sample_data(samples): affected=ArrayAgg('individual__affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: + # TODO from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + From 61e529dc30fff726fcc991d7f3c919da6e79fc4b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 14:45:44 -0400 Subject: [PATCH 058/109] clean up --- clickhouse_search/managers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1d57313751..40e64c87ce 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -156,12 +156,7 @@ def clinvar_field_prefix(self): def subquery_join(self, subquery, join_key='key'): - # Add key to intermediate select if not already present join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) - if join_key not in subquery.query.values_select: - # TODO unused - subquery.query.values_select = tuple([join_key, *subquery.query.values_select]) - subquery.query.select = tuple([Col(subquery.model._meta.db_table, join_field), *subquery.query.select]) # Add the join operation to the query table = SubqueryTable(subquery) @@ -876,7 +871,8 @@ def _quality_q(self, quality_filter, affected_samples, clinvar_override_q): return quality_q - def _annotate_failed_family_samples(self, entries, gt_filter, family_missing_type_samples): + @staticmethod + def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( ArrayFilter('calls', conditions=[{ 'gt': (gt_filter[0], f'not {gt_filter[1]}'), From 25a4128933f50311b40f85814ea4d9276d07e553 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 15:02:25 -0400 Subject: [PATCH 059/109] invalid search test --- clickhouse_search/search.py | 1 - clickhouse_search/search_tests.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 47b6fc35e4..9fb9a7fa36 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -317,7 +317,6 @@ def _get_sample_data(samples): affected=ArrayAgg('individual__affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: - # TODO from seqr.utils.search.utils import InvalidSearchException raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 26517c089e..3390ea85b6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -615,6 +615,17 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) + def test_invalid_search(self): + Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') + self._set_multi_project_search() + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([], locus={'rawItems': GENE_IDS[0]}) + self.assertEqual( + str(cm.exception), + 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', + ) + def test_variant_lookup(self): variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A')) self._assert_expected_variants([variant], [VARIANT_LOOKUP_VARIANT]) From 349e5280533a9d7945c723670ef8b24e7406b844 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 15:13:19 -0400 Subject: [PATCH 060/109] test multi project mult dat atype comp het --- clickhouse_search/search.py | 10 +++++----- clickhouse_search/search_tests.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 9fb9a7fa36..509107d6ba 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -126,8 +126,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) dataset_results = [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] if skip_individual_guid: - # TODO - _add_individual_guids(dataset_results, sv_sample_data, single_data_type=False) + _add_individual_guids(dataset_results, sv_sample_data) results += dataset_results return results @@ -219,18 +218,19 @@ def _result_as_tuple(results, field_prefix): return Tuple(*fields.keys(), output_field=NamedTupleField(list(fields.values()))) -def _add_individual_guids(results, sample_data, single_data_type=True): +def _add_individual_guids(results, sample_data): sample_map = {(s['family_guid'], s['sample_id']): s['individual_guid'] for s in sample_data['samples']} for result in results: if isinstance(result, list): for variant in result: - if single_data_type or 'svType' not in variant: - _set_individual_guids(variant, sample_map) + _set_individual_guids(variant, sample_map) else: _set_individual_guids(result, sample_map) def _set_individual_guids(result, sample_map): + if 'familyGenotypes' not in result: + return result['familyGuids'] = sorted(result['familyGenotypes'].keys()) individual_genotypes = defaultdict(list) for family_guid, genotypes in result.pop('familyGenotypes').items(): diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3390ea85b6..648072d378 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1087,7 +1087,7 @@ def test_secondary_annotations_filter(self): ) self._add_sample_type_samples('WES', individual__family__guid='F000014_14') - self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000014_14'])) + self.results_model.families.set(Family.objects.filter(guid__in=['F000002_2', 'F000011_11', 'F000014_14'])) self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', From a5bfb5c9fb1a699c9ff4366d3bb7ac3097902b13 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 15:54:24 -0400 Subject: [PATCH 061/109] test multi sample type multi dataset type --- clickhouse_search/managers.py | 1 - clickhouse_search/search.py | 3 +-- clickhouse_search/search_tests.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 40e64c87ce..0d2702ba31 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -933,7 +933,6 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an family_carriers = Cast(Tuple('familyGuids', GroupArray('carriers')), map_field) entries = entries.annotate(family_carriers=family_carriers) if annotate_hom_alts: - # TODO entries = entries.annotate(no_hom_alt_families=ArrayMap( ArrayFilter(GroupArray(Tuple('family_guid', 'has_hom_alt')), conditions=[{2: (None, 'NOT {field}')}]), mapped_expression='x.1', diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 509107d6ba..dbf23a36c9 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -164,12 +164,11 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ if results.has_annotation('primary_has_hom_alt'): results = results.filter(is_overlapped_del | Q(primary_has_hom_alt=False)) else: - # TODO results = results.annotate(primary_familyGuids=If( is_overlapped_del, F('primary_familyGuids'), ArrayIntersect('primary_familyGuids', 'primary_no_hom_alt_families'), - condition='', + condition='', output_field=ArrayField(StringField()), )) if num_families > 1: diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 648072d378..1ac25ff04f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -217,7 +217,7 @@ def test_multi_project_search(self): ) def test_both_sample_types_search(self): - Sample.objects.exclude(dataset_type='SNV_INDEL').update(is_active=False) + Sample.objects.filter(dataset_type='MITO').update(is_active=False) # One family (F000011_11) in a multi-project search has identical exome and genome data. self._set_multi_project_search() From c73574e19f6e032d9a56a67ec96a599055792173 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:13:15 -0400 Subject: [PATCH 062/109] better multi sample type tests --- clickhouse_search/managers.py | 2 -- clickhouse_search/search_tests.py | 26 ++++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 0d2702ba31..55974ea42b 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -779,7 +779,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit inheritance_q |= multi_sample_type_family_q entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples) elif inheritance_q is not None: - # TODO entries = entries.annotate(passes_inheritance=inheritance_q) inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q else: @@ -788,7 +787,6 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if quality_q is None: entries = entries.annotate(passes_quality=Value(True)) else: - # TODO entries = entries.annotate(passes_quality=quality_q) quality_q = Q(passes_quality=True) | multi_sample_type_family_q diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 1ac25ff04f..384d0a4eaf 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -234,20 +234,42 @@ def test_both_sample_types_search(self): # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome. # Variant 3 is inherited in both sample types. # Variant 4 is de novo in exome, but inherited in genome in the same parent that has variant 3. + self._assert_expected_search( + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, + GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], + inheritance_mode='any_affected', locus=None, + ) + + self._assert_expected_search( + [VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4], + inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, + ) + self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - inheritance_mode='de_novo', locus=None, + inheritance_mode='de_novo', quality_filter=None, ) self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) + self._assert_expected_search( + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, + VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], + inheritance_mode='any_affected', + ) + self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], inheritance_mode='de_novo', ) + self._assert_expected_search( + [VARIANT2_BOTH_SAMPLE_TYPES], + inheritance_mode='de_novo', quality_filter={'min_gq': 40} + ) + self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES]], - inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ + inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], ] ) From e794cdd9d499c9090010b990acf580cc9a69a2a5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:23:12 -0400 Subject: [PATCH 063/109] include svs in test results --- clickhouse_search/search_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 384d0a4eaf..db13126f7f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -246,7 +246,7 @@ def test_both_sample_types_search(self): ) self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, ) @@ -258,17 +258,18 @@ def test_both_sample_types_search(self): ) self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', ) self._assert_expected_search( - [VARIANT2_BOTH_SAMPLE_TYPES], + [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter={'min_gq': 40} ) self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES]], + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], + GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], ] From cc1f82ec358e070dca5b6573c03d9a0052474710 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:36:17 -0400 Subject: [PATCH 064/109] correctly filter both ends of padded interval --- clickhouse_search/managers.py | 8 ++++---- clickhouse_search/search_tests.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 55974ea42b..4fc611c520 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1045,14 +1045,14 @@ def filter_intervals(self, exclude_intervals=False, intervals=None, gene_interva # while the full variant_id filter is applied to the annotation table after the join intervals = [(chrom, pos, pos) for chrom, pos, _, _ in variant_ids] - if not (gene_intervals or intervals): - return entries - if padded_interval: - # TODO pos = padded_interval['start'] padding = int((padded_interval['end'] - pos) * padded_interval['padding']) intervals = [(padded_interval['chrom'], max(pos - padding, MIN_POS), min(pos + padding, MAX_POS))] + + if not (gene_intervals or intervals): + return entries + elif 'cn' in self.call_fields: # SV interval filtering occurs after joining on annotations to correctly incorporate end position if exclude_intervals: diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index db13126f7f..bc5ea96a47 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -677,8 +677,9 @@ def test_variant_lookup(self): variants = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', families, sample_type='WGS') self._assert_expected_variants(variants, [SV_VARIANT4, GCNV_VARIANT4]) + # reciprocal overlap does not meet the threshold for smaller events variants = sv_variant_lookup(self.user, 'suffix_140608_DUP', families, sample_type='WES') - self._assert_expected_variants(variants, [GCNV_VARIANT4, SV_VARIANT4]) + self._assert_expected_variants(variants, [GCNV_VARIANT4]) variants = sv_variant_lookup(self.user, 'suffix_140593_DUP', families, sample_type='WES') self._assert_expected_variants(variants, [GCNV_VARIANT3]) From 2329bc5c62c32542004ed58070c042fd236c2ae1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:39:19 -0400 Subject: [PATCH 065/109] fix comp het multi sample type multi data type --- clickhouse_search/search_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index bc5ea96a47..e3e765f382 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -268,7 +268,8 @@ def test_both_sample_types_search(self): ) self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], + [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, + [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT3], [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], From f2a3e1dde3b6443411c612616e2dd389830b590a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:44:03 -0400 Subject: [PATCH 066/109] test topmed --- clickhouse_search/managers.py | 1 - clickhouse_search/search_tests.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 4fc611c520..6a4a1ca48f 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -367,7 +367,6 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): if pop_filter.get('hh') is not None: for subfield in ['hom', 'hemi']: if subfield not in pop_subfields: - # TODO continue hh_q = Q(**{ f'populations__{population}__{subfield}__lte': pop_filter['hh'], diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e3e765f382..de9cdabb2a 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -748,6 +748,11 @@ def test_frequency_filter(self): [VARIANT2, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], freqs={'gnomad_genomes': {'af': 0.05, 'hh': 1}, 'gnomad_mito': {'af': 0.05}}, ) + self._assert_expected_search( + [VARIANT4, GCNV_VARIANT3, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + freqs={'topmed': {'af': 0.05, 'hh': 1}, 'sv_callset': {'ac': 50}}, + ) + self._set_sv_family_search() self._assert_expected_search( [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, From bd23ac126227433198a30349af67734bcee1954e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:51:33 -0400 Subject: [PATCH 067/109] fix end chrom filtering --- clickhouse_search/managers.py | 7 +++---- clickhouse_search/search_tests.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 6a4a1ca48f..da61ddc121 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -470,10 +470,9 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl def _interval_query(self, chrom, start, end): q = Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) - if hasattr(self.model, 'endChrom'): - # TODO - q |= Q(endChrom__isnull=True, chrom=chrom, end__range=(start, end)) - q |= Q(endChrom=chrom, end__range=(start, end)) + if hasattr(self.model, 'end_chrom'): + q |= Q(end_chrom__isnull=True, chrom=chrom, end__range=(start, end)) + q |= Q(end_chrom=chrom, end__range=(start, end)) elif hasattr(self.model, 'end'): q |= Q(chrom=chrom, end__range=(start, end)) q |= Q(chrom=chrom, pos__lte=start, end__gte=end) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index de9cdabb2a..25d7095a02 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -267,6 +267,7 @@ def test_both_sample_types_search(self): inheritance_mode='de_novo', quality_filter={'min_gq': 40} ) + self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT3], [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], From a99915f7c00579f7dc6a936cb9098d925a3a2fcc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 16:56:55 -0400 Subject: [PATCH 068/109] better hgmd tests --- clickhouse_search/managers.py | 2 -- clickhouse_search/search_tests.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index da61ddc121..e612a8bc0c 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -518,7 +518,6 @@ def _parse_annotation_filters(self, annotations, pathogenicity): for field, value in (pathogenicity or {}).items(): if not value: - # TODO continue elif field == HGMD_KEY: filters_by_field[HGMD_KEY] = self._hgmd_filter(value) @@ -567,7 +566,6 @@ def _hgmd_filter(hgmd): if min_class == max_class: return ('{field}__classification', min_class) elif min_class and max_class: - # TODO return ('{field}__classification__range', (min_class, max_class)) return ('{field}__classification__gt', min_class) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 25d7095a02..0c032b36c3 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -784,8 +784,9 @@ def test_frequency_filter(self): def test_annotations_filter(self): self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}) + self._assert_expected_search([], pathogenicity={'hgmd': ['disease_causing', 'likely_disease_causing']}) - pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting', 'benign']} + pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting', 'benign'], 'hgmd': []} self._assert_expected_search( [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, ) From 65c8efcb2e89a0ab752f92d9bcb127326d7c2661 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 17:05:24 -0400 Subject: [PATCH 069/109] fix recssive --- clickhouse_search/search_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 0c032b36c3..3ac6eeaa46 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -270,10 +270,14 @@ def test_both_sample_types_search(self): self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, - [VARIANT2_BOTH_SAMPLE_TYPES, GCNV_VARIANT3], [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], + [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4], + [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ - {}, {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], + {}, {}, [ + {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, + {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': {'geneId': 'ENSG00000277258', 'majorConsequence': 'LOF'}}, + ], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{}, {}], ] ) From 42a1abb98f2e00a9bf3fb62955f66b69b48d927d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 17:14:55 -0400 Subject: [PATCH 070/109] fix cached reults --- clickhouse_search/search_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3ac6eeaa46..e87c255423 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -274,10 +274,8 @@ def test_both_sample_types_search(self): [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ - {}, {}, [ - {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[2][3]}, - {'selectedGeneId': 'ENSG00000277258', 'selectedTranscript': {'geneId': 'ENSG00000277258', 'majorConsequence': 'LOF'}}, - ], [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{}, {}], + {}, {}, [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], + [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{}, {}], ] ) From 15b330e8fa09f247c67b24680b3655f5b4291a02 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 18 Aug 2025 17:25:17 -0400 Subject: [PATCH 071/109] fix cached reults --- clickhouse_search/search_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index e87c255423..5004618ccf 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -275,7 +275,8 @@ def test_both_sample_types_search(self): GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, quality_filter=None, cached_variant_fields=[ {}, {}, [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], - [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, [{}, {}], + [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, + [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], ] ) From 1bed4d57bcd92850182f14a33f71c20dd5dc0be3 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 19 Aug 2025 11:31:18 -0400 Subject: [PATCH 072/109] use array field --- clickhouse_search/backend/fields.py | 29 ------------------- clickhouse_search/migrations/0001_initial.py | 2 +- ..._annotationsdiskgrch37snvindel_and_more.py | 2 +- clickhouse_search/models.py | 4 +-- 4 files changed, 4 insertions(+), 33 deletions(-) diff --git a/clickhouse_search/backend/fields.py b/clickhouse_search/backend/fields.py index d729c4f3cf..a3f1a53510 100644 --- a/clickhouse_search/backend/fields.py +++ b/clickhouse_search/backend/fields.py @@ -93,35 +93,6 @@ class UInt64FieldDeltaCodecField(models.UInt64Field): def db_type(self, connection): return f'{super().db_type(connection)} CODEC(Delta(8), ZSTD(1))' -class BitmapField(models.ArrayField): - - def __init__(self, *args, null_when_empty=False, **kwargs): - self.null_when_empty = null_when_empty - super().__init__(*args, **kwargs) - - def clone(self): - clone = super().clone() - clone.null_when_empty = self.null_when_empty - return clone - - def get_internal_type(self): - return "BitmapField" - - @property - def description(self): - return "Bitmap of %s" % self.base_field.description - - def db_type(self, connection): - return "AggregateFunction(groupBitmap, %s)" % self.base_field.db_type(connection) - - def cast_db_type(self, connection): - return "bitmapBuild(%s)" % self.base_field.cast_db_type(connection) - - def from_db_value(self, value, expression, connection): - if self.null_when_empty and not value: - return None - return super().from_db_value(value, expression, connection) - class NamedTupleField(models.TupleField): def __init__(self, *args, null_if_empty=False, null_empty_arrays=False, rename_fields=None, **kwargs): diff --git a/clickhouse_search/migrations/0001_initial.py b/clickhouse_search/migrations/0001_initial.py index c308cca9cd..af44e984c9 100644 --- a/clickhouse_search/migrations/0001_initial.py +++ b/clickhouse_search/migrations/0001_initial.py @@ -90,7 +90,7 @@ class Migration(migrations.Migration): ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), - ('geneId_ids', clickhouse_search.backend.fields.BitmapField(base_field=clickhouse_backend.models.UInt32Field())), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.UInt32FieldDeltaCodecField())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), diff --git a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py index aec53d173d..0667ecb7e8 100644 --- a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py +++ b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py @@ -83,7 +83,7 @@ class Migration(migrations.Migration): ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), - ('geneId_ids', clickhouse_search.backend.fields.BitmapField(base_field=clickhouse_backend.models.UInt32Field())), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.UInt32FieldDeltaCodecField())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 054cc117ee..720badf4ca 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -4,7 +4,7 @@ from django.db.models import options, ForeignKey, OneToOneField, Func, CASCADE, PROTECT from clickhouse_search.backend.engines import CollapsingMergeTree, EmbeddedRocksDB, Join -from clickhouse_search.backend.fields import BitmapField, Enum8Field, NestedField, UInt32FieldDeltaCodecField, UInt64FieldDeltaCodecField, NamedTupleField +from clickhouse_search.backend.fields import Enum8Field, NestedField, UInt32FieldDeltaCodecField, UInt64FieldDeltaCodecField, NamedTupleField from clickhouse_search.backend.functions import ArrayDistinct, ArrayFlatten, ArrayMin, ArrayMax from clickhouse_search.managers import EntriesManager, AnnotationsQuerySet from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 @@ -589,7 +589,7 @@ class BaseEntriesSnvIndel(BaseEntries): sample_type = models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')]) is_gnomad_gt_5_percent = models.BoolField() is_annotated_in_any_gene = models.BoolField() - geneId_ids = BitmapField(models.UInt32Field(), null_when_empty=True) + geneId_ids = models.ArrayField(UInt32FieldDeltaCodecField()) calls = models.ArrayField(NamedTupleField(CALL_FIELDS)) class Meta: From 6516dc47bde5d1bb1c5eaa3b9a0d4ea0167ea396 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 19 Aug 2025 11:31:45 -0400 Subject: [PATCH 073/109] ws --- clickhouse_search/backend/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_search/backend/fields.py b/clickhouse_search/backend/fields.py index a3f1a53510..a43519218c 100644 --- a/clickhouse_search/backend/fields.py +++ b/clickhouse_search/backend/fields.py @@ -93,6 +93,7 @@ class UInt64FieldDeltaCodecField(models.UInt64Field): def db_type(self, connection): return f'{super().db_type(connection)} CODEC(Delta(8), ZSTD(1))' + class NamedTupleField(models.TupleField): def __init__(self, *args, null_if_empty=False, null_empty_arrays=False, rename_fields=None, **kwargs): From eb9852d5f475f7109823773f190ca1bb981a368b Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 19 Aug 2025 11:42:16 -0400 Subject: [PATCH 074/109] just use uint --- clickhouse_search/migrations/0001_initial.py | 2 +- .../migrations/0002_annotationsdiskgrch37snvindel_and_more.py | 2 +- clickhouse_search/models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/migrations/0001_initial.py b/clickhouse_search/migrations/0001_initial.py index af44e984c9..b733654f63 100644 --- a/clickhouse_search/migrations/0001_initial.py +++ b/clickhouse_search/migrations/0001_initial.py @@ -90,7 +90,7 @@ class Migration(migrations.Migration): ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), - ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.UInt32FieldDeltaCodecField())), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), diff --git a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py index 0667ecb7e8..27915f3895 100644 --- a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py +++ b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py @@ -83,7 +83,7 @@ class Migration(migrations.Migration): ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), - ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.UInt32FieldDeltaCodecField())), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 720badf4ca..c7e25c23d3 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -589,7 +589,7 @@ class BaseEntriesSnvIndel(BaseEntries): sample_type = models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')]) is_gnomad_gt_5_percent = models.BoolField() is_annotated_in_any_gene = models.BoolField() - geneId_ids = models.ArrayField(UInt32FieldDeltaCodecField()) + geneId_ids = models.ArrayField(models.UInt32Field()) calls = models.ArrayField(NamedTupleField(CALL_FIELDS)) class Meta: From 5a5bfaccee873b0917d20eb8dfa73f7158a8239b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 11:55:15 -0400 Subject: [PATCH 075/109] better bacthing for saved variant updates --- .../commands/set_saved_variant_key.py | 34 +++++++++---------- .../tests/set_saved_variant_key_tests.py | 4 +++ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 2a2c5649a8..07df47e39b 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -2258,31 +2258,29 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ if not variant_key_map: return set(variants_ids) - mapped_variant_ids = variant_key_map.keys() + mapped_variant_ids = list(variant_key_map.keys()) if variant_id_updates: reverse_lookup = {v: k for k, v in variant_id_updates.items()} mapped_variant_ids = [reverse_lookup[vid] for vid in mapped_variant_ids] - saved_variants = SavedVariant.objects.filter( - family__project__genome_version=genome_version, variant_id__in=mapped_variant_ids, - ) + update_fields = ['key', 'dataset_type'] if variant_id_updates: update_fields.append('variant_id') - update_variants = [] total_num_updated = 0 - for variant in saved_variants: - if variant_id_updates: - variant.variant_id = variant_id_updates[variant.variant_id] - variant.key = variant_key_map[variant.variant_id] - variant.dataset_type = dataset_type - update_variants.append(variant) - if len(update_variants) >= BATCH_SIZE: - num_updated = SavedVariant.objects.bulk_update(update_variants, update_fields) - logger.info(f'Updated batch of {num_updated}') - total_num_updated += num_updated - update_variants = [] - if update_variants: - total_num_updated += SavedVariant.objects.bulk_update(update_variants, update_fields) + for i in range(0, len(mapped_variant_ids), BATCH_SIZE): + batch_ids = mapped_variant_ids[i:i + BATCH_SIZE] + saved_variants = SavedVariant.objects.filter( + family__project__genome_version=genome_version, variant_id__in=batch_ids, + ) + for variant in saved_variants: + if variant_id_updates: + variant.variant_id = variant_id_updates[variant.variant_id] + variant.key = variant_key_map[variant.variant_id] + variant.dataset_type = dataset_type + num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields) + logger.info(f'Updated batch of {num_updated}') + total_num_updated += num_updated + logger.info(f'Updated keys for {total_num_updated} {dataset_type} (GRCh{genome_version}) variants') no_key = set(variants_ids) - set(variant_key_map.keys()) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index e8f1d7a79d..5f2488c68d 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -32,6 +32,7 @@ def test_command(self, mock_subprocess): ('Updated genotypes for 7 variants', None), ('Finding keys for 1 MITO (GRCh38) variant ids', None), ('Found 1 keys', None), + ('Updated batch of 1', None), ('Updated keys for 1 MITO (GRCh38) variants', None), ('Finding keys for 1 SNV_INDEL (GRCh38) variant ids', None), ('Found 1 keys', None), @@ -46,13 +47,16 @@ def test_command(self, mock_subprocess): ('Mapping reloaded SV_WES IDs to latest version', None), ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), ('Found 1 keys', None), + ('Updated batch of 1', None), ('Updated keys for 1 SV_WES (GRCh38) variants', None), ('Mapping reloaded SV_WGS IDs to latest version', None), ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None), ('Found 1 keys', None), + ('Updated batch of 1', None), ('Updated keys for 1 SV_WGS (GRCh38) variants', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), ('Found 1 keys', None), + ('Updated batch of 1', None), ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), ('No key found for 6 variants', None), ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), From 9235f1ba46855dd821accdf52c11061cfc23acde Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 19 Aug 2025 12:03:12 -0400 Subject: [PATCH 076/109] a few missing columns --- clickhouse_search/fixtures/clickhouse_search.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 9d5bffe75e..55afe33016 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -356,6 +356,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -374,6 +376,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -392,6 +396,8 @@ "sample_type": "WES", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], From bea3870d6807d114ece7d5e494ac4674d11543a2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 12:18:16 -0400 Subject: [PATCH 077/109] properly filter gnomad sv ac --- clickhouse_search/managers.py | 8 +++++++- clickhouse_search/search_tests.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index a849416473..085db052c2 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -366,7 +366,13 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): })) results = results.filter(af_q) elif pop_filter.get('ac') is not None: - results = results.filter(**{f'populations__{population}__ac__lte': pop_filter['ac']}) + if 'ac' in pop_subfields: + ac_field = f'populations__{population}__ac' + else: + ac_field = f'ac_{population}' + results = results.annotate(**{ac_field: F(f'populations__{population}__het') + (F(f'populations__{population}__hom') * 2)}) + + results = results.filter(**{f'{ac_field}__lte': pop_filter['ac']}) if pop_filter.get('hh') is not None: for subfield in ['hom', 'hemi']: diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 26517c089e..67ea29733a 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -716,6 +716,9 @@ def test_frequency_filter(self): self._assert_expected_search( [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'af': 0.001}}, ) + self._assert_expected_search( + [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], freqs={'gnomad_svs': {'ac': 4000}}, + ) self._reset_search_families() self._assert_expected_search( From f5d9bca6736d448c2a592fc9eb829087623be44f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 12:38:55 -0400 Subject: [PATCH 078/109] allow sample id mismatches across dataset types --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1f04b63aec..9f7ec4e2bd 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -310,7 +310,7 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): def _get_sample_data(samples): - mismatch_affected_samples = samples.values('sample_id').annotate( + mismatch_affected_samples = samples.values('sample_id', 'dataset_type').annotate( projects=ArrayAgg('individual__family__project__name', distinct=True), affected=ArrayAgg('individual__affected', distinct=True), ).filter(affected__len__gt=1) From 9e3103181108a8212d5de62460381bfe317ce322 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 13:31:18 -0400 Subject: [PATCH 079/109] fix number of families in log --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 9f7ec4e2bd..ae66bf15ab 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -40,7 +40,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} for dataset_type, sample_data in sample_data_by_dataset_type.items(): - logger.info(f'Loading {dataset_type} data for {len(sample_data)} families', user) + logger.info(f'Loading {dataset_type} data for {len(set(sample_data['family_guids']))} families', user) entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] From 602f72dbdf4504ca1985c3112955b2de409678c7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 13:42:03 -0400 Subject: [PATCH 080/109] fix estring encoding --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index ae66bf15ab..25f262833f 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -40,7 +40,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} for dataset_type, sample_data in sample_data_by_dataset_type.items(): - logger.info(f'Loading {dataset_type} data for {len(set(sample_data['family_guids']))} families', user) + logger.info(f'Loading {dataset_type} data for {len(set(sample_data["family_guids"]))} families', user) entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] From b2f087abfd869e3377907d43138d0b89718766bf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 14:20:04 -0400 Subject: [PATCH 081/109] correctly handle different genotype annotations for different dataset types --- clickhouse_search/search.py | 43 ++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 25f262833f..2cb90915c8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -178,27 +178,40 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ secondary_family_expr = 'secondary_familyGuids' if results.has_annotation('secondary_familyGuids') else ArrayMap( 'secondary_familyGenotypes', mapped_expression='x.1', ) - genotype_expressions = { - 'primary_genotypes': ArrayFilter('primary_genotypes', conditions=[ - {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, - ]), - 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ - {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, - ]), - 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ - {1: (None, 'arrayExists(g -> g.1 = {field}, secondary_familyGenotypes)')}, - ]), - 'secondary_familyGenotypes': ArrayFilter('secondary_familyGenotypes', conditions=[ - {1: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, - ]), - } + if results.has_annotation('primary_genotypes'): + genotype_expressions = { + 'primary_genotypes': ArrayFilter('primary_genotypes', conditions=[ + {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, + ]), + 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ + {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, + ]), + } + elif results.has_annotation('secondary_genotypes'): + genotype_expressions = { + 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ + {1: ('secondary_familyGuids', 'has({value}, {field})')}, + ]), + 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ + {2: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, + ]), + } + else: + genotype_expressions = { + 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ + {1: (None, 'arrayExists(g -> g.1 = {field}, secondary_familyGenotypes)')}, + ]), + 'secondary_familyGenotypes': ArrayFilter('secondary_familyGenotypes', conditions=[ + {1: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, + ]), + } results = results.annotate( primary_familyGuids=ArrayIntersect( primary_family_expr, secondary_family_expr, output_field=ArrayField(StringField()), ), ).filter(primary_familyGuids__not_empty=True).annotate( secondary_familyGuids=F('primary_familyGuids'), - **{k: v for k, v in genotype_expressions.items() if results.has_annotation(k)}, + **genotype_expressions, ) return results.annotate( From d761e34fa3b72653e8bcbe2dc0401d4f572d93fa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 15:26:42 -0400 Subject: [PATCH 082/109] fix reassignment error --- clickhouse_search/search.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2cb90915c8..da3c1b74fb 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -101,15 +101,11 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b if not families: continue - snv_indel_sample_data = { + entries = entry_cls.objects.search({ **snv_indel_sample_data, 'family_guids': list(families), 'samples': [s for s in snv_indel_sample_data['samples'] if s['family_guid'] in families] - } - - entries = entry_cls.objects.search( - snv_indel_sample_data, skip_individual_guid=skip_individual_guid, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True, - ) + }, skip_individual_guid=skip_individual_guid, **search_kwargs, annotations=annotations, inheritance_mode=COMPOUND_HET_ALLOW_HOM_ALTS, annotate_carriers=True, annotate_hom_alts=True) snv_indel_q = annotations_cls.objects.subquery_join(entries).search(**search_kwargs, annotations=annotations) sv_sample_data = { From 6368e1defbb54e37e475923c8d19f08116a022a1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 10:28:22 -0400 Subject: [PATCH 083/109] actually test get saved variant sv --- seqr/views/apis/summary_data_api_tests.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 5b1393ca6a..72615c3941 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -406,6 +406,14 @@ def test_saved_variants_page(self): if 'totalSampleCounts' in response_json: self.assertDictEqual(response_json['totalSampleCounts'], {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}}) + all_tag_url = reverse(saved_variants_page, args=['ALL']) + response = self.client.get('{}?gene=ENSG00000135953'.format(all_tag_url)) + self.assertEqual(response.status_code, 200) + report_variants = {'SV0027168_191912632_r0384_rare', 'SV0027167_191912633_r0384_rare'} + self.assertSetEqual(set(response.json()['savedVariantsByGuid'].keys()), { + 'SV0000002_1248367227_r0390_100', 'SV0000013_prefix_19107_DEL_r00', *report_variants, *expected_variant_guids, + }) + # Test analyst behavior self.login_analyst_user() response = self.client.get(gene_url) @@ -414,13 +422,6 @@ def test_saved_variants_page(self): self.assertSetEqual(set(response_json.keys()), self.SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json['savedVariantsByGuid'].keys()), expected_variant_guids) - all_tag_url = reverse(saved_variants_page, args=['ALL']) - response = self.client.get('{}?gene=ENSG00000135953'.format(all_tag_url)) - self.assertEqual(response.status_code, 200) - expected_variant_guids.add('SV0000002_1248367227_r0390_100') - report_variants = {'SV0027168_191912632_r0384_rare', 'SV0027167_191912633_r0384_rare'} - self.assertSetEqual(set(response.json()['savedVariantsByGuid'].keys()), {*report_variants, *expected_variant_guids}) - multi_tag_url = reverse(saved_variants_page, args=['Review;Tier 1 - Novel gene and phenotype']) response = self.client.get('{}?gene=ENSG00000135953'.format(multi_tag_url)) self.assertEqual(response.status_code, 200) From ca53d9a5bdbf9b46fb0254b81d307b852594463c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 19 Aug 2025 16:52:11 -0400 Subject: [PATCH 084/109] fix single sample type property --- clickhouse_search/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 445937c2bb..f080beb275 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -134,7 +134,7 @@ def transcript_fields(self): @property def single_sample_type(self): - return getattr(self.model, 'SV_TYPES', None) + return getattr(self.entry_model, 'SAMPLE_TYPE', None) @property def entry_field(self): From 41d3a2cfd530d1a1e942768edb7839575a7070a6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 10:29:51 -0400 Subject: [PATCH 085/109] fix test --- seqr/views/apis/summary_data_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 72615c3941..d3011e4d03 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -873,7 +873,7 @@ def test_mme_details(self, *args): def test_saved_variants_page(self): super(AnvilSummaryDataAPITest, self).test_saved_variants_page() assert_has_expected_calls(self, [ - self.no_access_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user + self.no_access_user, self.manager_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') From 633c637748dc4e807ed207e4789e553150754adf Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 20 Aug 2025 10:56:40 -0400 Subject: [PATCH 086/109] feat: implement new fields (#4942) * implement new fields * use array field * ws * just use uint * a few missing columns --- .../fixtures/clickhouse_saved_variants.json | 6 ++ .../fixtures/clickhouse_search.json | 72 +++++++++++++++++++ clickhouse_search/migrations/0001_initial.py | 6 +- ..._annotationsdiskgrch37snvindel_and_more.py | 6 +- clickhouse_search/models.py | 6 +- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 8ad858a65f..6a7e7706e7 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -66,6 +66,8 @@ "sample_type": "WGS", "xpos": 1248367227, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [6, 48], "filters": [], "sign": 1, "calls": [ @@ -118,6 +120,8 @@ "sample_type": "WES", "xpos": 2103343353, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [48, 45], "filters": [], "sign": 1, "calls": [ @@ -196,6 +200,8 @@ "sample_type": "WGS", "xpos": 1248367227, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index ea8a261dc7..55afe33016 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -274,6 +274,8 @@ "sample_type": "WES", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -292,6 +294,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -310,6 +314,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -328,6 +334,8 @@ "sample_type": "WES", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], @@ -348,6 +356,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -366,6 +376,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -384,6 +396,8 @@ "sample_type": "WES", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], @@ -404,6 +418,8 @@ "sample_type": "WES", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -420,6 +436,8 @@ "sample_type": "WES", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -436,6 +454,8 @@ "sample_type": "WES", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -452,6 +472,8 @@ "sample_type": "WES", "xpos": 1000010146, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -468,6 +490,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -484,6 +508,8 @@ "sample_type": "WGS", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -500,6 +526,8 @@ "sample_type": "WGS", "xpos": 1000010146, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -516,6 +544,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -534,6 +564,8 @@ "sample_type": "WGS", "xpos": 1038724419, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [60, 72], "filters": [], "sign": 1, "calls": [ @@ -552,6 +584,8 @@ "sample_type": "WGS", "xpos": 1091502721, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [61, 60], "filters": [], "sign": 1, "calls": [ @@ -570,6 +604,8 @@ "sample_type": "WGS", "xpos": 1091511686, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [61], "filters": [ "VQSRTrancheSNP99.95to100.00" ], @@ -590,6 +626,8 @@ "sample_type": "WGS", "xpos": 1009310123, "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [71], "filters": [], "sign": 1, "calls": [ @@ -636,6 +674,8 @@ "sample_type": "WES", "xpos": 7143270172, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": true, + "geneId_ids": [73, 74], "filters": [ "VQSRTrancheSNP99.90to99.95" ], @@ -655,6 +695,8 @@ "sample_type": "WGS", "xpos": 1000010439, "is_gnomad_gt_5_percent": true, + "is_annotated_in_any_gene": false, + "geneId_ids": [], "filters": [], "sign": 1, "calls": [ @@ -1454,5 +1496,35 @@ "gencode_gene_type": "protein_coding", "gencode_release": 27 } +}, +{ + "model": "reference_data.geneinfo", + "pk": 73, + "fields": { + "gene_id": "ENSG00000271079", + "gene_symbol": "CTAGE15", + "chrom_grch38": "7", + "start_grch38": 143571801, + "end_grch38": 143574387, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } +}, +{ + "model": "reference_data.geneinfo", + "pk": 74, + "fields": { + "gene_id": "ENSG00000176227", + "gene_symbol": "CTAGE6", + "chrom_grch38": "7", + "start_grch38": 91500851, + "end_grch38": 91525764, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } } ] \ No newline at end of file diff --git a/clickhouse_search/migrations/0001_initial.py b/clickhouse_search/migrations/0001_initial.py index 05d5815bc7..b733654f63 100644 --- a/clickhouse_search/migrations/0001_initial.py +++ b/clickhouse_search/migrations/0001_initial.py @@ -89,14 +89,16 @@ class Migration(migrations.Migration): ('sample_type', clickhouse_backend.models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')])), ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), + ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), ], options={ 'db_table': 'GRCh38/SNV_INDEL/entries', - 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), partition_by='project_guid'), - 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent'), + 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'), + 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py index ef42cbe289..27915f3895 100644 --- a/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py +++ b/clickhouse_search/migrations/0002_annotationsdiskgrch37snvindel_and_more.py @@ -82,6 +82,8 @@ class Migration(migrations.Migration): ('sample_type', clickhouse_backend.models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')])), ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), ('is_gnomad_gt_5_percent', clickhouse_backend.models.BoolField()), + ('is_annotated_in_any_gene', clickhouse_backend.models.BoolField()), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('ab', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('dp', clickhouse_backend.models.UInt16Field(blank=True, null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), @@ -89,8 +91,8 @@ class Migration(migrations.Migration): options={ 'db_table': 'GRCh37/SNV_INDEL/entries', 'abstract': False, - 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), partition_by='project_guid'), - 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent'), + 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'), + 'projection': clickhouse_search.models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index c23171abcf..c7e25c23d3 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -588,18 +588,20 @@ class BaseEntriesSnvIndel(BaseEntries): sample_type = models.Enum8Field(choices=[(1, 'WES'), (2, 'WGS')]) is_gnomad_gt_5_percent = models.BoolField() + is_annotated_in_any_gene = models.BoolField() + geneId_ids = models.ArrayField(models.UInt32Field()) calls = models.ArrayField(NamedTupleField(CALL_FIELDS)) class Meta: abstract = True engine = CollapsingMergeTree( 'sign', - order_by=('project_guid', 'family_guid', 'is_gnomad_gt_5_percent', 'sample_type', 'key'), + order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, ) - projection = Projection('xpos_projection', order_by='xpos, is_gnomad_gt_5_percent') + projection = Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos') class EntriesGRCh37SnvIndel(BaseEntriesSnvIndel): From 30f8341b2bd4f5d54f46438b21d45ded0a09530d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 11:19:29 -0400 Subject: [PATCH 087/109] add large list specific logic --- clickhouse_search/managers.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index f080beb275..868bdd84d0 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -683,7 +683,7 @@ def clinvar_model(self): return self.model.clinvar_join.rel.related_model def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): - entries = self.filter_intervals(**(parsed_locus or {})) + entries = self.filter_locus(**(parsed_locus or {})) entries = self._join_annotations(entries) @@ -1040,7 +1040,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_intervals(self, exclude_intervals=False, intervals=None, gene_intervals=None, variant_ids=None, padded_interval=None, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_id_ids=None, variant_ids=None, padded_interval=None, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, @@ -1051,10 +1051,6 @@ def filter_intervals(self, exclude_intervals=False, intervals=None, gene_interva pos = padded_interval['start'] padding = int((padded_interval['end'] - pos) * padded_interval['padding']) intervals = [(padded_interval['chrom'], max(pos - padding, MIN_POS), min(pos + padding, MAX_POS))] - - if not (gene_intervals or intervals): - return entries - elif 'cn' in self.call_fields: # SV interval filtering occurs after joining on annotations to correctly incorporate end position if exclude_intervals: @@ -1062,17 +1058,30 @@ def filter_intervals(self, exclude_intervals=False, intervals=None, gene_interva else: chromosomes = {chrom for chrom, _, _ in (gene_intervals or []) + (intervals or [])} intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] - elif gene_intervals: - intervals = (intervals or []) + gene_intervals + gene_intervals = None + + if not (gene_intervals or intervals): + return entries + + locus_q = None + if gene_intervals: + if exclude_intervals or 'is_annotated_in_any_gene' not in self.query.annotations or len(gene_intervals) < 100: # TODO real threshhold + intervals = (gene_intervals or []) + (intervals or []) + else: + if not intervals: + entries = entries.filter(is_annotated_in_any_gene=Value(True)) if intervals: interval_q = self._interval_query(*intervals[0]) for interval in intervals[1:]: interval_q |= self._interval_query(*interval) - filter_func = entries.exclude if exclude_intervals else entries.filter - entries = filter_func(interval_q) + if locus_q is None: + locus_q = interval_q + else: + locus_q |= interval_q - return entries + filter_func = entries.exclude if exclude_intervals else entries.filter + return filter_func(locus_q) @staticmethod def _interval_query(chrom, start, end): From f7ebe1e83ee3d12bb4a36d21552223fbf6150a58 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 11:42:15 -0400 Subject: [PATCH 088/109] get gene db ids --- clickhouse_search/managers.py | 2 +- seqr/utils/gene_utils.py | 16 ++++++++-------- seqr/utils/search/utils.py | 32 +++++++++++++++++--------------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 868bdd84d0..0141191b9e 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1065,7 +1065,7 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N locus_q = None if gene_intervals: - if exclude_intervals or 'is_annotated_in_any_gene' not in self.query.annotations or len(gene_intervals) < 100: # TODO real threshhold + if exclude_intervals or not hasattr(self.model, 'is_annotated_in_any_gene') or len(gene_intervals) < 100: # TODO real threshhold intervals = (gene_intervals or []) + (intervals or []) else: if not intervals: diff --git a/seqr/utils/gene_utils.py b/seqr/utils/gene_utils.py index 06b2572981..1b2b615715 100644 --- a/seqr/utils/gene_utils.py +++ b/seqr/utils/gene_utils.py @@ -16,8 +16,8 @@ def get_gene(gene_id, user): return gene_json -def get_genes(gene_ids, genome_version=None): - return _get_genes(gene_ids, genome_version=genome_version) +def get_genes(gene_ids, genome_version=None, **kwargs): + return _get_genes(gene_ids, genome_version=genome_version, **kwargs) def get_genes_for_variant_display(gene_ids, genome_version): @@ -32,13 +32,13 @@ def get_genes_with_detail(gene_ids, user): return _get_genes(gene_ids, user=user, gene_fields=ALL_GENE_FIELDS) -def _get_genes(gene_ids, user=None, gene_fields=None, genome_version=None): +def _get_genes(gene_ids, user=None, genome_version=None, **kwargs): gene_filter = {} _add_genome_version_filter(gene_filter, genome_version) if gene_ids is not None: gene_filter['gene_id__in'] = gene_ids genes = GeneInfo.objects.filter(**gene_filter) - return {gene['geneId']: gene for gene in _get_json_for_genes(genes, user=user, gene_fields=gene_fields)} + return {gene['geneId']: gene for gene in _get_json_for_genes(genes, user=user, **kwargs)} def _add_genome_version_filter(gene_filter, genome_version): @@ -124,7 +124,7 @@ def _add_mgi(gene): } ALL_GENE_FIELDS.update(VARIANT_GENE_FIELDS) -def _get_json_for_genes(genes, user=None, gene_fields=None): +def _get_json_for_genes(genes, user=None, gene_fields=None, **kwargs): if not gene_fields: gene_fields = {} @@ -155,10 +155,10 @@ def _process_result(result, gene): '{}_set'.format(model.__name__.lower()), queryset=model.objects.only('gene__gene_id', *model._meta.json_fields))) - return _get_json_for_models(genes, process_result=_process_result) + return _get_json_for_models(genes, process_result=_process_result, **kwargs) -def parse_locus_list_items(request_json, genome_version=None): +def parse_locus_list_items(request_json, genome_version=None, **kwargs): raw_items = request_json.get('rawItems') if not raw_items: return None, None, None @@ -196,6 +196,6 @@ def parse_locus_list_items(request_json, genome_version=None): gene_symbols_to_ids = get_gene_ids_for_gene_symbols(gene_symbols, genome_version=genome_version) invalid_items += [symbol for symbol in gene_symbols if not gene_symbols_to_ids.get(symbol)] gene_ids.update({gene_ids[0] for gene_ids in gene_symbols_to_ids.values() if len(gene_ids)}) - genes_by_id = get_genes(list(gene_ids), genome_version=genome_version) if gene_ids else {} + genes_by_id = get_genes(list(gene_ids), genome_version=genome_version, **kwargs) if gene_ids else {} invalid_items += [gene_id for gene_id in gene_ids if not genes_by_id.get(gene_id)] return genes_by_id, intervals, invalid_items \ No newline at end of file diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index cb8fcf792e..1d7a7a6619 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -324,7 +324,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, rs_ids = None variant_ids = None parsed_variant_ids = None - genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version) + genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) if not (genes or intervals): @@ -619,31 +619,33 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, **kwargs): +def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, include_db_ids=True, **kwargs): intervals = [_format_interval(**interval) for interval in intervals or []] - gene_intervals = gene_ids = None + parsed_locus = { + 'intervals': intervals or None, + 'gene_intervals': None, + 'exclude_intervals': exclude_locations, + 'gene_ids': None, + 'variant_ids': parsed_variant_ids, + 'rs_ids': rs_ids, + } if genes: gene_intervals = sorted([ [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() ]) if exclude_locations: - intervals += gene_intervals - gene_intervals = None + parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + gene_intervals else: - gene_ids = sorted(genes.keys()) + parsed_locus['gene_ids'] = sorted(genes.keys()) + parsed_locus['gene_intervals'] = gene_intervals + if include_db_ids: + parsed_locus['gene_id_ids'] = [gene['id'] for gene in genes.values()] - return { - 'intervals': intervals or None, - 'gene_intervals': gene_intervals, - 'exclude_intervals': exclude_locations, - 'gene_ids': gene_ids, - 'variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, - } + return parsed_locus def _parse_locus_intervals(*args, **kwargs): - parsed_locus = _parse_locus_gene_intervals(*args, **kwargs) + parsed_locus = _parse_locus_gene_intervals(*args, include_db_ids=False, **kwargs) gene_intervals = parsed_locus.pop('gene_intervals') if gene_intervals: parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + gene_intervals From 09d71478e67b047b578598521aecb013bd7db881 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 12:12:46 -0400 Subject: [PATCH 089/109] filter on db ids --- clickhouse_search/backend/functions.py | 14 ++++++++++++++ clickhouse_search/managers.py | 1 + 2 files changed, 15 insertions(+) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 1a3300a220..07329222b7 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -120,6 +120,20 @@ class ArrayNotEmptyTransform(lookups.Transform): output_field = BooleanField() +@ArrayField.register_lookup +class BitmapHasAny(ArrayLookup): + lookup_name = "bitmap_has_any" + function = "bitmapHasAny" + + def process_lhs(self, compiler, connection): + lhs, lhs_params = super().process_lhs(compiler, connection) + return f'bitmapBuild({lhs})', lhs_params + + def process_rhs(self, compiler, connection): + rhs, rhs_params = super().process_rhs(compiler, connection) + return f'bitmapBuild{rhs.split("::")[0]}', rhs_params + + class DictGet(Func): function = 'dictGet' template = '%(function)s("%(dict_name)s", (%(fields)s), %(expressions)s)' diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 0141191b9e..dd3a6ca6ac 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1068,6 +1068,7 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N if exclude_intervals or not hasattr(self.model, 'is_annotated_in_any_gene') or len(gene_intervals) < 100: # TODO real threshhold intervals = (gene_intervals or []) + (intervals or []) else: + locus_q = Q(geneId_ids__bitmap_has_any=gene_id_ids) if not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) From 64d022e086a8e49391b1621e88b2d3ae0022c84b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 13:57:27 -0400 Subject: [PATCH 090/109] fix finction call --- clickhouse_search/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f00a2feb37..df4998b144 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -510,7 +510,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): keys = KEY_LOOKUP_CLASS_MAP[genome_version][data_type].objects.filter(variant_id=variant_id).values_list('key', flat=True) entries = entry_cls.objects.filter(key__in=keys) else: - entries = entry_cls.objects.filter_intervals(variant_ids=[variant_id]) + entries = entry_cls.objects.filter_locus(variant_ids=[variant_id]) entries = entries.result_values(sample_data) results = annotations_cls.objects.subquery_join(entries) @@ -545,7 +545,7 @@ def _add_liftover_genotypes(variant, data_type, variant_id): ).values_list('key', flat=True) if not keys: return - lifted_entries = lifted_entry_cls.objects.filter_intervals(variant_ids=[variant_id]).filter(key=keys[0]) + lifted_entries = lifted_entry_cls.objects.filter_locus(variant_ids=[variant_id]).filter(key=keys[0]) lifted_entry_data = lifted_entries.values('key').annotate( familyGenotypes=GroupArrayArray(lifted_entry_cls.objects.genotype_expression()) ) From 64baf6b01dfe00c7c3dff4ce9f40a1279a9da0d6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 14:27:50 -0400 Subject: [PATCH 091/109] clean up --- seqr/utils/search/search_utils_tests.py | 12 +++++---- seqr/utils/search/utils.py | 33 +++++++++++++------------ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 746dde4d13..14f9e60b09 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -330,18 +330,20 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs) self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - gene_ids = intervals = None + gene_ids = intervals = gene_id_ids= None has_included_gene_search = has_gene_search and not exclude_locations if has_gene_search: gene_ids = ['ENSG00000186092', 'ENSG00000227232'] + gene_id_ids = [2, 7] intervals = [['2', 1234, 5678], ['7', 1, 11100], ['1', 14404, 29570], ['1', 65419, 71585]] if single_gene_search: gene_ids = gene_ids[1:] + gene_id_ids = gene_id_ids[:1] intervals = intervals[2:3] self._assert_expected_search_locus( mock_get_variants.call_args.args[1], dataset_type='MITO_missing' if has_included_gene_search else dataset_type, gene_ids=gene_ids, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, + parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, gene_id_ids=gene_id_ids, ) def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): @@ -560,7 +562,7 @@ def _assert_expected_search_locus(self, search_body, dataset_type, gene_ids=None if gene_ids: parsed_genes = search_body['parsed_locus']['genes'] for gene in parsed_genes.values(): - self.assertSetEqual(set(gene.keys()), GENE_FIELDS) + self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') if len(gene_ids) > 1: self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') @@ -718,7 +720,7 @@ def _assert_expected_cached_variants(self, variants, num_results): self.PARSED_CACHED_VARIANTS[:num_results], ) - def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): + def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, gene_id_ids=None, **kwargs): gene_ids = None if exclude_locations else gene_ids gene_intervals = None if gene_ids: @@ -726,7 +728,7 @@ def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, ex intervals = intervals[:2] if len(gene_ids) > 1 else None super()._assert_expected_search_locus( *args, gene_ids=gene_ids, gene_intervals=gene_intervals, intervals=intervals, - variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, + variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, gene_id_ids=gene_id_ids, **kwargs, ) @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 1d7a7a6619..e8bdc2487c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -619,33 +619,34 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, include_db_ids=True, **kwargs): +def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, **kwargs): intervals = [_format_interval(**interval) for interval in intervals or []] - parsed_locus = { - 'intervals': intervals or None, - 'gene_intervals': None, - 'exclude_intervals': exclude_locations, - 'gene_ids': None, - 'variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, - } + gene_intervals = gene_ids = gene_id_ids =None if genes: + gene_id_ids = [gene.pop('id') for gene in genes.values()] gene_intervals = sorted([ [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() ]) if exclude_locations: - parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + gene_intervals + intervals += gene_intervals + gene_intervals = None else: - parsed_locus['gene_ids'] = sorted(genes.keys()) - parsed_locus['gene_intervals'] = gene_intervals - if include_db_ids: - parsed_locus['gene_id_ids'] = [gene['id'] for gene in genes.values()] + gene_ids = sorted(genes.keys()) - return parsed_locus + return { + 'intervals': intervals or None, + 'gene_intervals': gene_intervals, + 'exclude_intervals': exclude_locations, + 'gene_ids': gene_ids, + 'gene_id_ids': gene_id_ids, + 'variant_ids': parsed_variant_ids, + 'rs_ids': rs_ids, + } def _parse_locus_intervals(*args, **kwargs): - parsed_locus = _parse_locus_gene_intervals(*args, include_db_ids=False, **kwargs) + parsed_locus = _parse_locus_gene_intervals(*args, **kwargs) + parsed_locus.pop('gene_id_ids') gene_intervals = parsed_locus.pop('gene_intervals') if gene_intervals: parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + gene_intervals From b2d24bad3b9b73aa69e41ebed87beb6c03279182 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 16:49:41 -0400 Subject: [PATCH 092/109] better interval representation --- clickhouse_search/managers.py | 17 ++++++++++++++--- seqr/utils/search/utils.py | 17 +++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index dd3a6ca6ac..2e30f0fc15 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1056,7 +1056,7 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N if exclude_intervals: intervals = None else: - chromosomes = {chrom for chrom, _, _ in (gene_intervals or []) + (intervals or [])} + chromosomes = {chrom for chrom, _, _ in list((gene_intervals or {}).values()) + (intervals or [])} intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] gene_intervals = None @@ -1066,11 +1066,22 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N locus_q = None if gene_intervals: if exclude_intervals or not hasattr(self.model, 'is_annotated_in_any_gene') or len(gene_intervals) < 100: # TODO real threshhold - intervals = (gene_intervals or []) + (intervals or []) + intervals = list((gene_intervals or {}).values()) + (intervals or []) else: - locus_q = Q(geneId_ids__bitmap_has_any=gene_id_ids) + locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) if not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) + # clustered_intervals = defaultdict(lambda: [MAX_POS, MIN_POS, []]) + # for gene_id, (chrom, start, end) in gene_intervals.items(): + # clustered_intervals[chrom][0] = min(clustered_intervals[chrom][0], start) + # clustered_intervals[chrom][1] = max(clustered_intervals[chrom][1], end) + # clustered_intervals[chrom][2].append(gene_id) + # for chrom, (start, end, gene_ids) in clustered_intervals.items(): + # chrom_q = self._interval_query(chrom, start, end) & Q(geneId_ids__bitmap_has_any=gene_ids) + # if locus_q is None: + # locus_q = chrom_q + # else: + # locus_q |= chrom_q if intervals: interval_q = self._interval_query(*intervals[0]) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index e8bdc2487c..612ec4c6e7 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -487,7 +487,7 @@ def _search_dataset_type(search): lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type - intervals = (locus['intervals'] or locus.get('gene_intervals')) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None + intervals = (locus['intervals'] or locus.get('gene_intervals', {}).values()) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None dataset_type = _annotation_dataset_type(search.get('annotations'), intervals, pathogenicity=search.get('pathogenicity')) secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], intervals) if search.get('annotations_secondary') else None @@ -621,14 +621,13 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, **kwargs): intervals = [_format_interval(**interval) for interval in intervals or []] - gene_intervals = gene_ids = gene_id_ids =None + gene_intervals = gene_ids = None if genes: - gene_id_ids = [gene.pop('id') for gene in genes.values()] - gene_intervals = sorted([ - [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() - ]) + gene_intervals = { + gene['id']: [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() + } if exclude_locations: - intervals += gene_intervals + intervals += sorted(gene_intervals.values()) gene_intervals = None else: gene_ids = sorted(genes.keys()) @@ -638,7 +637,6 @@ def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_i 'gene_intervals': gene_intervals, 'exclude_intervals': exclude_locations, 'gene_ids': gene_ids, - 'gene_id_ids': gene_id_ids, 'variant_ids': parsed_variant_ids, 'rs_ids': rs_ids, } @@ -646,10 +644,9 @@ def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_i def _parse_locus_intervals(*args, **kwargs): parsed_locus = _parse_locus_gene_intervals(*args, **kwargs) - parsed_locus.pop('gene_id_ids') gene_intervals = parsed_locus.pop('gene_intervals') if gene_intervals: - parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + gene_intervals + parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + sorted(gene_intervals.values()) return parsed_locus From 55931513903c223dad53b4b125d71f17de887612 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 17:16:19 -0400 Subject: [PATCH 093/109] clean up --- clickhouse_search/managers.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 2e30f0fc15..199d83b2d5 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -624,6 +624,7 @@ def has_annotation(self, field): class EntriesManager(SearchQuerySet): + MAX_XPOS_FILTER_INTERVALS = 500 GENOTYPE_LOOKUP = { REF_REF: [0], REF_ALT: [1], @@ -1040,7 +1041,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_id_ids=None, variant_ids=None, padded_interval=None, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, variant_ids=None, padded_interval=None, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, @@ -1065,23 +1066,13 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N locus_q = None if gene_intervals: - if exclude_intervals or not hasattr(self.model, 'is_annotated_in_any_gene') or len(gene_intervals) < 100: # TODO real threshhold + has_entry_genes = hasattr(self.model, 'is_annotated_in_any_gene') + if has_entry_genes and not intervals: + entries = entries.filter(is_annotated_in_any_gene=Value(True)) + if (not has_entry_genes) or exclude_intervals or len(gene_intervals) < self.MAX_XPOS_FILTER_INTERVALS: intervals = list((gene_intervals or {}).values()) + (intervals or []) else: locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) - if not intervals: - entries = entries.filter(is_annotated_in_any_gene=Value(True)) - # clustered_intervals = defaultdict(lambda: [MAX_POS, MIN_POS, []]) - # for gene_id, (chrom, start, end) in gene_intervals.items(): - # clustered_intervals[chrom][0] = min(clustered_intervals[chrom][0], start) - # clustered_intervals[chrom][1] = max(clustered_intervals[chrom][1], end) - # clustered_intervals[chrom][2].append(gene_id) - # for chrom, (start, end, gene_ids) in clustered_intervals.items(): - # chrom_q = self._interval_query(chrom, start, end) & Q(geneId_ids__bitmap_has_any=gene_ids) - # if locus_q is None: - # locus_q = chrom_q - # else: - # locus_q |= chrom_q if intervals: interval_q = self._interval_query(*intervals[0]) From 9b9d8c17562109fec0ecbd92aff999558463c211 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 17:21:34 -0400 Subject: [PATCH 094/109] fix tests --- seqr/utils/search/search_utils_tests.py | 15 ++++++++------- seqr/utils/search/utils.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 14f9e60b09..6bfacf5cfc 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -330,20 +330,18 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs) self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - gene_ids = intervals = gene_id_ids= None + gene_ids = intervals = None has_included_gene_search = has_gene_search and not exclude_locations if has_gene_search: gene_ids = ['ENSG00000186092', 'ENSG00000227232'] - gene_id_ids = [2, 7] intervals = [['2', 1234, 5678], ['7', 1, 11100], ['1', 14404, 29570], ['1', 65419, 71585]] if single_gene_search: gene_ids = gene_ids[1:] - gene_id_ids = gene_id_ids[:1] intervals = intervals[2:3] self._assert_expected_search_locus( mock_get_variants.call_args.args[1], dataset_type='MITO_missing' if has_included_gene_search else dataset_type, gene_ids=gene_ids, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, gene_id_ids=gene_id_ids, + parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, ) def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): @@ -720,15 +718,18 @@ def _assert_expected_cached_variants(self, variants, num_results): self.PARSED_CACHED_VARIANTS[:num_results], ) - def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, gene_id_ids=None, **kwargs): + def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): gene_ids = None if exclude_locations else gene_ids gene_intervals = None if gene_ids: - gene_intervals = intervals[2:] if len(gene_ids) > 1 else intervals + if len(gene_ids) > 1: + gene_intervals = {2: intervals[2], 7: intervals[3]} + else: + gene_intervals = {2: intervals[0]} intervals = intervals[:2] if len(gene_ids) > 1 else None super()._assert_expected_search_locus( *args, gene_ids=gene_ids, gene_intervals=gene_intervals, intervals=intervals, - variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, gene_id_ids=gene_id_ids, **kwargs, + variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, ) @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 612ec4c6e7..355b098eef 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -487,7 +487,7 @@ def _search_dataset_type(search): lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type - intervals = (locus['intervals'] or locus.get('gene_intervals', {}).values()) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None + intervals = (locus['intervals'] or (locus.get('gene_intervals') or {}).values()) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None dataset_type = _annotation_dataset_type(search.get('annotations'), intervals, pathogenicity=search.get('pathogenicity')) secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], intervals) if search.get('annotations_secondary') else None From 3952e55c0be5bceaf2dd0ebf34619f8dca5b528d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 Aug 2025 17:38:06 -0400 Subject: [PATCH 095/109] show error when results are truncated --- clickhouse_search/search.py | 13 ++++++++++--- clickhouse_search/search_tests.py | 5 +++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f00a2feb37..1807cb6355 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -51,10 +51,10 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno dataset_results = [] if inheritance_mode != COMPOUND_HET: result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) - dataset_results += list(result_q[:MAX_VARIANTS + 1]) + dataset_results += _evaluate_results(result_q) if has_comp_het: result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=skip_individual_guid, **search) - dataset_results += [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + dataset_results += _evaluate_results(result_q, is_comp_het=True) if skip_individual_guid: _add_individual_guids(dataset_results, sample_data) @@ -79,6 +79,13 @@ def _get_search_results_queryset(entry_cls, annotations_cls, sample_data, **sear return results.result_values() +def _evaluate_results(result_q, is_comp_het=False): + results = [list(result[1:]) if is_comp_het else result for result in result_q[:MAX_VARIANTS + 1]] + if len(results) > MAX_VARIANTS: + from seqr.utils.search.utils import InvalidSearchException + raise InvalidSearchException('This search returned too many results') + return results + def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { @@ -120,7 +127,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations) result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) - dataset_results = [list(result[1:]) for result in result_q[:MAX_VARIANTS + 1]] + dataset_results = _evaluate_results(result_q, is_comp_het=True) if skip_individual_guid: _add_individual_guids(dataset_results, sv_sample_data) results += dataset_results diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8042d2e3fc..2d5a8b5f93 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -643,7 +643,12 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) + @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): + with self.assertRaises(InvalidSearchException) as cm: + self._assert_expected_search([]) + self.assertEqual(str(cm.exception),'This search returned too many results') + Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') self._set_multi_project_search() with self.assertRaises(InvalidSearchException) as cm: From 868424e3abc7edb7767ec1c9518742a12f933b28 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 11:57:03 -0400 Subject: [PATCH 096/109] include user-readable family id in discovery variant error --- seqr/views/apis/report_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index b85f2d852a..4d748d14ab 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -121,6 +121,7 @@ def anvil_export(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) parsed_rows = defaultdict(list) + family_id_map = {} family_diseases = {} def _add_row(row, family_id, row_type): @@ -130,7 +131,7 @@ def _add_row(row, family_id, row_type): if not (discovery_row.get(GENE_COLUMN) or discovery_row.get('sv_type'))] if missing_gene_rows: raise ErrorsWarningsException( - [f'Discovery variant(s) {", ".join(sorted(missing_gene_rows))} in family {family_id} have no associated gene']) + [f'Discovery variant(s) {", ".join(sorted(missing_gene_rows))} in family {family_id_map[family_id]} have no associated gene']) parsed_rows[row_type] += [{ 'entity:discovery_id': f'{discovery_row["chrom"]}_{discovery_row["pos"]}_{discovery_row["participant_id"]}', **{k: str(discovery_row.get(k.lower()) or '') for k in ['Zygosity', 'Chrom', 'Pos', 'Ref', 'Alt', 'Transcript']}, @@ -164,6 +165,7 @@ def _add_row(row, family_id, row_type): 'disease_id': row.get('condition_id', '').replace('|', ';'), 'disease_description': row.get('known_condition_name', '').replace('|', ';'), } + family_id_map[family_id] = row[id_field] parsed_rows[row_type].append(row) max_loaded_date = request.GET.get('loadedBefore') or (datetime.now() - timedelta(days=365)).strftime('%Y-%m-%d') From d1d8c1f0055c609a34d959563eb1fba124a903e0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:08:10 -0400 Subject: [PATCH 097/109] update tests --- seqr/fixtures/1kg_project.json | 2 +- .../tests/check_for_new_samples_from_pipeline_tests.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 6 +++--- seqr/views/apis/family_api_tests.py | 2 +- seqr/views/apis/report_api_tests.py | 2 +- seqr/views/apis/summary_data_api_tests.py | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index a6e9ec55f3..4a9e17b50b 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -375,7 +375,7 @@ "created_by": null, "last_modified_date": "2017-03-12T22:37:17.555Z", "project": 4, - "family_id": "14", + "family_id": "fam14", "analysis_status": "Rncc", "success_story": "Differential treatement", "success_story_types": ["A", "D"] diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 550fdc393c..d46c613e38 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -679,7 +679,7 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): """Encountered the following errors loading Non-Analyst Project: The following 1 families failed sex check: -- 14: Sample NA21987 has pedigree sex M but imputed sex F""", +- fam14: Sample NA21987 has pedigree sex M but imputed sex F""", ), mock.call( 'seqr-data-loading', diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 50a96415fd..33a6b9bc60 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1495,8 +1495,8 @@ def _has_expected_ped_files(self, mock_open, mock_mkdir, dataset_type, sample_ty self.assertEqual(len(file), 3) self.assertListEqual(file, [ pedigree_header, - ['R0004_non_analyst_project', 'F000014_14', '14', 'NA21234', '', '', 'F'] + (['ABC123'] if has_remap else []), - ['R0004_non_analyst_project', 'F000014_14', '14', 'NA21987', '', '', 'M'] + ([''] if has_remap else []), + ['R0004_non_analyst_project', 'F000014_14', 'fam14', 'NA21234', '', '', 'F'] + (['ABC123'] if has_remap else []), + ['R0004_non_analyst_project', 'F000014_14', 'fam14', 'NA21987', '', '', 'M'] + ([''] if has_remap else []), ]) def _test_load_single_project(self, mock_open, mock_mkdir, response, *args, **kwargs): @@ -1919,7 +1919,7 @@ def _trigger_error(self, url, body, dag_json, mock_open, mock_mkdir): 'warnings': None, 'errors': [ 'The following samples are included in airtable but are missing from the VCF: NA21987', - 'The following families have previously loaded samples absent from airtable\nFamily 14: NA21234, NA21654', + 'The following families have previously loaded samples absent from airtable\nFamily fam14: NA21234, NA21654', ], }) self.assertEqual(len(responses.calls), 2) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index eb5450060e..17a1dd9a7e 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -459,7 +459,7 @@ def test_update_family_fields(self): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertEqual(response_json['F000014_14']['description'], 'Updated description') - expected_id = 'new_id' if self._anvil_enabled() else '14' + expected_id = 'new_id' if self._anvil_enabled() else 'fam14' self.assertEqual(response_json['F000014_14'][FAMILY_ID_FIELD], expected_id) self.assertEqual(response_json['F000014_14']['displayName'], expected_id) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 86350c7a2b..e8c67004d1 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -784,7 +784,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur response = self.client.get(no_analyst_project_url) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['errors'], - ['Discovery variant(s) 1-248367227-TC-T, MT-14783-T-C in family 14 have no associated gene']) + ['Discovery variant(s) 1-248367227-TC-T, MT-14783-T-C in family fam14 have no associated gene']) @mock.patch('seqr.views.apis.report_api.GREGOR_DATA_MODEL_URL', MOCK_DATA_MODEL_URL) @mock.patch('seqr.views.apis.report_api.datetime') diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index d3011e4d03..a917d78829 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -129,8 +129,8 @@ EXPECTED_NO_GENE_SAMPLE_METADATA_ROW = { 'participant_id': 'NA21234', 'familyGuid': 'F000014_14', - 'family_id': '14', - 'displayName': '14', + 'family_id': 'fam14', + 'displayName': 'fam14', 'projectGuid': 'R0004_non_analyst_project', 'internal_project_id': 'Non-Analyst Project', 'affected_status': 'Affected', From 323f2a2cbf13174242ba77eaee1ca2b8845639ec Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:16:54 -0400 Subject: [PATCH 098/109] add build support info to faq --- ui/pages/Public/components/Faq.jsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ui/pages/Public/components/Faq.jsx b/ui/pages/Public/components/Faq.jsx index b2d0aec7b0..a9d276571d 100644 --- a/ui/pages/Public/components/Faq.jsx +++ b/ui/pages/Public/components/Faq.jsx @@ -68,6 +68,12 @@ const FAQS = [ seqr is not designed for cohort or gene burden analyses. You can search for variants in a candidate gene across your data but seqr will not provide a quantification of how much variation you should expect to see. + + seqr does not support data aligned to any reference genome other than GRCh37 and GRCh38. While both of these + builds are available, GRCh38 is recommended whenever possible as it includes the most up-to-date + annotations. Additionally, seqr does not support loading data from different builds into the same project, + or searching across multiple projects that have different builds. + seqr is not an annotation pipeline for a VCF. While annotations are added when data is loaded in seqr, you cannot output the annotated VCF from seqr. @@ -88,6 +94,12 @@ const FAQS = [ candidato a través de sus datos, pero seqr no proporcionará una cuantificación de cuánta variación debe esperar ver. + + seqr no admite datos alineados con ningún genoma de referencia que no sean GRCh37 y GRCh38. Si bien ambas + compilaciones están disponibles, se recomienda GRCh38 siempre que sea posible, ya que incluye las + anotaciones más actualizadas. Además, seqr no permite cargar datos de diferentes compilaciones en el mismo + proyecto ni realizar búsquedas en varios proyectos con compilaciones diferentes. + seqr no es un canal de anotación para un VCF. Aunque las anotaciones se agregan cuando los datos se cargan en seqr, no se puede generar el VCF anotado de seqr. From fd5032995f99c47b6f78c6d56acbddc29ab55c79 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:27:43 -0400 Subject: [PATCH 099/109] fix tests --- .../management/tests/set_saved_variant_key_tests.py | 2 +- matchmaker/views/external_api_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 5f2488c68d..be47bf5bb5 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -115,7 +115,7 @@ def test_command(self, mock_subprocess): ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), ('Found 0 keys', None), ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), - ('1 remaining variants: M-14783-T-C - 14', None), + ('1 remaining variants: M-14783-T-C - fam14', None), ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), ('Found 0 keys', None), ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), diff --git a/matchmaker/views/external_api_tests.py b/matchmaker/views/external_api_tests.py index f486a8e091..d1117ce86e 100644 --- a/matchmaker/views/external_api_tests.py +++ b/matchmaker/views/external_api_tests.py @@ -253,7 +253,7 @@ def test_mme_match_proxy(self, mock_post_to_slack, mock_email, mock_logger): be found found at https://seqr.broadinstitute.org/matchmaker/disclaimer.""" match1 = 'seqr ID NA19675_1 from project 1kg project n\u00e5me with uni\u00e7\u00f8de in family 1 inserted into matchbox on May 23, 2018, with seqr link /project/R0001_1kg/family_page/F000001_1/matchmaker_exchange' match2 = 'seqr ID NA20888 from project Test Reprocessed Project in family 12 inserted into matchbox on Feb 05, 2019, with seqr link /project/R0003_test/family_page/F000012_12/matchmaker_exchange' - match3 = 'seqr ID NA21234 from project Non-Analyst Project in family 14 inserted into matchbox on Feb 05, 2019, with seqr link /project/R0004_non_analyst_project/family_page/F000014_14/matchmaker_exchange' + match3 = 'seqr ID NA21234 from project Non-Analyst Project in family fam14 inserted into matchbox on Feb 05, 2019, with seqr link /project/R0004_non_analyst_project/family_page/F000014_14/matchmaker_exchange' mock_post_to_slack.assert_called_with('matchmaker_matches', message_template.format( matches='\n'.join([match1, match2, match3]), From e3427db9a42a1c6224c30da2af824c0adb50a211 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:14:06 -0400 Subject: [PATCH 100/109] backend specific project data deletion --- .../commands/deactivate_project_search.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index ea758a3ed2..59ff8ac4f2 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -1,6 +1,9 @@ from django.core.management.base import BaseCommand, CommandError +from sqlalchemy.util import raise_ +from clickhouse_search.search import delete_clickhouse_project from seqr.models import Project, Sample +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.airflow_utils import is_airflow_enabled, trigger_airflow_dag, DELETE_PROJECTS_DAG_NAME import logging @@ -24,5 +27,11 @@ def handle(self, *args, **options): if updated and is_airflow_enabled(): dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', flat=True).distinct() for dataset_type in dataset_types: - trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') - logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') + backend_specific_call( + lambda: True, self._trigger_delete_dag, delete_clickhouse_project, + )(project, dataset_type) + + @staticmethod + def _trigger_delete_dag(project, dataset_type): + trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') + logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') From 7bd50ce8249e75f6ed1f2f2868326d9891c8e141 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:46:13 -0400 Subject: [PATCH 101/109] test all backends --- .../commands/deactivate_project_search.py | 15 ++-- .../tests/deactivate_project_search_tests.py | 83 +++++++++++++++---- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 59ff8ac4f2..63fbdf8078 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -1,5 +1,4 @@ from django.core.management.base import BaseCommand, CommandError -from sqlalchemy.util import raise_ from clickhouse_search.search import delete_clickhouse_project from seqr.models import Project, Sample @@ -24,14 +23,20 @@ def handle(self, *args, **options): logger.info(f'Deactivated {len(updated)} samples') - if updated and is_airflow_enabled(): + if updated: dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', flat=True).distinct() for dataset_type in dataset_types: backend_specific_call( - lambda: True, self._trigger_delete_dag, delete_clickhouse_project, + lambda *args, **kwargs: True, self._trigger_delete_dag, self._delete_clickhouse_project, )(project, dataset_type) @staticmethod def _trigger_delete_dag(project, dataset_type): - trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') - logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') + if is_airflow_enabled(): + trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') + logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') + + @staticmethod + def _delete_clickhouse_project(project, dataset_type): + info = delete_clickhouse_project(project, dataset_type) + logger.info(info) diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index 2b97314b92..e8906f71ac 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -5,22 +5,19 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.management import call_command from django.core.management.base import CommandError -from seqr.models import Sample -from seqr.views.utils.test_utils import AirflowTestCase + +from clickhouse_search.models import EntriesSnvIndel, EntriesMito, EntriesSv, ProjectGtStatsSnvIndel, \ + ProjectGtStatsMito, ProjectGtStatsSv, AnnotationsSnvIndel, AnnotationsMito, AnnotationsSv +from seqr.models import Sample, Project +from seqr.views.utils.test_utils import AirflowTestCase, AnvilAuthenticationTestCase, AuthenticationTestCase PROJECT_GUID = 'R0001_1kg' VARIANT_ID = '21-3343353-GAGA-G' -class DeactivateProjectSearchTest(AirflowTestCase): - fixtures = ['users', '1kg_project'] +class DeactivateProjectSearchTest(object): - DAG_NAME = 'DELETE_PROJECTS' - DAG_VARIABLES = { - 'projects_to_run': [PROJECT_GUID], - 'dataset_type': 'SNV_INDEL', - 'reference_genome': 'GRCh37', - } + DELETE_SUCCESS_LOGS = [] @responses.activate @mock.patch('seqr.management.commands.deactivate_project_search.input') @@ -37,6 +34,7 @@ def test_command(self, mock_input): self.assertEqual(str(e.exception), 'Error: user did not confirm') # Test success + Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') mock_input.return_value = 'y' self.reset_logs() self.maxDiff = None @@ -49,23 +47,78 @@ def test_command(self, mock_input): 'updateType': 'bulk_update'}, }), ('Deactivated 11 samples', None), - ('Successfully triggered DELETE_PROJECTS DAG for MITO R0001_1kg', None), - ('Successfully triggered DELETE_PROJECTS DAG for SV R0001_1kg', None), - ('Successfully triggered DELETE_PROJECTS DAG for SNV_INDEL R0001_1kg', None), - ]) + ] + self.DELETE_SUCCESS_LOGS) active_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True) self.assertEqual(active_samples.count(), 0) - self.assert_airflow_calls(self.DAG_VARIABLES, 5) + self._assert_expected_delete() # Re-running has no effect self.reset_logs() call_command('deactivate_project_search', PROJECT_GUID) self.assert_json_logs(user=None, expected=[('Deactivated 0 samples', None)]) + def _assert_expected_delete(self): + pass + +class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest): + fixtures = ['users', '1kg_project'] + + +class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase, DeactivateProjectSearchTest): + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] + + DELETE_SUCCESS_LOGS = [ + ('Deleted all MITO search data for project 1kg project nåme with uniçøde', None), + ('Deleted all SV search data for project 1kg project nåme with uniçøde', None), + ('Deleted all SNV_INDEL search data for project 1kg project nåme with uniçøde', None), + ] + + def _assert_expected_delete(self): + self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(EntriesMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(ProjectGtStatsMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(EntriesSv.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(ProjectGtStatsSv.objects.filter(project_guid=PROJECT_GUID).count(), 0) + + updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) + self.assertDictEqual(updated_seqr_pops_by_key, { + 1: (2, 2, 1, 1), + 2: (1, 1, 0, 0), + 3: (0, 0, 0, 0), + 4: (0, 0, 0, 0), + 5: (1, 1, 0, 0), + 6: (0, 0, 0, 0), + 22: (0, 3, 0, 1), + }) + + +class HailBackendDeactivateProjectSearchTest(AirflowTestCase, DeactivateProjectSearchTest): + fixtures = ['users', '1kg_project'] + + CLICKHOUSE_HOSTNAME = '' + DAG_NAME = 'DELETE_PROJECTS' + DAG_VARIABLES = { + 'projects_to_run': [PROJECT_GUID], + 'dataset_type': 'SNV_INDEL', + 'reference_genome': 'GRCh38', + } + + DELETE_SUCCESS_LOGS = [ + ('Successfully triggered DELETE_PROJECTS DAG for MITO R0001_1kg', None), + ('Successfully triggered DELETE_PROJECTS DAG for SV R0001_1kg', None), + ('Successfully triggered DELETE_PROJECTS DAG for SNV_INDEL R0001_1kg', None), + ] + + def _assert_expected_delete(self): + self.assert_airflow_calls(self.DAG_VARIABLES, 5) + def _add_update_check_dag_responses(self, **kwargs): return self._add_check_dag_variable_responses(self.DAG_VARIABLES, **kwargs) def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): variables_update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' super()._assert_update_check_airflow_calls(call_count, offset, variables_update_check_path) + + From ab01339e68660868494ef4dd3006e73a827eefa1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:56:23 -0400 Subject: [PATCH 102/109] correctly handle gcv --- clickhouse_search/search.py | 11 ++++++----- .../commands/deactivate_project_search.py | 12 +++++++----- .../tests/deactivate_project_search_tests.py | 18 ++++++++++++------ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 678b554184..f1e82f99b8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -626,9 +626,10 @@ def delete_clickhouse_project(project, dataset_type=None, **kwargs): table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) - cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid]) - view_name = f'{table_base}/project_gt_stats_to_gt_stats_mv' - cursor.execute(f'SYSTEM REFRESH VIEW "{view_name}"') - cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') - cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') + if dataset_type != 'GCNV': + cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid]) + view_name = f'{table_base}/project_gt_stats_to_gt_stats_mv' + cursor.execute(f'SYSTEM REFRESH VIEW "{view_name}"') + cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') + cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') return f'Deleted all {dataset_type} search data for project {project.name}' diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 63fbdf8078..57088c089e 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -24,19 +24,21 @@ def handle(self, *args, **options): logger.info(f'Deactivated {len(updated)} samples') if updated: - dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', flat=True).distinct() - for dataset_type in dataset_types: + dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() + for dataset_type, sample_type in dataset_types: backend_specific_call( lambda *args, **kwargs: True, self._trigger_delete_dag, self._delete_clickhouse_project, - )(project, dataset_type) + )(project, dataset_type, sample_type) @staticmethod - def _trigger_delete_dag(project, dataset_type): + def _trigger_delete_dag(project, dataset_type, sample_type): if is_airflow_enabled(): trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') @staticmethod - def _delete_clickhouse_project(project, dataset_type): + def _delete_clickhouse_project(project, dataset_type, sample_type): + if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: + dataset_type = 'GCNV' info = delete_clickhouse_project(project, dataset_type) logger.info(info) diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index e8906f71ac..3a32a408e0 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -6,8 +6,8 @@ from django.core.management import call_command from django.core.management.base import CommandError -from clickhouse_search.models import EntriesSnvIndel, EntriesMito, EntriesSv, ProjectGtStatsSnvIndel, \ - ProjectGtStatsMito, ProjectGtStatsSv, AnnotationsSnvIndel, AnnotationsMito, AnnotationsSv +from clickhouse_search.models import EntriesSnvIndel, EntriesMito, EntriesGcnv, ProjectGtStatsSnvIndel, \ + ProjectGtStatsMito, AnnotationsSnvIndel, AnnotationsMito from seqr.models import Sample, Project from seqr.views.utils.test_utils import AirflowTestCase, AnvilAuthenticationTestCase, AuthenticationTestCase @@ -70,8 +70,8 @@ class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase, Deactiv DELETE_SUCCESS_LOGS = [ ('Deleted all MITO search data for project 1kg project nåme with uniçøde', None), - ('Deleted all SV search data for project 1kg project nåme with uniçøde', None), ('Deleted all SNV_INDEL search data for project 1kg project nåme with uniçøde', None), + ('Deleted all GCNV search data for project 1kg project nåme with uniçøde', None), ] def _assert_expected_delete(self): @@ -79,8 +79,7 @@ def _assert_expected_delete(self): self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) self.assertEqual(EntriesMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) self.assertEqual(ProjectGtStatsMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(EntriesSv.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(ProjectGtStatsSv.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(EntriesGcnv.objects.filter(project_guid=PROJECT_GUID).count(), 0) updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) self.assertDictEqual(updated_seqr_pops_by_key, { @@ -93,6 +92,13 @@ def _assert_expected_delete(self): 22: (0, 3, 0, 1), }) + updated_mito_pops_by_key = dict(AnnotationsMito.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) + self.assertDictEqual(updated_mito_pops_by_key, { + 6: (0, 0, 0, 0), + 7: (0, 0, 0, 0), + 8: (0, 0, 0, 0), + }) + class HailBackendDeactivateProjectSearchTest(AirflowTestCase, DeactivateProjectSearchTest): fixtures = ['users', '1kg_project'] @@ -107,8 +113,8 @@ class HailBackendDeactivateProjectSearchTest(AirflowTestCase, DeactivateProjectS DELETE_SUCCESS_LOGS = [ ('Successfully triggered DELETE_PROJECTS DAG for MITO R0001_1kg', None), - ('Successfully triggered DELETE_PROJECTS DAG for SV R0001_1kg', None), ('Successfully triggered DELETE_PROJECTS DAG for SNV_INDEL R0001_1kg', None), + ('Successfully triggered DELETE_PROJECTS DAG for SV R0001_1kg', None), ] def _assert_expected_delete(self): From 9549f3e0cbefcdf6606d3886a08c646a3097e3a8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:05:16 -0400 Subject: [PATCH 103/109] backend specific transfer families --- .../commands/transfer_families_to_different_project.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 80bd0fdb2a..27a44c6a85 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -12,6 +12,7 @@ def _disable_search(families, from_project): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) + updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() updated_family_dataset_types = list(search_samples.values_list('dataset_type', 'individual__family__guid').distinct()) @@ -20,9 +21,10 @@ def _disable_search(families, from_project): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - _trigger_delete_families_dags(from_project, updated_family_dataset_types) + return updated_family_dataset_types -def _trigger_delete_families_dags(from_project, updated_family_dataset_types): +def _disable_search_trigger_delete_families_dags(families, from_project): + updated_family_dataset_types = _disable_search(families, from_project) updated_families_by_dataset_type = defaultdict(list) for dataset_type, family_guid in updated_family_dataset_types: updated_families_by_dataset_type[dataset_type].append(family_guid) @@ -62,7 +64,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search_trigger_delete_families_dags, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) From b77eb4069b1e88e5ed3a1344d014e84615621155 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:28:07 -0400 Subject: [PATCH 104/109] test transfer families all backends --- ...fer_families_to_different_project_tests.py | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 634cc6b6f7..b4a93735d4 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -1,22 +1,25 @@ import responses from django.core.management import call_command -import mock from seqr.models import Family, VariantTagType, VariantTag, Sample -from seqr.views.utils.test_utils import AirflowTestCase, AuthenticationTestCase +from seqr.views.utils.test_utils import AirflowTestCase, AuthenticationTestCase, AnvilAuthenticationTestCase class TransferFamiliesTest(object): - def _test_command(self, additional_family, logs): + DEACTIVATE_SEARCH = True + LOGS = [] + + def test_command(self): call_command( - 'transfer_families_to_different_project', '--from-project=R0001_1kg', '--to-project=R0003_test', additional_family, '2', '5', + 'transfer_families_to_different_project', '--from-project=R0001_1kg', '--to-project=R0003_test', '2', '4', '5', '12', ) + self.maxDiff = None self.assert_json_logs(user=None, expected=[ - logs[0], + ('Found 3 out of 4 families. No match for: 12.', None), ('Skipping 1 families with analysis groups in the project: 5 (Test Group 1)', None), - *logs[1:], + *self.LOGS, ('Updating "Excluded" tags', None), ('Updating families', None), ('Done.', None), @@ -36,23 +39,46 @@ def _test_command(self, additional_family, logs): self.assertEqual(len(new_tags), 1) self.assertEqual(new_tags[0].saved_variants.first().family, family) - return family + samples = Sample.objects.filter(individual__family=family) + self.assertEqual(samples.count(), 7) + self.assertEqual(samples.filter(is_active=True).count(), 0 if self.DEACTIVATE_SEARCH else 7) + + family = Family.objects.get(family_id='4') + self.assertEqual(family.project.guid, 'R0003_test') + self.assertEqual(family.individual_set.count(), 1) class TransferFamiliesLocalTest(TransferFamiliesTest, AuthenticationTestCase): fixtures = ['users', '1kg_project'] + DEACTIVATE_SEARCH = False - def test_es_command(self): - self._test_command( - additional_family='12', logs=[('Found 2 out of 3 families. No match for: 12.', None)] - ) + +class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTestCase): + fixtures = ['users', '1kg_project'] + + ES_HOSTNAME = '' + LOGS = [ + ('Disabled search for 7 samples in the following 1 families: 2', None), + ] class TransferFamiliesAirflowTest(TransferFamiliesTest, AirflowTestCase): fixtures = ['users', '1kg_project'] PROJECT_GUID = 'R0001_1kg' # from-project DAG_NAME = 'DELETE_FAMILIES' + ES_HOSTNAME = '' + CLICKHOUSE_HOSTNAME = '' + + LOGS = [ + ('Disabled search for 7 samples in the following 1 families: 2', None), + ('Successfully triggered DELETE_FAMILIES DAG for 1 MITO families', None), + ('Successfully triggered DELETE_FAMILIES DAG for 1 SNV_INDEL families', None), + ('400 Client Error: Bad Request for url: http://testairflowserver/api/v1/variables/DELETE_FAMILIES', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }) + ] def setUp(self): super().setUp() @@ -89,25 +115,6 @@ def _assert_update_check_airflow_calls(self, call_count, offset, update_check_pa super()._assert_update_check_airflow_calls(call_count, offset, variables_update_check_path) @responses.activate - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', '') - def test_hail_backend_command(self): - searchable_family = self._test_command(additional_family='4', logs=[ - ('Found 3 out of 3 families.', None), - ('Disabled search for 7 samples in the following 1 families: 2', None), - ('Successfully triggered DELETE_FAMILIES DAG for 1 MITO families', None), - ('Successfully triggered DELETE_FAMILIES DAG for 1 SNV_INDEL families', None), - ('400 Client Error: Bad Request for url: http://testairflowserver/api/v1/variables/DELETE_FAMILIES', { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }) - ]) - - samples = Sample.objects.filter(individual__family=searchable_family) - self.assertEqual(samples.count(), 7) - self.assertEqual(samples.filter(is_active=True).count(), 0) - - family = Family.objects.get(family_id='4') - self.assertEqual(family.project.guid, 'R0003_test') - self.assertEqual(family.individual_set.count(), 1) - + def test_command(self): + super().test_command() self.assert_airflow_delete_families_calls() From c10e8036dd515396d3d1a3ba51b1092c846813c9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:46:37 -0400 Subject: [PATCH 105/109] fix flappy test --- seqr/management/commands/reload_saved_variant_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 5b38509d02..1f76eb2fa4 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -32,6 +32,6 @@ def handle(self, *args, **options): logging.info("Processing all %s projects" % len(projects)) family_ids = [family_guid] if family_guid else None - project_list = [(*project, family_ids) for project in projects.values_list('id', 'guid', 'name', 'genome_version')] + project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] update_projects_saved_variant_json(project_list, user_email='manage_command') logger.info("Done") From f31fcf92aadab0e5d304ffd72fe092ee3af69aa0 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Thu, 21 Aug 2025 17:32:04 -0400 Subject: [PATCH 106/109] chore: add handling for clinvar edge case (#4953) * add handling for clinvar edge case * ws * try values_list * tearDown * Update reload_clinvar_all_variants_tests.py * suggestion --- .../commands/reload_clinvar_all_variants.py | 11 +++++++---- .../tests/reload_clinvar_all_variants_tests.py | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/seqr/management/commands/reload_clinvar_all_variants.py b/seqr/management/commands/reload_clinvar_all_variants.py index ab9ddce7a7..3b05a63883 100644 --- a/seqr/management/commands/reload_clinvar_all_variants.py +++ b/seqr/management/commands/reload_clinvar_all_variants.py @@ -260,10 +260,13 @@ def handle(self, *args, **options): return logger.info(f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') # Drop any currently existing variants in the table that may exist due to a - # previously failed partial run. - clinvar_run_sql( - Template(f"ALTER TABLE `$reference_genome/$dataset_type/clinvar_all_variants` DROP PARTITION '{new_version}';") - ) + # previously failed partial run. Note that we validate that the Postgresql existing version + # is present in ClickHouse to account for the situation where Postgresql has an incorrect + # version. + if existing_version_obj and ClinvarAllVariantsSnvIndel.objects.filter(version=existing_version_obj.version).exists(): + clinvar_run_sql( + Template(f"ALTER TABLE `$reference_genome/$dataset_type/clinvar_all_variants` DROP PARTITION '{new_version}';") + ) # Handle parsing variants if event == 'end' and elem.tag == 'VariationArchive' and new_version: diff --git a/seqr/management/tests/reload_clinvar_all_variants_tests.py b/seqr/management/tests/reload_clinvar_all_variants_tests.py index 9ffa96ae02..c1056d7d44 100644 --- a/seqr/management/tests/reload_clinvar_all_variants_tests.py +++ b/seqr/management/tests/reload_clinvar_all_variants_tests.py @@ -14,7 +14,6 @@ ) from reference_data.models import DataVersions from seqr.management.commands.reload_clinvar_all_variants import BATCH_SIZE, WEEKLY_XML_RELEASE -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin WEEKLY_XML_RELEASE_HEADER = '''''' WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' @@ -83,7 +82,7 @@ @mock.patch('seqr.management.commands.reload_clinvar_all_variants.safe_post_to_slack') @mock.patch('seqr.management.commands.reload_clinvar_all_variants.logger.info') -class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): +class ReloadClinvarAllVariantsTest(TestCase): databases = '__all__' fixtures = ['clinvar_all_variants'] From d5855754a9109f474401487c233050b3dd0b219d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 11:52:54 -0400 Subject: [PATCH 107/109] use clickhosue fixture data in lookup test --- seqr/views/apis/variant_search_api_tests.py | 38 +++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 721d156826..4e0bf7eb69 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -7,7 +7,7 @@ from django.urls.base import reverse from elasticsearch.exceptions import ConnectionTimeout, TransportError -from hail_search.test_utils import HAIL_BACKEND_SINGLE_FAMILY_VARIANTS, VARIANT_LOOKUP_VARIANT, GCNV_VARIANT1, SV_VARIANT1 +from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_VARIANT1, SV_VARIANT1 from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch from seqr.utils.search.utils import InvalidSearchException from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException @@ -25,7 +25,9 @@ SEARCH = {'filters': {}, 'inheritance': None} PROJECT_FAMILIES = [{'projectGuid': PROJECT_GUID, 'familyGuids': ['F000001_1', 'F000002_2']}] -VARIANTS_WITH_DISCOVERY_TAGS = deepcopy(VARIANTS + HAIL_BACKEND_SINGLE_FAMILY_VARIANTS) +ALL_VARIANTS = VARIANTS + [VARIANT2, VARIANT3] + +VARIANTS_WITH_DISCOVERY_TAGS = deepcopy(ALL_VARIANTS) DISCOVERY_TAGS = [{ 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', @@ -87,7 +89,7 @@ ] EXPECTED_SEARCH_RESPONSE = { - 'searchedVariants': VARIANTS + HAIL_BACKEND_SINGLE_FAMILY_VARIANTS, + 'searchedVariants': ALL_VARIANTS, 'savedVariantsByGuid': { 'SV0000001_2103343353_r0390_100': expected_detail_saved_variant, 'SV0000002_1248367227_r0390_100': EXPECTED_SAVED_VARIANT, @@ -260,7 +262,7 @@ def _get_es_variants(results_model, **kwargs): results_model.save() - return deepcopy(VARIANTS + HAIL_BACKEND_SINGLE_FAMILY_VARIANTS), len(VARIANTS + HAIL_BACKEND_SINGLE_FAMILY_VARIANTS) + return deepcopy(ALL_VARIANTS), len(ALL_VARIANTS) def _get_empty_es_variants(results_model, **kwargs): @@ -529,15 +531,15 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '28', '0.29499998688697815', '0', - '0.28899794816970825', '0.24615199863910675', '20.899999618530273', '0.19699999690055847', - '2.000999927520752', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.295', '0', + '0.289', '0.24615', '20.9', '0.197', + '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1.0', 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '4', '0.0', '0.38041073083877563', '0.0', - '0.36268100142478943', '2.753999948501587', '', '1.378000020980835', '0.009999999776482582', '', '', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.0', '0.38041', '0.0', + '0.36268', '2.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', '', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.4594594594594595', 'HG00733', '1', '', '99', '0.4074074074074074'], + '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', '0.40741'], ] self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) @@ -563,16 +565,16 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '28', '0.29499998688697815', '0', - '0.28899794816970825', '0.24615199863910675', '20.899999618530273', '0.19699999690055847', - '2.000999927520752', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.295', '0', + '0.289', '0.24615', '20.9', '0.197', + '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', 'HG00731', '2', '', '99', '1.0', 'HG00732', '1', '', '99', '0.625', 'HG00733', '0', '', '40', '0.0'], - ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '4', '0.0', '0.38041073083877563', '0.0', - '0.36268100142478943', '2.753999948501587', '', '1.378000020980835', '0.009999999776482582', '', '', + ['1', '91502721', 'G', 'A', 'ENSG00000097046', 'intron_variant', '7', '0.0', '0.38041', '0.0', + '0.36268', '2.754', '', '1.378', '0.01', '', '', '', '', 'rs13447464', 'ENST00000234626.11:c.-63-251G>A', '', '', '', '2', '', '', 'HG00731', - '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.4594594594594595', 'HG00733', '1', '', '99', - '0.4074074074074074'], + '1', '', '99', '1.0', 'HG00732', '0', '', '99', '0.45946', 'HG00733', '1', '', '99', + '0.40741'], ] self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], expected_content) @@ -668,7 +670,7 @@ def _get_variants(results_model, **kwargs): results_model.save() self.assertSetEqual(expected_searched_families, {f.guid for f in results_model.families.all()}) matched_variants = [ - deepcopy(variant) for variant in VARIANTS + HAIL_BACKEND_SINGLE_FAMILY_VARIANTS + deepcopy(variant) for variant in ALL_VARIANTS if any(family_guid in expected_searched_families for family_guid in variant['familyGuids']) ] return matched_variants, len(matched_variants) From 82dfa4692a993bc9bdca554d4727d7a629eb3b69 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 12:22:46 -0400 Subject: [PATCH 108/109] properly handle multiple sample types in variant lookup --- seqr/views/apis/variant_search_api.py | 23 ++++++---- seqr/views/apis/variant_search_api_tests.py | 49 ++++++++++++++++----- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ed699a6649..c8d6781bce 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -592,31 +592,38 @@ def _update_lookup_variant(variant, response): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { - (i.pop('family__guid'), i.pop('individual_id')): i + (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) for i in Individual.objects.filter(family__guid__in=no_access_families).values( - 'family__guid', 'individual_id', 'affected', 'sex', 'features', + 'family__guid', 'individual_id', 'affected', 'sex', 'features', 'guid', vlmContactEmail=F('family__project__vlm_contact_email'), ) } - add_individual_hpo_details(individual_summary_map.values()) + add_individual_hpo_details([i for _, i in individual_summary_map.values()]) variant['genotypes'] = {} variant['lookupFamilyGuids'] = sorted(variant.pop('familyGuids')) variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: - variant['genotypes'].update({ - individual_guid_map[(family_guid, genotype['sampleId'])]: genotype - for genotype in variant['familyGenotypes'].pop(family_guid) - }) + for genotype in variant['familyGenotypes'].pop(family_guid): + individual_guid = individual_guid_map[(family_guid, genotype['sampleId'])] + if individual_guid in variant['genotypes']: + genotype = [variant['genotypes'][individual_guid], genotype] + variant['genotypes'][individual_guid] = genotype for i, (unmapped_family_guid, genotypes) in enumerate(variant.pop('familyGenotypes').items()): family_guid = f'F{i}_{variant["variantId"]}' variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): variant['liftedFamilyGuids'][variant['liftedFamilyGuids'].index(unmapped_family_guid)] = family_guid + individual_guid_map = {} for j, genotype in enumerate(genotypes): + unmapped_individual_guid, individual = individual_summary_map[(genotype.pop('familyGuid'), genotype.pop('sampleId'))] + if unmapped_individual_guid in individual_guid_map: + individual_guid = individual_guid_map[unmapped_individual_guid] + variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] + continue individual_guid = f'I{j}_{family_guid}' - individual = individual_summary_map[(genotype.pop('familyGuid'), genotype.pop('sampleId'))] + individual_guid_map[unmapped_individual_guid] = individual_guid feature_category_count = defaultdict(int) for feature in individual['features'] or []: feature_category_count[feature.get('category', 'Other')] += 1 diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 4e0bf7eb69..619ce40632 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -916,11 +916,23 @@ def test_variant_lookup(self, mock_variant_lookup): 'lookupFamilyGuids': ['F0_1-10439-AC-A', 'F1_1-10439-AC-A', 'F2_1-10439-AC-A'], 'liftedFamilyGuids': ['F2_1-10439-AC-A'], 'genotypes': { - 'I0_F0_1-10439-AC-A': {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - 'I1_F0_1-10439-AC-A': {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, - 'I2_F0_1-10439-AC-A': {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, - 'I0_F1_1-10439-AC-A': {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, - 'I0_F2_1-10439-AC-A': {'ab': 0.531000018119812, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': None, 'sampleType': 'WGS'}, + 'I0_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 60, 'gq': 20, 'numAlt': 0, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I1_F0_1-10439-AC-A': [ + {'ab': 0.0, 'dp': 24, 'gq': 0, 'numAlt': 0, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.0, 'dp': 24, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I2_F0_1-10439-AC-A': [ + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 1, 'filters': [], 'sampleType': 'WES'}, + {'ab': 0.5, 'dp': 10, 'gq': 99, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F1_1-10439-AC-A': [ + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, + {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, + ], + 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, }, } del expected_variant['familyGenotypes'] @@ -983,18 +995,33 @@ def test_variant_lookup(self, mock_variant_lookup): self.assertEqual(response.status_code, 200) individual_guid_map = [ - ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), - ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), - ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), - ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), + ('I000006_hg00733', 'I0_F0_1-10439-AC-A', [ + {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}, + {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}, + ],), + ('I000005_hg00732', 'I1_F0_1-10439-AC-A', [ + {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}, + {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}, + ]), + ('I000004_hg00731', 'I2_F0_1-10439-AC-A', [ + {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}, + {'sampleId': 'HG00731', 'familyGuid': 'F000002_2', 'numAlt': 2}, + ]), + ('I000015_na20885', 'I0_F1_1-10439-AC-A', [ + {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}, + {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}, + ]), ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), ] expected_variant.update({ 'lookupFamilyGuids': ['F000002_2', 'F000011_11', 'F000014_14'], 'liftedFamilyGuids': ['F000014_14'], 'genotypes': { - individual_guid: {**expected_variant['genotypes'][anon_individual_guid], **genotype} - for individual_guid, anon_individual_guid, genotype in individual_guid_map + individual_guid: [ + {**sample_genotype, **genotype[i]} for i, sample_genotype in enumerate(expected_variant['genotypes'][anon_individual_guid]) + ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { + **expected_variant['genotypes'][anon_individual_guid], **genotype, + } for individual_guid, anon_individual_guid, genotype in individual_guid_map }, 'genomeVersion': '37', 'variantId': '1-248367227-TC-T', From b2ba56292ac2e63de0e3bf36c47aa8e439025148 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 12:25:40 -0400 Subject: [PATCH 109/109] clean up --- seqr/views/apis/variant_search_api_tests.py | 22 +++++---------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 619ce40632..89017e7d2d 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -995,22 +995,10 @@ def test_variant_lookup(self, mock_variant_lookup): self.assertEqual(response.status_code, 200) individual_guid_map = [ - ('I000006_hg00733', 'I0_F0_1-10439-AC-A', [ - {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}, - {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}, - ],), - ('I000005_hg00732', 'I1_F0_1-10439-AC-A', [ - {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}, - {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}, - ]), - ('I000004_hg00731', 'I2_F0_1-10439-AC-A', [ - {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}, - {'sampleId': 'HG00731', 'familyGuid': 'F000002_2', 'numAlt': 2}, - ]), - ('I000015_na20885', 'I0_F1_1-10439-AC-A', [ - {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}, - {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}, - ]), + ('I000006_hg00733', 'I0_F0_1-10439-AC-A', {'sampleId': 'HG00733', 'familyGuid': 'F000002_2'}), + ('I000005_hg00732', 'I1_F0_1-10439-AC-A', {'sampleId': 'HG00732', 'familyGuid': 'F000002_2'}), + ('I000004_hg00731', 'I2_F0_1-10439-AC-A', {'sampleId': 'HG00731', 'familyGuid': 'F000002_2'}), + ('I000015_na20885', 'I0_F1_1-10439-AC-A', {'sampleId': 'NA20885', 'familyGuid': 'F000011_11'}), ('I000018_na21234', 'I0_F2_1-10439-AC-A', {'sampleId': 'NA21234', 'familyGuid': 'F000014_14'}), ] expected_variant.update({ @@ -1018,7 +1006,7 @@ def test_variant_lookup(self, mock_variant_lookup): 'liftedFamilyGuids': ['F000014_14'], 'genotypes': { individual_guid: [ - {**sample_genotype, **genotype[i]} for i, sample_genotype in enumerate(expected_variant['genotypes'][anon_individual_guid]) + {**sample_genotype, **genotype} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { **expected_variant['genotypes'][anon_individual_guid], **genotype, } for individual_guid, anon_individual_guid, genotype in individual_guid_map