From c0dc36d7ee442a04cae13dd7fd7a2dbdd4e188d5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 11:04:44 -0400 Subject: [PATCH 01/23] add lat feature update to ui metadata --- seqr/views/apis/feature_updates_api.py | 14 ++-- seqr/views/react_app.py | 22 ++++++ seqr/views/react_app_tests.py | 93 +++++++++++++++++++++++--- 3 files changed, 115 insertions(+), 14 deletions(-) diff --git a/seqr/views/apis/feature_updates_api.py b/seqr/views/apis/feature_updates_api.py index 2fccead30e..140b650182 100644 --- a/seqr/views/apis/feature_updates_api.py +++ b/seqr/views/apis/feature_updates_api.py @@ -11,18 +11,20 @@ TIMEOUT = 5 +def fetch_feature_updates(): + response = requests.get(FEED_URL, timeout=TIMEOUT) + response.raise_for_status() + feed = parse(response.content) + return feed.entries + + def get_feature_updates(request): """ Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns markdown and information for each feed entry. """ - response = requests.get(FEED_URL, timeout=TIMEOUT) - response.raise_for_status() - - feed = parse(response.content) - entries = [] - for entry in feed.entries: + for entry in fetch_feature_updates(): # Atom feeds can have multiple content elements per feed entry markdown = "".join(md(content.value) for content in entry.content) entries.append( diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 0b40591a39..23c9992f44 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -5,6 +5,8 @@ from django.middleware.csrf import rotate_token from django.template import loader from django.http import HttpResponse +import logging + from settings import ( SEQR_VERSION, CSRF_COOKIE_NAME, @@ -15,9 +17,13 @@ VLM_CLIENT_ID, ) from seqr.models import WarningMessage +from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json +from seqr.views.apis.feature_updates_api import fetch_feature_updates from seqr.views.utils.orm_to_json_utils import get_json_for_user, get_json_for_current_user from seqr.views.utils.permissions_utils import login_active_required +logger = logging.getLogger(__name__) + @login_active_required(login_url=LOGIN_URL) def main_app(request, *args, **kwargs): @@ -56,6 +62,7 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], + 'lastFeatureUpdate': _get_latest_feature_update_date(), }} if include_user: initial_json['user'] = get_json_for_current_user(request.user) @@ -89,3 +96,18 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 ) return HttpResponse(html, content_type="text/html", status=status) + + +FEATURE_CACHE_KEY = 'feature_updates_latest_date' + +def _get_latest_feature_update_date(): + update_date = safe_redis_get_json(FEATURE_CACHE_KEY) + if not update_date: + try: + entries = fetch_feature_updates() + except Exception as e: + logger.error(f'Unable to fetch feature update date: {e}') + return None + update_date = entries[0].published + safe_redis_set_json(FEATURE_CACHE_KEY, update_date, expire=60*60*3) + return update_date diff --git a/seqr/views/react_app_tests.py b/seqr/views/react_app_tests.py index f363edcb09..09d177e82a 100644 --- a/seqr/views/react_app_tests.py +++ b/seqr/views/react_app_tests.py @@ -1,19 +1,81 @@ -from datetime import datetime from django.urls.base import reverse import mock +import responses from seqr.views.react_app import main_app, no_login_main_app from seqr.views.utils.terra_api_utils import TerraRefreshTokenFailedException from seqr.views.utils.test_utils import TEST_OAUTH2_PROVIDER, AuthenticationTestCase, AnvilAuthenticationTestCase, USER_FIELDS MOCK_GA_TOKEN = 'mock_ga_token' # nosec +FEATURE_UPDATE_FEED = """ + + + tag:github.com,2008:/broadinstitute/seqr/discussions/categories/feature-updates + + + Recent discussions in broadinstitute/seqr, category: feature-updates + 2023-11-08T15:11:40+00:00 + + tag:github.com,2008:5828412 + + Welcome to Feature Updates + 2023-11-08T15:11:31+00:00 + 2023-11-08T15:11:40+00:00 + + + hanars + https://github.com/hanars + + + <p dir="auto">Welcome to the seqr feature update discussion channel! This channel will be used for all announcements of new seqr functionality</p> + + + <h2 dir="auto">Welcome to our discussion forum!</h2> + <p dir="auto">We\xe2\x80\x99re using Discussions as a place to connect members of the seqr community with our development team and with one another. We hope that you:</p> + <ul dir="auto"> + <li>Ask questions</li> + <li>Describe issues that may be user errors instead of bugs</li> + <li>Answer and engage with one another</li> + <li>Discuss ideas for enhancements</li> + </ul> + + + + tag:github.com,2008:3913526 + + [ACTION REQUIRED] Upcoming Deprecation of older docker images and static JS/CSS assets + + 2022-03-03T16:25:56+00:00 + 2022-06-23T13:32:55+00:00 + + + sjahl + https://github.com/sjahl + + + <h3 dir="auto">What is happening?</h3> + <h4 dir="auto">Summary</h4> + <p dir="auto">Seqr is making updates to how our docker images work, and you will need to update your deployment by 2022/04/04. Please see the information below for more details.</p> + <h4 dir="auto">Slightly Longer Version</h4> + <p dir="auto">We\xe2\x80\x99re making two functional changes to the way seqr is packaged in Docker.</p> + <ol dir="auto"> + <li>We will be removing the statically built javascript/CSS/HTML assets that are checked into git. These will now be generated in the docker image when it\xe2\x80\x99s built, rather than provided in the source code tree.</li> + <li>New docker images will be built every time we release seqr. These images will no longer <code class="notranslate">git pull</code> to update themselves on startup.</li> + </ol> + <p dir="auto">However, if you are using a docker image from prior to when we made these changes, your installation still has the <code class="notranslate">git pull</code> update functionality. If you do nothing, your seqr installation may break after 2022/04/04 when it pulls down the version of the seqr code that has the assets removed. You will find instructions below on how to proceed.</p> + + + +""" + @mock.patch('seqr.views.react_app.DEBUG', False) +@mock.patch('seqr.utils.redis_utils.redis.StrictRedis') class AppPageTest(object): databases = ['default'] fixtures = ['users'] - def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, user_email=None, user_fields=None, ga_token_id=None): + def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, user_email=None, user_fields=None, ga_token_id=None, last_feature_update=None): user_fields = user_fields or USER_FIELDS self.assertEqual(response.status_code, 200) initial_json = self.get_initial_page_json(response) @@ -26,6 +88,7 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, 'oauthLoginProvider': self.OAUTH_PROVIDER, 'vlmEnabled': vlm_enabled, 'warningMessages': [{'id': 1, 'header': 'Warning!', 'message': 'A sample warning'}], + 'lastFeatureUpdate': last_feature_update, }) self.assertEqual(self.get_initial_page_window('gaTrackingId', response), ga_token_id) @@ -39,14 +102,24 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, @mock.patch('seqr.views.react_app.VLM_CLIENT_ID', 'abc123') @mock.patch('seqr.views.react_app.GA_TOKEN_ID', MOCK_GA_TOKEN) - def test_react_page(self): + @responses.activate + def test_react_page(self, mock_redis): url = reverse(main_app) self.check_require_login_no_policies(url, login_redirect_url='/login') + responses.add( + responses.GET, + 'https://github.com/broadinstitute/seqr/discussions/categories/feature-updates.atom', + body=FEATURE_UPDATE_FEED, + ) response = self.client.get(url) - self._check_page_html(response, 'test_user_no_policies', user_email='test_user_no_policy@test.com', ga_token_id=MOCK_GA_TOKEN, vlm_enabled=True) + self._check_page_html(response, 'test_user_no_policies', user_email='test_user_no_policy@test.com', ga_token_id=MOCK_GA_TOKEN, vlm_enabled=True, last_feature_update='2023-11-08T15:11:31+00:00') - def test_local_react_page(self): + mock_redis.return_value.set.assert_called_with('feature_updates_latest_date', '"2023-11-08T15:11:31+00:00"') + mock_redis.return_value.expire.assert_called_with('feature_updates_latest_date', 10800) + + @responses.activate + def test_local_react_page(self, mock_redis): url = reverse(no_login_main_app) response = self.client.get(url, HTTP_HOST='localhost:3000') self.assertEqual(response.status_code, 200) @@ -56,7 +129,8 @@ def test_local_react_page(self): self.assertContains(response, 'src="/app.js"') self.assertNotRegex(content, r']*>') - def test_no_login_react_page(self): + @responses.activate + def test_no_login_react_page(self, mock_redis): url = reverse(no_login_main_app) response = self.client.get(url) @@ -81,12 +155,15 @@ def test_no_login_react_page(self): response = self.client.get(url) self._check_page_html(response, 'test_user') - def test_react_page_additional_configs(self): + @responses.activate + def test_react_page_additional_configs(self, mock_redis): url = reverse(main_app) self.check_require_login_no_policies(url, login_redirect_url='/login') + mock_redis.return_value.get.return_value = '"2025-11-18T00:00:00Z"' + response = self.client.get(url) - self._check_page_html(response, 'test_user_no_policies') + self._check_page_html(response, 'test_user_no_policies', last_feature_update='2025-11-18T00:00:00Z') class LocalAppPageTest(AuthenticationTestCase, AppPageTest): From bc45ea45878233a1a0cfd75c7cb68c6cace45c0b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 11:33:03 -0400 Subject: [PATCH 02/23] add ui label for new faeture --- ui/redux/selectors.js | 1 + ui/shared/components/page/Header.jsx | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 3fe2aace45..93918f9f33 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -41,6 +41,7 @@ export const getOauthLoginProvider = state => state.meta.oauthLoginProvider export const getVlmEnabled = state => state.meta.vlmEnabled export const getHijakEnabled = state => state.meta.hijakEnabled export const getWarningMessages = state => state.meta.warningMessages +export const getLastFeatureUpdate = state => state.meta.lastFeatureUpdate export const getSavedVariantsIsLoading = state => state.savedVariantsLoading.isLoading export const getSavedVariantsLoadingError = state => state.savedVariantsLoading.errorMessage export const getSearchesByHash = state => state.searchesByHash diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index f56839d732..93ee0326db 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -2,12 +2,12 @@ import React from 'react' import PropTypes from 'prop-types' import styled from 'styled-components' -import { Menu, Header, Dropdown } from 'semantic-ui-react' +import { Menu, Header, Dropdown, Label } from 'semantic-ui-react' import { connect } from 'react-redux' import { Link } from 'react-router-dom' import { updateUser } from 'redux/rootReducer' -import { getUser, getOauthLoginProvider } from 'redux/selectors' +import { getUser, getOauthLoginProvider, getLastFeatureUpdate } from 'redux/selectors' import { USER_NAME_FIELDS, LOCAL_LOGIN_URL, FEATURE_UPDATES_PATH } from 'shared/utils/constants' import UpdateButton from '../buttons/UpdateButton' @@ -18,7 +18,7 @@ const HeaderMenu = styled(Menu)` padding-right: 100px; ` -const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { +const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit, lastFeatureUpdate }) => { const loginUrl = oauthLoginProvider ? `/login/${oauthLoginProvider}` : LOCAL_LOGIN_URL return ( @@ -34,7 +34,11 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { , ] : null } - + + Feature Updates + {(lastFeatureUpdate && (new Date()).setMonth(new Date().getMonth() - 1) < new Date(lastFeatureUpdate)) && + } + {Object.keys(user).length ? [ ({ user: getUser(state), oauthLoginProvider: getOauthLoginProvider(state), + lastFeatureUpdate: getLastFeatureUpdate(state), }) const mapDispatchToProps = { From e28f70b367512e9592725ff304ffb3e5c7ba6c81 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 11:54:27 -0400 Subject: [PATCH 03/23] move search validation --- clickhouse_search/search.py | 31 ++++++++++++++++++++++++++++--- clickhouse_search/search_tests.py | 13 ++++++++----- seqr/utils/search/utils.py | 26 -------------------------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fd37ecb84b..5dffffef0a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -29,6 +29,8 @@ class InvalidSearchException(Exception): pass BATCH_SIZE = 10000 +MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 +MIN_MULTI_FAMILY_SEQR_AC = 5000 TRANSCRIPT_CONSEQUENCES_FIELD = 'sortedTranscriptConsequences' SELECTED_GENE_FIELD = 'selectedGeneId' @@ -40,22 +42,28 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) + has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) sample_data_by_dataset_type = sample_data_by_dataset_type or {} results = [] exclude_keys = search.pop('exclude_keys', None) or {} exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} searched_dataset_types = set() sample_data_errors = set() + + _validate_num_families(families, has_comp_het, has_location_filter, **search) + for dataset_type in ENTRY_CLASS_MAP[genome_version]: try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, - annotate_affected_males=has_x_chrom_comp_het or has_x_linked, **search, + annotate_affected_males=has_x_chrom_comp_het or has_x_linked, has_location_filter=has_location_filter, **search, ) except InvalidDatasetTypeException: continue if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and search.get('no_access_project_genome_version'): + if has_comp_het: + raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') logger.info('Looking up variants in projects with no user access', user) results += _get_search_results( entry_qs, variants_qs, **search, **parsed_filters, sample_data=None, skip_entry_fields=True, @@ -83,6 +91,8 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s ) if has_comp_het: + if not search.get('annotations'): + raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') comp_het_sample_data = sample_data if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: comp_het_sample_data = _no_affected_male_families(sample_data, user) @@ -107,7 +117,18 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s logger.info(f'Total results: {len(results)}', user) return get_sorted_search_results(results, sort, families) -def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, no_access_project_genome_version=None, **search_kwargs): +def _validate_num_families(families, has_comp_het, has_location_filter, freqs=None, **kwargs): + if has_comp_het and not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: + raise InvalidSearchException( + 'Location must be specified to search for compound heterozygous variants across many families', + ) + seqr_ac_filter = (freqs or {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) + if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: + raise InvalidSearchException( + f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' + ) + +def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, no_access_project_genome_version=None, has_location_filter=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects parsed_filters = variants_qs.get_parsed_annotations_filters(**search_kwargs) @@ -122,6 +143,7 @@ def _parse_dataset_type_query(genome_version, dataset_type, families, sample_dat inheritance_filter=inheritance_filter, annotate_affected_males=annotate_affected_males, allow_no_samples=bool(no_access_project_genome_version), + has_location_filter=has_location_filter, ) sample_data_by_dataset_type[dataset_type] = sample_data @@ -551,7 +573,7 @@ def _get_sample_metadata(samples, affected_family_only, annotate_affected_males) return samples.aggregate(**annotations) -def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None): +def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None, has_location_filter=False): sample_type = None if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): dataset_type, sample_type = dataset_type.split('_') @@ -563,6 +585,9 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo affected_family_only = inheritance_mode and not individual_affected_status sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males) + if not has_location_filter and len(sample_data['project_guids']) > 1: + raise InvalidSearchException('Location must be specified to search across multiple projects') + family_guids = set(sample_data.pop('family_guids')) sample_data['num_families'] = len(family_guids) if sample_type: diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a5078a8996..f8a5349259 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -848,7 +848,7 @@ def test_variant_id_search(self): @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) - @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) + @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): url = reverse(query_variants_handler, args=['not_a_hash']) @@ -896,12 +896,14 @@ def test_invalid_search(self): pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}, ) - self._assert_expected_search_error('Annotations must be specified to search for compound heterozygous variants', inheritance_mode='recessive') - - annotations = {'frameshift': ['frameshift_variant']} self._assert_expected_search_error( 'Location must be specified to search for compound heterozygous variants across many families', - annotations=annotations, inheritance_mode='recessive', + inheritance_mode='recessive', + ) + + self._assert_expected_search_error( + 'Annotations must be specified to search for compound heterozygous variants', + inheritance_mode='recessive', locus={'rawItems': GENE_IDS[0]}, ) self._assert_expected_search_error( @@ -911,6 +913,7 @@ def test_invalid_search(self): self._assert_expected_search_error('seqr AC frequency of at least 5000 must be specified to search across multiple families', freqs={}) + annotations = {'frameshift': ['frameshift_variant']} self._assert_expected_search_error( 'Inheritance based search is disabled in families with no data loaded for affected individuals', inheritance_mode='recessive', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000005_5']}], diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2b52f3fe96..1188d041cc 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -19,11 +19,9 @@ logger = SeqrLogger(__name__) - MAX_GENES_FOR_FILTER = 10000 MIN_MULTI_FAMILY_SEQR_AC = 5000 MAX_EXPORT_VARIANTS = 1000 -MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 def _get_search_genome_version(search_model): @@ -123,7 +121,6 @@ def _query_variants(search_model, user, sort=None): _validate_sort(sort, families) parsed_search = _parse_search(search, genome_version, user) - _validate_search(parsed_search, families) results = get_clickhouse_variants(families, parsed_search, user, genome_version, sort=sort) @@ -270,29 +267,6 @@ def _parse_inheritance(search): raise InvalidSearchException('Inheritance must be specified if custom affected status is set') -def _validate_search(search, families): - has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} - has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) - if has_comp_het_search: - if not search.get('annotations'): - raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') - - if not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: - raise InvalidSearchException( - 'Location must be specified to search for compound heterozygous variants across many families') - - if search.get('no_access_project_genome_version'): - raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - - if not has_location_filter and families.values('project_id').distinct().count() > 1: - raise InvalidSearchException('Location must be specified to search across multiple projects') - seqr_ac_filter = search.get('freqs', {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) - if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: - raise InvalidSearchException( - f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' - ) - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, From 52c310946093238fbd45cb879f99716cdf2f764c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 11:58:40 -0400 Subject: [PATCH 04/23] clean up sort validation --- clickhouse_search/search.py | 16 +++++++++++----- seqr/utils/search/utils.py | 6 ------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 5dffffef0a..dba9d17b7e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -679,6 +679,16 @@ def _has_x_chrom(genome_version, genes=None, intervals=None, **kwargs): return any('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) or any('X' in interval['chrom'] for interval in (intervals or [])) +def _prioritized_gene_sort(gene_ids, families): + if len(families) != 1: + raise InvalidSearchException('Phenotype sort is only supported for single-family search.') + return { + agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( + gene_id__in=gene_ids, individual__family_id=families[0].id, rank__lte=100, + ).values('gene_id').annotate(min_rank=Min('rank')) + } + + OMIM_SORT = 'in_omim' GENE_SORTS = { 'constraint': lambda gene_ids, _: { @@ -688,11 +698,7 @@ def _has_x_chrom(genome_version, genes=None, intervals=None, **kwargs): OMIM_SORT: lambda gene_ids, _: set(Omim.objects.filter( gene__gene_id__in=gene_ids, phenotype_mim_number__isnull=False, ).values_list('gene__gene_id', flat=True)), - PRIORITIZED_GENE_SORT: lambda gene_ids, families: { - agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( - gene_id__in=gene_ids, individual__family_id=families[0].id, rank__lte=100, - ).values('gene_id').annotate(min_rank=Min('rank')) - }, + PRIORITIZED_GENE_SORT: _prioritized_gene_sort, } def _get_sort_gene_metadata(sort, results, families): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 1188d041cc..ef1e61d471 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -118,7 +118,6 @@ def _query_variants(search_model, user, sort=None): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() - _validate_sort(sort, families) parsed_search = _parse_search(search, genome_version, user) @@ -248,11 +247,6 @@ def _parse_valid_variant_id(variant_id): return chrom, pos, ref, alt -def _validate_sort(sort, families): - if sort == PRIORITIZED_GENE_SORT and len(families) != 1: - raise InvalidSearchException('Phenotype sort is only supported for single-family search.') - - def _parse_inheritance(search): inheritance = search.pop('inheritance') inheritance_mode = inheritance.get('mode') From 1fc3688a0915fb35867a916e8fa771e22d6522eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 12:16:25 -0400 Subject: [PATCH 05/23] pass search deconstructed --- clickhouse_search/search.py | 33 +++++++++---------- .../commands/tag_seqr_prioritized_variants.py | 4 +-- seqr/utils/search/utils.py | 5 ++- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index dba9d17b7e..262fe0d1b6 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -37,16 +37,13 @@ class InvalidSearchException(Exception): SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, search, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False): - inheritance_mode = search.get('inheritance_mode') +def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance_mode=None, annotations=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) sample_data_by_dataset_type = sample_data_by_dataset_type or {} results = [] - exclude_keys = search.pop('exclude_keys', None) or {} - exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} searched_dataset_types = set() sample_data_errors = set() @@ -55,19 +52,20 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s for dataset_type in ENTRY_CLASS_MAP[genome_version]: try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( - genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, - annotate_affected_males=has_x_chrom_comp_het or has_x_linked, has_location_filter=has_location_filter, **search, + genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, **search, + annotate_affected_males=has_x_chrom_comp_het or has_x_linked, has_location_filter=has_location_filter, + allow_no_samples=bool(no_access_project_genome_version), inheritance_mode=inheritance_mode, annotations=annotations, ) except InvalidDatasetTypeException: continue - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and search.get('no_access_project_genome_version'): + if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: if has_comp_het: raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') logger.info('Looking up variants in projects with no user access', user) results += _get_search_results( entry_qs, variants_qs, **search, **parsed_filters, sample_data=None, skip_entry_fields=True, - exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), + exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), inheritance_mode=inheritance_mode, ) searched_dataset_types.add(dataset_type) @@ -80,24 +78,24 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s dataset_results = [] if inheritance_mode != COMPOUND_HET: dataset_results += _get_search_results( - entry_qs, variants_qs, sample_data, exclude_keys=exclude_keys.get(dataset_type), **search, **parsed_filters, + entry_qs, variants_qs, sample_data, inheritance_mode=inheritance_mode, exclude_keys=(exclude_keys or {}).get(dataset_type), **search, **parsed_filters, ) run_x_linked_male_search = has_x_linked and not (inheritance_mode == X_LINKED_RECESSIVE and sample_data.get('samples')) if run_x_linked_male_search: dataset_results += _get_x_linked_male_search_results( - entry_qs, variants_qs, dataset_type, user, sample_data, exclude_keys=exclude_keys.get(dataset_type), + entry_qs, variants_qs, dataset_type, user, sample_data, exclude_keys=(exclude_keys or {}).get(dataset_type), **search, **parsed_filters, ) if has_comp_het: - if not search.get('annotations'): + if not annotations: raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') comp_het_sample_data = sample_data if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: comp_het_sample_data = _no_affected_male_families(sample_data, user) result_q = _get_data_type_comp_het_results_queryset( - entry_qs, variants_qs, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), + entry_qs, variants_qs, comp_het_sample_data, exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), **search, **parsed_filters, ) if result_q is not None: @@ -109,7 +107,7 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s searched_dataset_types.add(dataset_type) if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): - results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, **search) + results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, annotations=annotations, **search) if not searched_dataset_types: _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type) @@ -128,7 +126,7 @@ def _validate_num_families(families, has_comp_het, has_location_filter, freqs=No f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' ) -def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, no_access_project_genome_version=None, has_location_filter=False, **search_kwargs): +def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, has_location_filter=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects parsed_filters = variants_qs.get_parsed_annotations_filters(**search_kwargs) @@ -142,7 +140,6 @@ def _parse_dataset_type_query(genome_version, dataset_type, families, sample_dat inheritance_mode=inheritance_mode, inheritance_filter=inheritance_filter, annotate_affected_males=annotate_affected_males, - allow_no_samples=bool(no_access_project_genome_version), has_location_filter=has_location_filter, ) sample_data_by_dataset_type[dataset_type] = sample_data @@ -177,7 +174,7 @@ def _get_search_results(entry_qs, variants_qs, sample_data, skip_entry_fields=Fa return _evaluate_results(results.result_values(skip_entry_fields=skip_entry_fields)) -def _get_x_linked_male_search_results(entry_qs, variants_qs, dataset_type, user, sample_data, inheritance_mode=None, **search_kwargs): +def _get_x_linked_male_search_results(entry_qs, variants_qs, dataset_type, user, sample_data, **search_kwargs): affected_male_family_guids = { s['family_guid'] for s in sample_data['samples'] if s['affected'] == AFFECTED and s['sex'] in MALE_SEXES } if 'samples' in sample_data else sample_data['affected_male_family_guids'] @@ -201,7 +198,7 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, annotations=None, annotations_secondary=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -295,7 +292,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d return results -def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, annotations_secondary=None, inheritance_mode=None, exclude_key_pairs=None, **search_kwargs): +def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): parsed_secondary_filters = {} if annotations_secondary: try: diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 71b4a1cd9b..c51ebb448f 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -628,10 +628,10 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by @staticmethod def _execute_search(sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, **kwargs): results = get_clickhouse_variants( - families=None, search=kwargs, user=None, genome_version=GENOME_VERSION_GRCh38, + families=None, user=None, genome_version=GENOME_VERSION_GRCh38, encode_genotypes_json=True, sample_data_by_dataset_type={ **{dt: None for dt in ENTRY_CLASS_MAP[GENOME_VERSION_GRCh38]}, **sample_data_by_dataset_type, - }, + }, **kwargs ) for result in results: if isinstance(result, list): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index ef1e61d471..03e825c9ad 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -45,9 +45,8 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user): parsed_variant_id = parse_variant_id(variant_id) parsed_variant_ids = [parsed_variant_id] if parsed_variant_id else None - search = {'parsed_variant_ids': parsed_variant_ids, 'variant_ids': [variant_id]} genome_version = family.project.genome_version - variants = get_clickhouse_variants([family], search, user, genome_version) + variants = get_clickhouse_variants([family], user, genome_version, parsed_variant_ids=parsed_variant_ids, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return format_clickhouse_results(variants, genome_version)[0] @@ -121,7 +120,7 @@ def _query_variants(search_model, user, sort=None): parsed_search = _parse_search(search, genome_version, user) - results = get_clickhouse_variants(families, parsed_search, user, genome_version, sort=sort) + results = get_clickhouse_variants(families, user, genome_version, sort=sort, **parsed_search) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) From 61040936024221cff7d959dc75295963eca87acf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 12:51:39 -0400 Subject: [PATCH 06/23] fix bugs --- clickhouse_search/search.py | 11 +++++----- .../commands/tag_seqr_prioritized_variants.py | 21 +++++++++++++------ seqr/utils/search/utils.py | 3 +-- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 262fe0d1b6..536fa3766a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -37,7 +37,7 @@ class InvalidSearchException(Exception): SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance_mode=None, annotations=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): +def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance_mode=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) @@ -52,9 +52,9 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da for dataset_type in ENTRY_CLASS_MAP[genome_version]: try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( - genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, **search, + genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males=has_x_chrom_comp_het or has_x_linked, has_location_filter=has_location_filter, - allow_no_samples=bool(no_access_project_genome_version), inheritance_mode=inheritance_mode, annotations=annotations, + allow_no_samples=bool(no_access_project_genome_version), inheritance_mode=inheritance_mode, **search, ) except InvalidDatasetTypeException: continue @@ -89,7 +89,7 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da ) if has_comp_het: - if not annotations: + if not any(parsed_filters.values()): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') comp_het_sample_data = sample_data if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: @@ -107,7 +107,7 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da searched_dataset_types.add(dataset_type) if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): - results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, annotations=annotations, **search) + results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, **search) if not searched_dataset_types: _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type) @@ -141,6 +141,7 @@ def _parse_dataset_type_query(genome_version, dataset_type, families, sample_dat inheritance_filter=inheritance_filter, annotate_affected_males=annotate_affected_males, has_location_filter=has_location_filter, + allow_no_samples=allow_no_samples, ) sample_data_by_dataset_type[dataset_type] = sample_data diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index c51ebb448f..5ff958aa19 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -225,13 +225,22 @@ **RECESSIVE_SEARCH, }, 'Compound Heterozygous - Both High Splice AI': { + 'family_filter': { + CONFIRMED_FAMILY_FILTER: False + }, 'inheritance_mode': COMPOUND_HET, - **HIGH_SPLICE_AI_SEARCH, + 'annotations':{ + 'splice_ai': 0.8, + }, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'Compound Heterozygous - Both High Splice AI - Confirmed': { - 'inheritance_mode': COMPOUND_HET, - **CONFIRMED_HIGH_SPLICE_AI_SEARCH, + 'family_filter': { + CONFIRMED_FAMILY_FILTER: True + }, + 'annotations': { + 'splice_ai': 0.5, + }, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'Compound Heterozygous - Clinvar Pathogenic/ High Splice AI': { @@ -247,7 +256,6 @@ CONFIRMED_FAMILY_FILTER: False }, 'inheritance_mode': COMPOUND_HET, - 'no_secondary_annotations': True, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.8, @@ -263,7 +271,6 @@ CONFIRMED_FAMILY_FILTER: True }, 'inheritance_mode': COMPOUND_HET, - 'no_secondary_annotations': True, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.5, @@ -393,6 +400,7 @@ 'mitomap_pathogenic': True, }, 'freqs': { + 'callset': {'ac': 5000}, 'gnomad_mito': {'af': 0.05}, }, }, @@ -401,6 +409,7 @@ 'inheritance_mode': DE_NOVO, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'freqs': { + 'callset': {'ac': 5000}, 'gnomad_mito': {'af': 0.001}, }, 'in_silico': { @@ -628,7 +637,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by @staticmethod def _execute_search(sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, **kwargs): results = get_clickhouse_variants( - families=None, user=None, genome_version=GENOME_VERSION_GRCh38, + families=family_guid_map.keys(), user=None, genome_version=GENOME_VERSION_GRCh38, encode_genotypes_json=True, sample_data_by_dataset_type={ **{dt: None for dt in ENTRY_CLASS_MAP[GENOME_VERSION_GRCh38]}, **sample_data_by_dataset_type, }, **kwargs diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 03e825c9ad..4f6b90047a 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -10,8 +10,7 @@ from seqr.models import Sample, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json -from clickhouse_search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY +from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom from seqr.views.utils.permissions_utils import user_is_analyst From 26a6f63dd708a1b67e112693934a08e44694d46e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 13:09:52 -0400 Subject: [PATCH 07/23] clean up parsing --- clickhouse_search/search.py | 10 ++++++++-- seqr/utils/search/utils.py | 21 --------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 536fa3766a..dfafa558a6 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -37,7 +37,13 @@ class InvalidSearchException(Exception): SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance_mode=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): +def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): + inheritance_filter = inheritance.get('filter') or {} + search['inheritance_filter'] = inheritance_filter + inheritance_mode = None if inheritance_filter.get('genotype') else inheritance.get('mode') + if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: + raise InvalidSearchException('Inheritance must be specified if custom affected status is set') + has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) @@ -203,7 +209,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d if annotations_secondary: annotations = { **annotations, - **{k: v + annotations[k] if k in annotations else v for k, v in annotations_secondary.items()}, + **{k: v + annotations[k] if annotations.get(k) else v for k, v in annotations_secondary.items()}, } try: diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 4f6b90047a..d45b40799e 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -161,13 +161,6 @@ def _parse_search(search, genome_version, user): if exclude_previous_hash: parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user)) - for annotation_key in ['annotations', 'annotations_secondary']: - if parsed_search.get(annotation_key): - parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - - if parsed_search.get('inheritance'): - _parse_inheritance(parsed_search) - return parsed_search @@ -245,20 +238,6 @@ def _parse_valid_variant_id(variant_id): return chrom, pos, ref, alt -def _parse_inheritance(search): - inheritance = search.pop('inheritance') - inheritance_mode = inheritance.get('mode') - inheritance_filter = inheritance.get('filter') or {} - - if inheritance_filter.get('genotype'): - inheritance_mode = None - - search.update({'inheritance_mode': inheritance_mode, 'inheritance_filter': inheritance_filter}) - - if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: - raise InvalidSearchException('Inheritance must be specified if custom affected status is set') - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, From c10660017c4019d1559df52bb7d5c8508cd593a2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 13:25:27 -0400 Subject: [PATCH 08/23] move validation into manager --- clickhouse_search/managers.py | 18 ++++++++++++++++-- clickhouse_search/search.py | 35 +++++++++++------------------------ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index b514d2620d..bd51af58ac 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -22,6 +22,9 @@ from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS, CHROMOSOME_CHOICES +class InvalidSearchException(Exception): + pass + class InvalidDatasetTypeException(Exception): pass @@ -1036,7 +1039,7 @@ def search(self, sample_data, exclude_keys=None, **kwargs): entries = entries.exclude(key__in=exclude_keys) entries = self._join_annotations(entries) - entries = self._prefilter_entries(entries, **kwargs) + entries = self._prefilter_entries(entries, sample_data=sample_data, **kwargs) return self._search_call_data(entries, sample_data, **kwargs) @@ -1090,6 +1093,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_q = None gt_filter = None quality_filter = qualityFilter or {} + if not inheritance_mode and list((inheritance_filter or {}).keys()) == ['affected']: + raise InvalidSearchException('Inheritance must be specified if custom affected status is set') if inheritance_mode or (inheritance_filter or {}).get('genotype') or quality_filter: clinvar_override_q = self._clinvar_path_q(pathogenicity) inheritance_q, quality_q, gt_filter, carriers_expression = self._get_inheritance_quality_qs( @@ -1507,6 +1512,7 @@ def _filter_seqr_freq_field(self, entries, freq_field, value): class EntriesManager(BaseEntriesManager): + MIN_MULTI_FAMILY_SEQR_AC = 5000 @staticmethod def _clinvar_range_q(path_range): @@ -1533,7 +1539,15 @@ def _prefilter_entries(self, entries, freqs=None, **kwargs): entries = self._filter_in_silico(entries, **kwargs) return entries - def _filter_frequency(self, entries, frequencies, pathogenicity=None, **kwargs): + def _filter_frequency(self, entries, frequencies, pathogenicity=None, sample_data=None, **kwargs): + callset_filter = frequencies.get(self.callset_filter_field) or {} + if (not callset_filter.get('ac') or callset_filter['ac'] >= self.MIN_MULTI_FAMILY_SEQR_AC) and ( + len(sample_data['sample_type_families']) > 0 or len(next((sample_data['sample_type_families'].values()))) > 0 + ): + raise InvalidSearchException( + f'seqr AC frequency of at least {self.MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' + ) + gnomad_filter = frequencies.get('gnomad_genomes') or {} if hasattr(self.model, 'is_gnomad_gt_5_percent') and ( (gnomad_filter.get('af') or 1) <= 0.05 or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh']) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index dfafa558a6..cbc73f2e86 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -10,7 +10,7 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ ArrayMap, Modulo -from clickhouse_search.managers import InvalidDatasetTypeException +from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ @@ -25,8 +25,6 @@ logger = SeqrLogger(__name__) -class InvalidSearchException(Exception): - pass BATCH_SIZE = 10000 MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 @@ -38,11 +36,8 @@ class InvalidSearchException(Exception): def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): - inheritance_filter = inheritance.get('filter') or {} - search['inheritance_filter'] = inheritance_filter - inheritance_mode = None if inheritance_filter.get('genotype') else inheritance.get('mode') - if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: - raise InvalidSearchException('Inheritance must be specified if custom affected status is set') + search['inheritance_filter'] = inheritance.get('filter') or {} + inheritance_mode = None if search['inheritance_filter'].get('genotype') else inheritance.get('mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) @@ -52,9 +47,6 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da results = [] searched_dataset_types = set() sample_data_errors = set() - - _validate_num_families(families, has_comp_het, has_location_filter, **search) - for dataset_type in ENTRY_CLASS_MAP[genome_version]: try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( @@ -121,16 +113,6 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da logger.info(f'Total results: {len(results)}', user) return get_sorted_search_results(results, sort, families) -def _validate_num_families(families, has_comp_het, has_location_filter, freqs=None, **kwargs): - if has_comp_het and not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: - raise InvalidSearchException( - 'Location must be specified to search for compound heterozygous variants across many families', - ) - seqr_ac_filter = (freqs or {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) - if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: - raise InvalidSearchException( - f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' - ) def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, has_location_filter=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) @@ -589,10 +571,15 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo affected_family_only = inheritance_mode and not individual_affected_status sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males) - if not has_location_filter and len(sample_data['project_guids']) > 1: - raise InvalidSearchException('Location must be specified to search across multiple projects') - family_guids = set(sample_data.pop('family_guids')) + if not has_location_filter: + if len(sample_data['project_guids']) > 1: + raise InvalidSearchException('Location must be specified to search across multiple projects') + if inheritance_mode in {RECESSIVE, COMPOUND_HET} and len(family_guids) > MAX_NO_LOCATION_COMP_HET_FAMILIES: + raise InvalidSearchException( + 'Location must be specified to search for compound heterozygous variants across many families', + ) + sample_data['num_families'] = len(family_guids) if sample_type: sample_data['sample_type_families'] = {sample_type: family_guids} From d7b12c96c664894dda7fefe72752d0e2a96896af Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 13:31:41 -0400 Subject: [PATCH 09/23] more parsing clean up --- clickhouse_search/managers.py | 7 ++++++- seqr/utils/search/utils.py | 5 ----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index bd51af58ac..1d1c265d77 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -611,13 +611,18 @@ def _parse_in_silico_qs(self, results, in_silico, require_score, in_silico_qs, i return results, False, in_silico_q, missing_q - def _filter_annotations(self, results, *args, exclude=None, **kwargs): + def _filter_annotations(self, results, *args, exclude=None, pathogenicity=None, **kwargs): screen_expression = self._screen_expression() if screen_expression: results = results.annotate(screen=self._screen_expression()) results = super()._filter_annotations(results, *args, **kwargs) + if (exclude or {}).get(CLINVAR_KEY) and (pathogenicity or {}).get(CLINVAR_KEY): + duplicates = set(pathogenicity[CLINVAR_KEY]).intersection(exclude[CLINVAR_KEY]) + if duplicates: + raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') + exclude_clinvar_q = self._clinvar_filter_q(exclude) if exclude_clinvar_q is not None: results = results.exclude(exclude_clinvar_q) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index d45b40799e..ad03336503 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -151,11 +151,6 @@ def _parse_search(search, genome_version, user): parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) exclude.pop('rawItems', None) - if exclude.get('clinvar') and (search.get('pathogenicity') or {}).get('clinvar'): - duplicates = set(search['pathogenicity']['clinvar']).intersection(exclude['clinvar']) - if duplicates: - raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') - exclude.pop('previousSearch', None) exclude_previous_hash = exclude.pop('previousSearchHash', None) if exclude_previous_hash: From 8a366f9cf7dc57420bf820bb554ce247e302861d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 13:58:09 -0400 Subject: [PATCH 10/23] fix frequency validation --- clickhouse_search/managers.py | 2 +- clickhouse_search/search.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1d1c265d77..d6b602768a 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1547,7 +1547,7 @@ def _prefilter_entries(self, entries, freqs=None, **kwargs): def _filter_frequency(self, entries, frequencies, pathogenicity=None, sample_data=None, **kwargs): callset_filter = frequencies.get(self.callset_filter_field) or {} if (not callset_filter.get('ac') or callset_filter['ac'] >= self.MIN_MULTI_FAMILY_SEQR_AC) and ( - len(sample_data['sample_type_families']) > 0 or len(next((sample_data['sample_type_families'].values()))) > 0 + len(sample_data['sample_type_families']) > 1 or len(next(iter(sample_data['sample_type_families'].values()))) > 1 ): raise InvalidSearchException( f'seqr AC frequency of at least {self.MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index cbc73f2e86..7666e95831 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -36,8 +36,8 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): - search['inheritance_filter'] = inheritance.get('filter') or {} - inheritance_mode = None if search['inheritance_filter'].get('genotype') else inheritance.get('mode') + search['inheritance_filter'] = (inheritance or {}).get('filter') or {} + inheritance_mode = None if search['inheritance_filter'].get('genotype') else (inheritance or {}).get('mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) From 0ddf7973363d0cb3984f25d9510b532990447da7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 14:39:40 -0400 Subject: [PATCH 11/23] move variant id parsing to manager --- .../commands/set_saved_variant_key.py | 3 +- clickhouse_search/managers.py | 30 ++++++++---- clickhouse_search/search.py | 17 ++++--- clickhouse_search/search_tests.py | 2 +- seqr/utils/search/utils.py | 49 ++----------------- seqr/utils/xpos_utils.py | 11 +++++ seqr/views/apis/variant_search_api.py | 3 +- 7 files changed, 49 insertions(+), 66 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 2569dfc3a8..c76769effd 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -2,13 +2,12 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand -from django.db.models import F import logging from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import SavedVariant, Sample -from seqr.utils.search.utils import parse_variant_id +from seqr.utils.xpos_utils import parse_variant_id logger = logging.getLogger(__name__) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index d6b602768a..8f5ced7faf 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -19,7 +19,7 @@ CLINVAR_CONFLICTING_P_LP, CLINVAR_CONFLICTING_NO_P, CLINVAR_CONFLICTING, PATH_FREQ_OVERRIDE_CUTOFF, \ HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, \ X_LINKED_RECESSIVE_MALE_AFFECTED, FEMALE_SEXES, SV_ANNOTATION_TYPES -from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS, CHROMOSOME_CHOICES +from seqr.utils.xpos_utils import get_xpos, parse_variant_id, MIN_POS, MAX_POS, CHROMOSOME_CHOICES class InvalidSearchException(Exception): @@ -1462,6 +1462,14 @@ def filter_locus(self, exclude_locations=False, require_gene_filter=False, inter return entries + @staticmethod + def _parse_variant_ids(raw_variant_items): + parsed_variant_ids = {} + for item in (raw_variant_items or '').replace(',', ' ').split(): + variant_id = item.replace('chr', '') + parsed_variant_ids[variant_id] = parse_variant_id(variant_id) + return parsed_variant_ids + def _filter_locations(self, entries, genes, intervals, exclude_locations=False, require_gene_filter=False): locus_q = None if genes: @@ -1655,14 +1663,18 @@ def _join_annotations(self, entries): ) return super()._join_annotations(entries) - def filter_locus(self, *args, require_any_gene=False, parsed_variant_ids=None, intervals=None, genes=None, variant_ids=None, **kwargs): - if parsed_variant_ids: + def filter_locus(self, *args, require_any_gene=False, rawVariantItems=None, intervals=None, genes=None, variant_ids=None, **kwargs): + if rawVariantItems: + parsed_variant_ids = self._parse_variant_ids(rawVariantItems) + invalid_items = [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id] + if invalid_items: + if variant_ids: + raise InvalidDatasetTypeException + raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) # although technically redundant, the interval query is applied to the entries table to # improve performance by using the xpos projection - intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids] - variant_ids = ['-'.join([str(o) for o in variant_id]) for variant_id in parsed_variant_ids] - elif variant_ids: - raise InvalidDatasetTypeException + intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids.values()] + variant_ids = parsed_variant_ids.keys() entries = super().filter_locus(*args, intervals=intervals, genes=genes, variant_ids=variant_ids, **kwargs) @@ -1734,8 +1746,8 @@ def _annotate_calls(self, entries, *args, **kwargs): return entries - def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, parsed_variant_ids=None, **kwargs): - if exclude_svs or parsed_variant_ids: + def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, rawVariantItems=None, **kwargs): + if exclude_svs or any(self._parse_variant_ids(rawVariantItems).values()): raise InvalidDatasetTypeException # SV interval filtering occurs after joining on annotations to correctly incorporate end position can_filter_gene_interval = self._can_filter_gene_interval(genes) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 7666e95831..36cb3f4407 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -831,12 +831,12 @@ def _filter_lookup_entries(entries, affected_only, hom_only): entries = entries.filter(calls__array_exists={'gt': (2,)}) return entries -def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, genome_version, affected_only, hom_only): +def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only): data_type = entry_qs = None for dataset_type, entry_cls in sorted(ENTRY_CLASS_MAP[genome_version].items()): try: entry_qs = entry_cls.objects.filter_locus( - variant_ids=[variant_id], parsed_variant_ids=[parsed_variant_id] if parsed_variant_id else [], + variant_ids=[variant_id], rawVariantItems=variant_id, ) except InvalidDatasetTypeException: continue @@ -856,16 +856,17 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, entry_qs, genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) if variant: - _add_liftover_genotypes(variant, data_type, parsed_variant_id, affected_only, hom_only) + _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only) else: lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: from seqr.utils.search.utils import run_liftover - liftover_results = run_liftover(lifted_genome_version, parsed_variant_id[0], parsed_variant_id[1]) + chrom, pos, ref, alt = variant_id.split('-') + liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: - lifted_id = (liftover_results[0], liftover_results[1], *parsed_variant_id[2:]) - entry_qs = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) + lifted_id = variant_id.replace(pos, str(liftover_results[1])) + entry_qs = lifted_entry_cls.objects.filter_locus(rawVariantItems=lifted_id) variant = _clickhouse_variant_lookup( entry_qs, lifted_genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) @@ -899,8 +900,8 @@ def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_o lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return - lifted_id = (variant['liftedOverChrom'], variant['liftedOverPos'], *variant_id[2:]) - lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) + lifted_id = variant_id.replace(str(variant['pos']), str(variant['liftedOverPos'])) + lifted_entries = lifted_entry_cls.objects.filter_locus(rawVariantItems=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f8a5349259..82cc532d0c 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -861,7 +861,7 @@ def test_invalid_search(self): self._assert_expected_search_error('Invalid search: no projects/ families specified', project_families=[]) search_hash = self._assert_expected_search_error( - 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, + 'Invalid variants: 2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, ) self.set_cache(f'search_results__VRS{search_hash:07d}__xpos', [VARIANT1, VARIANT2, VARIANT3, VARIANT4]) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index ad03336503..662014bf5d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -12,7 +12,7 @@ from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.gene_utils import parse_locus_list_items -from seqr.utils.xpos_utils import get_xpos, format_chrom +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.permissions_utils import user_is_analyst logger = SeqrLogger(__name__) @@ -42,10 +42,8 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user): - parsed_variant_id = parse_variant_id(variant_id) - parsed_variant_ids = [parsed_variant_id] if parsed_variant_id else None genome_version = family.project.genome_version - variants = get_clickhouse_variants([family], user, genome_version, parsed_variant_ids=parsed_variant_ids, variant_ids=[variant_id]) + variants = get_clickhouse_variants([family], user, genome_version, rawVariantItems=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return format_clickhouse_results(variants, genome_version)[0] @@ -62,8 +60,7 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ if variants: return variants - parsed_variant_id = parse_variant_id(variant_id) - variants = clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, genome_version, affected_only, hom_only) + variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) return variants @@ -145,10 +142,7 @@ def _parse_search(search, genome_version, user): raise InvalidSearchException('Too many genes/intervals') parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) if not (genes or intervals): - variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) - if invalid_items: - raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) - parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) + parsed_search.update({'rawVariantItems': locus.get('rawVariantItems')}) exclude.pop('rawItems', None) exclude.pop('previousSearch', None) @@ -198,41 +192,6 @@ def get_variant_query_gene_counts(search_model, user): return gene_aggs -def _parse_variant_items(search_json): - raw_items = search_json.get('rawVariantItems') - if not raw_items: - return None, None, None - - invalid_items = [] - variant_ids = [] - parsed_variant_ids = [] - for item in raw_items.replace(',', ' ').split(): - variant_id = item.lstrip('chr') - parsed_variant_id = parse_variant_id(variant_id) - if parsed_variant_id: - parsed_variant_ids.append(parsed_variant_id) - variant_ids.append(variant_id) - else: - invalid_items.append(item) - - return variant_ids, parsed_variant_ids, invalid_items - - -def parse_variant_id(variant_id): - try: - return _parse_valid_variant_id(variant_id) - except (KeyError, ValueError): - return None - - -def _parse_valid_variant_id(variant_id): - chrom, pos, ref, alt = variant_id.split('-') - chrom = format_chrom(chrom) - pos = int(pos) - get_xpos(chrom, pos) - return chrom, pos, ref, alt - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/utils/xpos_utils.py b/seqr/utils/xpos_utils.py index dcab3b78c1..6d06eac680 100644 --- a/seqr/utils/xpos_utils.py +++ b/seqr/utils/xpos_utils.py @@ -80,3 +80,14 @@ def get_chrom_pos(xpos): CHROM_NUMBER_TO_CHROM[chrom_idx], xpos % int(1e9) ) + + +def parse_variant_id(variant_id): + try: + chrom, pos, ref, alt = variant_id.split('-') + chrom = format_chrom(chrom) + pos = int(pos) + get_xpos(chrom, pos) + return chrom, pos, ref, alt + except (KeyError, ValueError): + return None diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index e0c62e77d8..b3ff656ae7 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -15,11 +15,12 @@ from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ - variant_lookup, parse_variant_id, export_variants + variant_lookup, export_variants from seqr.utils.search.utils import InvalidSearchException from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json From 2e0f452899059f4cd9bd4202bc309e8c72af73c9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 16:09:16 -0400 Subject: [PATCH 12/23] fix priority search formatting --- .../commands/tag_seqr_prioritized_variants.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 5ff958aa19..cde133dd29 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -162,7 +162,7 @@ } NO_PANEL_APP_DE_NOVO_SEARCH = { - 'inheritance_mode': DE_NOVO, + 'inheritance': {'mode': DE_NOVO}, 'freqs': { 'callset': {'ac': 100}, 'gnomad_exomes': {'ac': 100}, @@ -181,7 +181,7 @@ 'SNV_INDEL': { 'Clinvar Pathogenic': { 'gene_list_moi': DOMINANT_MOI, - 'inheritance_mode': ANY_AFFECTED, + 'inheritance': {'mode': ANY_AFFECTED}, 'pathogenicity': CLINVAR_FILTER, 'freqs': { 'callset': {'ac': 150}, @@ -190,28 +190,28 @@ }, }, 'Clinvar Pathogenic - Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': {}, 'annotations_secondary': HIGH_MODERATE_ANNOTATIONS, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Both Pathogenic - Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Pathogenic - Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Pathogenic - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True }, **CLINVAR_RECESSIVE_SEARCH, }, 'Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_ANNOTATIONS, 'annotations_secondary': HIGH_MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, @@ -220,7 +220,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, }, @@ -228,7 +228,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: False }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations':{ 'splice_ai': 0.8, }, @@ -244,7 +244,7 @@ **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'Compound Heterozygous - Clinvar Pathogenic/ High Splice AI': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': {}, 'annotations_secondary': { 'splice_ai': 0.5, @@ -255,7 +255,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: False }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.8, @@ -270,7 +270,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.5, @@ -319,17 +319,17 @@ **NO_PANEL_APP_DE_NOVO_SEARCH, }, 'High Splice AI - Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'High Splice AI - Recessive Confirmed': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'High Splice AI - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, **HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, 'family_filter': { @@ -338,7 +338,7 @@ }, }, 'High Splice AI - X-Linked Recessive Confirmed': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, 'family_filter': { @@ -347,12 +347,12 @@ }, }, 'Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, 'annotations': HIGH_MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, }, 'X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True }, @@ -365,7 +365,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, **SV_RECESSIVE_SEARCH, 'qualityFilter': PASS_QUALITY_FILTER, }, @@ -380,11 +380,11 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **SV_RECESSIVE_SEARCH, }, 'SV - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True, CONFIRMED_FAMILY_FILTER: True, @@ -394,7 +394,7 @@ }, 'MITO': { 'Mitochondrial - Pathogenic': { - 'inheritance_mode': ANY_AFFECTED, + 'inheritance': {'mode': ANY_AFFECTED}, 'pathogenicity': {'clinvar': CLINVAR_FILTER['clinvar']}, 'annotations': { 'mitomap_pathogenic': True, @@ -406,7 +406,7 @@ }, 'Mitochondrial - De Novo/ Dominant': { 'gene_list_moi': MITO_MOI, - 'inheritance_mode': DE_NOVO, + 'inheritance': {'mode': DE_NOVO}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'freqs': { 'callset': {'ac': 5000}, @@ -628,7 +628,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by } num_results = cls._execute_search( sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, - inheritance_mode=COMPOUND_HET, **config_search, **ALL_SEARCHES_CRITERIA, genes=genes, + inheritance={'mode': COMPOUND_HET}, **config_search, **ALL_SEARCHES_CRITERIA, genes=genes, ) search_counts[search_name] = num_results From 3e17c2f4467bb80ffd49de0746ef0e0c24066e69 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 16:28:33 -0400 Subject: [PATCH 13/23] move gene parsing logic --- clickhouse_search/managers.py | 10 +++---- clickhouse_search/search.py | 32 ++++++++++++++++++--- clickhouse_search/search_tests.py | 2 +- seqr/utils/search/utils.py | 40 ++++----------------------- seqr/views/utils/orm_to_json_utils.py | 2 +- 5 files changed, 41 insertions(+), 45 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8f5ced7faf..46d6c569d2 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1663,9 +1663,9 @@ def _join_annotations(self, entries): ) return super()._join_annotations(entries) - def filter_locus(self, *args, require_any_gene=False, rawVariantItems=None, intervals=None, genes=None, variant_ids=None, **kwargs): - if rawVariantItems: - parsed_variant_ids = self._parse_variant_ids(rawVariantItems) + def filter_locus(self, *args, require_any_gene=False, raw_variant_items=None, intervals=None, genes=None, variant_ids=None, **kwargs): + if raw_variant_items: + parsed_variant_ids = self._parse_variant_ids(raw_variant_items) invalid_items = [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id] if invalid_items: if variant_ids: @@ -1746,8 +1746,8 @@ def _annotate_calls(self, entries, *args, **kwargs): return entries - def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, rawVariantItems=None, **kwargs): - if exclude_svs or any(self._parse_variant_ids(rawVariantItems).values()): + def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, raw_variant_items=None, **kwargs): + if exclude_svs or any(self._parse_variant_ids(raw_variant_items).values()): raise InvalidDatasetTypeException # SV interval filtering occurs after joining on annotations to correctly incorporate end position can_filter_gene_interval = self._can_filter_gene_interval(genes) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 36cb3f4407..6b6df6a960 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -17,6 +17,7 @@ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization, Family, Individual +from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.logging_utils import SeqrLogger from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ @@ -27,6 +28,7 @@ BATCH_SIZE = 10000 +MAX_GENES_FOR_FILTER = 10000 MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 MIN_MULTI_FAMILY_SEQR_AC = 5000 @@ -35,9 +37,10 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): +def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, locus=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): search['inheritance_filter'] = (inheritance or {}).get('filter') or {} inheritance_mode = None if search['inheritance_filter'].get('genotype') else (inheritance or {}).get('mode') + _parse_locus_search(locus or {}, genome_version, search) has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) @@ -58,6 +61,8 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da continue if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: + if len(search.get('genes') or []) != 1: + raise InvalidSearchException('Including external projects is only available when searching for a single gene') if has_comp_het: raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') logger.info('Looking up variants in projects with no user access', user) @@ -114,6 +119,25 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da return get_sorted_search_results(results, sort, families) +def _parse_locus_search(locus, genome_version, search): + search['raw_variant_items'] = locus.get('rawVariantItems') + + exclude = search.get('exclude') or {} + exclude_locations = bool(exclude.get('rawItems')) + if locus and exclude_locations: + raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') + + genes, intervals, invalid_items = parse_locus_list_items( + locus or exclude, genome_version=genome_version, additional_model_fields=['id'], + ) + if invalid_items: + raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) + if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: + raise InvalidSearchException('Too many genes/intervals') + search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) + exclude.pop('rawItems', None) + + def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, has_location_filter=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects @@ -836,7 +860,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff for dataset_type, entry_cls in sorted(ENTRY_CLASS_MAP[genome_version].items()): try: entry_qs = entry_cls.objects.filter_locus( - variant_ids=[variant_id], rawVariantItems=variant_id, + variant_ids=[variant_id], raw_variant_items=variant_id, ) except InvalidDatasetTypeException: continue @@ -866,7 +890,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: lifted_id = variant_id.replace(pos, str(liftover_results[1])) - entry_qs = lifted_entry_cls.objects.filter_locus(rawVariantItems=lifted_id) + entry_qs = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) variant = _clickhouse_variant_lookup( entry_qs, lifted_genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) @@ -901,7 +925,7 @@ def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_o if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return lifted_id = variant_id.replace(str(variant['pos']), str(variant['liftedOverPos'])) - lifted_entries = lifted_entry_cls.objects.filter_locus(rawVariantItems=lifted_id) + lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 82cc532d0c..6b8188e078 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -847,7 +847,7 @@ def test_variant_id_search(self): ) @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) - @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) + @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2) @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 662014bf5d..835205d287 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -11,14 +11,12 @@ from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY -from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.permissions_utils import user_is_analyst logger = SeqrLogger(__name__) -MAX_GENES_FOR_FILTER = 10000 MIN_MULTI_FAMILY_SEQR_AC = 5000 MAX_EXPORT_VARIANTS = 1000 @@ -43,7 +41,7 @@ def _get_search_genome_version(search_model): def get_single_variant(family, variant_id, user): genome_version = family.project.genome_version - variants = get_clickhouse_variants([family], user, genome_version, rawVariantItems=variant_id, variant_ids=[variant_id]) + variants = get_clickhouse_variants([family], user, genome_version, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return format_clickhouse_results(variants, genome_version)[0] @@ -114,9 +112,12 @@ def _query_variants(search_model, user, sort=None): search = deepcopy(search_model.variant_search.search) families = search_model.families.all() - parsed_search = _parse_search(search, genome_version, user) + exclude_search = {} + exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) + if exclude_previous_hash: + exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - results = get_clickhouse_variants(families, user, genome_version, sort=sort, **parsed_search) + results = get_clickhouse_variants(families, user, genome_version, sort=sort, **search, **exclude_search) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) @@ -124,35 +125,6 @@ def _query_variants(search_model, user, sort=None): return results, genome_version -def _parse_search(search, genome_version, user): - no_access_project_genome_version = search.get('no_access_project_genome_version') - locus = search.pop('locus', None) or {} - exclude = search.get('exclude', None) or {} - exclude_locations = bool(exclude.get('rawItems')) - if locus and exclude_locations: - raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - - parsed_search = {**search} - genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) - if invalid_items: - raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) - if no_access_project_genome_version and len(genes or []) != 1: - raise InvalidSearchException('Including external projects is only available when searching for a single gene') - if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: - raise InvalidSearchException('Too many genes/intervals') - parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) - if not (genes or intervals): - parsed_search.update({'rawVariantItems': locus.get('rawVariantItems')}) - - exclude.pop('rawItems', None) - exclude.pop('previousSearch', None) - exclude_previous_hash = exclude.pop('previousSearchHash', None) - if exclude_previous_hash: - parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user)) - - return parsed_search - - def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) results, _ = _query_variants(previous_search_model, user) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 44a0d4c803..c39e47b609 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -10,7 +10,6 @@ from guardian.shortcuts import get_users_with_perms, get_groups_with_perms import json -from clickhouse_search.search import get_clickhouse_variant_annotations from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ @@ -420,6 +419,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo def _add_clickhouse_annotations(results, genome_version): + from clickhouse_search.search import get_clickhouse_variant_annotations results_by_genome_version_dataset_type = defaultdict(lambda: defaultdict(list)) for result in results: dataset_type = result.pop('datasetType') From 43e9da5501ecb1b80804aca9195bc885ce032233 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 17:07:06 -0400 Subject: [PATCH 14/23] first pass exclude intervals clean up --- clickhouse_search/managers.py | 51 +++++++++++++++-------------------- clickhouse_search/search.py | 15 ++++++++--- seqr/utils/search/utils.py | 1 - 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 46d6c569d2..e679ae945c 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -100,6 +100,7 @@ def _annotate_seqr_pop_expression(self, results): return results.annotate(seqrPop=self.gt_stats_dict.dict_get_expression('key')) def _format_gene_intervals(self, genes): + # TODO deprecate return [ {field: gene[f'{field}Grch{self.genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() ] @@ -421,12 +422,8 @@ def populations(self): def _filter_frequency(self, results, **kwargs): return results - def _filter_annotations(self, results, annotations_filter_q=None, transcript_filters=None, genes=None, intervals=None, exclude_locations=False, **kwargs): - if exclude_locations and genes: - intervals = self._format_gene_intervals(genes) - genes = None - - results = self._filter_locations(results, genes, intervals, exclude_locations=exclude_locations, **kwargs) + def _filter_annotations(self, results, annotations_filter_q=None, transcript_filters=None, genes=None, **kwargs): + results = self._filter_locations(results, genes, **kwargs) if not (annotations_filter_q or transcript_filters): return results @@ -451,7 +448,7 @@ def _annotate_filtered_transcripts(self, results, consequence_field, transcript_ self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=transcript_filters), }) - def _filter_locations(self, results, genes, intervals, **kwargs): + def _filter_locations(self, results, genes, intervals=None, **kwargs): if genes: results = results.annotate(**{ self.GENE_CONSEQUENCE_FIELD: ArrayFilter(self.TRANSCRIPT_FIELD, conditions=[{ @@ -843,7 +840,7 @@ def _parse_annotation_filters(self, annotations, pathogenicity): return filter_qs, transcript_filters - def _filter_locations(self, results, genes, intervals, exclude_locations=False, padded_interval_end=None, **kwargs): + def _filter_locations(self, results, genes, intervals=None, exclude_intervals=None, padded_interval_end=None, **kwargs): results = super()._filter_locations(results, genes, intervals, **kwargs) if genes: @@ -852,7 +849,7 @@ def _filter_locations(self, results, genes, intervals, exclude_locations=False, end, padding = padded_interval_end interval_qs = [Q(end__range=(end - padding, end + padding))] else: - interval_qs = [self._interval_query(**interval) for interval in (intervals or [])] + interval_qs = [self._interval_query(**interval) for interval in (intervals or exclude_intervals or [])] if not interval_qs: return results @@ -860,7 +857,7 @@ def _filter_locations(self, results, genes, intervals, exclude_locations=False, interval_q = interval_qs[0] for q in interval_qs[1:]: interval_q |= q - filter_func = results.exclude if exclude_locations else results.filter + filter_func = results.exclude if exclude_intervals else results.filter return filter_func(interval_q) def _interval_query(self, chrom, start, end, *args, **kwargs): @@ -1441,7 +1438,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_locations=False, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): + def filter_locus(self, exclude_intervals=None, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): entries = self if variant_ids: @@ -1454,11 +1451,13 @@ def filter_locus(self, exclude_locations=False, require_gene_filter=False, inter if intervals and self.filtered_chrom: intervals = [interval for interval in intervals if interval['chrom'] == self.filtered_chrom] - if not intervals and not exclude_locations: + if not intervals: raise InvalidDatasetTypeException if genes or intervals: - entries = self._filter_locations(entries, genes, intervals, exclude_locations=exclude_locations, require_gene_filter=require_gene_filter) + entries = entries.filter(self._filter_locations_q(intervals, genes, require_gene_filter)) + elif exclude_intervals: + entries = entries.exclude(self._filter_locations_q(exclude_intervals)) return entries @@ -1470,10 +1469,10 @@ def _parse_variant_ids(raw_variant_items): parsed_variant_ids[variant_id] = parse_variant_id(variant_id) return parsed_variant_ids - def _filter_locations(self, entries, genes, intervals, exclude_locations=False, require_gene_filter=False): + def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): locus_q = None if genes: - should_filter_interval = self._can_filter_gene_interval(genes) or exclude_locations + should_filter_interval = self._can_filter_gene_interval(genes) if should_filter_interval: intervals = self._format_gene_intervals(genes) + (intervals or []) if require_gene_filter or (not should_filter_interval): @@ -1490,8 +1489,7 @@ def _filter_locations(self, entries, genes, intervals, exclude_locations=False, else: locus_q |= interval_q - filter_func = entries.exclude if exclude_locations else entries.filter - return filter_func(locus_q) + return locus_q def _can_filter_gene_interval(self, genes): return (not hasattr(self.model, 'geneId_ids')) or len(genes) < self.MAX_XPOS_FILTER_INTERVALS or self.filtered_chrom @@ -1746,24 +1744,19 @@ def _annotate_calls(self, entries, *args, **kwargs): return entries - def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, raw_variant_items=None, **kwargs): + def filter_locus(self, *args, exclude_intervals=None, intervals=None, genes=None, exclude_svs=False, raw_variant_items=None, **kwargs): if exclude_svs or any(self._parse_variant_ids(raw_variant_items).values()): raise InvalidDatasetTypeException # SV interval filtering occurs after joining on annotations to correctly incorporate end position can_filter_gene_interval = self._can_filter_gene_interval(genes) - if exclude_locations: - intervals = None - else: - chromosomes = {i['chrom'] for i in (intervals or [])} - if can_filter_gene_interval: - chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) - intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] + chromosomes = {i['chrom'] for i in (intervals or [])} + if can_filter_gene_interval: + chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) + intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] - entries = super().filter_locus( - *args, exclude_locations=exclude_locations, intervals=intervals, genes=genes, **kwargs, - ) + entries = super().filter_locus(*args, intervals=intervals, genes=genes, **kwargs) - if can_filter_gene_interval and not exclude_locations: + if can_filter_gene_interval: entries = entries.filter(calls__array_all={'OR': [ {'geneIds': (list(genes.keys()), 'hasAny({value}, {field})')}, {'gt': (None, 'isNull({field})')}, diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 6b6df6a960..5f9f45f1cb 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -120,8 +120,6 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da def _parse_locus_search(locus, genome_version, search): - search['raw_variant_items'] = locus.get('rawVariantItems') - exclude = search.get('exclude') or {} exclude_locations = bool(exclude.get('rawItems')) if locus and exclude_locations: @@ -134,7 +132,18 @@ def _parse_locus_search(locus, genome_version, search): raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: raise InvalidSearchException('Too many genes/intervals') - search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) + + exclude_intervals = None + if exclude_locations: + exclude_intervals = intervals + # TODO move into db query + exclude_intervals += [ + {field: gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() + ] + + search.update({ + 'genes': genes, 'intervals': intervals, 'exclude_intervals': exclude_intervals, 'raw_variant_items': locus.get('rawVariantItems'), + }) exclude.pop('rawItems', None) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 835205d287..f80b1568b7 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -17,7 +17,6 @@ logger = SeqrLogger(__name__) -MIN_MULTI_FAMILY_SEQR_AC = 5000 MAX_EXPORT_VARIANTS = 1000 From 089749300d0969ae571869a0c46fc0ec7cbbc5dd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 27 Mar 2026 17:07:51 -0400 Subject: [PATCH 15/23] clean up --- clickhouse_search/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 5f9f45f1cb..129c5fb164 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -140,6 +140,8 @@ def _parse_locus_search(locus, genome_version, search): exclude_intervals += [ {field: gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() ] + genes = None + intervals = None search.update({ 'genes': genes, 'intervals': intervals, 'exclude_intervals': exclude_intervals, 'raw_variant_items': locus.get('rawVariantItems'), From 3f7c006a10bd14980e99d1083b65b5c262e8025d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 11:23:07 -0400 Subject: [PATCH 16/23] better gene parsing --- clickhouse_search/managers.py | 10 ++-------- clickhouse_search/search.py | 14 +++++++++----- seqr/utils/gene_utils.py | 4 ++-- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index e679ae945c..c9293b7924 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -99,12 +99,6 @@ def _annotate_seqr_pop_expression(self, results): return results return results.annotate(seqrPop=self.gt_stats_dict.dict_get_expression('key')) - def _format_gene_intervals(self, genes): - # TODO deprecate - return [ - {field: gene[f'{field}Grch{self.genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() - ] - @classmethod def _prediction_expression(cls, model): pred_expressions = [] @@ -1474,7 +1468,7 @@ def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): if genes: should_filter_interval = self._can_filter_gene_interval(genes) if should_filter_interval: - intervals = self._format_gene_intervals(genes) + (intervals or []) + intervals = list((genes or {}).values()) + (intervals or []) if require_gene_filter or (not should_filter_interval): locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) @@ -1751,7 +1745,7 @@ def filter_locus(self, *args, exclude_intervals=None, intervals=None, genes=None can_filter_gene_interval = self._can_filter_gene_interval(genes) chromosomes = {i['chrom'] for i in (intervals or [])} if can_filter_gene_interval: - chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) + chromosomes.update({gene['chrom'] for gene in genes.values()}) intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] entries = super().filter_locus(*args, intervals=intervals, genes=genes, **kwargs) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 129c5fb164..06ed8f02aa 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -126,7 +126,7 @@ def _parse_locus_search(locus, genome_version, search): raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') genes, intervals, invalid_items = parse_locus_list_items( - locus or exclude, genome_version=genome_version, additional_model_fields=['id'], + locus or exclude, genome_version=genome_version, get_genes_func=_get_genes, ) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) @@ -136,10 +136,7 @@ def _parse_locus_search(locus, genome_version, search): exclude_intervals = None if exclude_locations: exclude_intervals = intervals - # TODO move into db query - exclude_intervals += [ - {field: gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() - ] + exclude_intervals += list(genes.values()) genes = None intervals = None @@ -149,6 +146,13 @@ def _parse_locus_search(locus, genome_version, search): exclude.pop('rawItems', None) +def _get_genes(gene_ids, genome_version): + genes = GeneInfo.objects.filter(**{'gene_id__in': gene_ids, f'start_grch{genome_version}__isnull': False}).values( + 'id', 'gene_id', **{field: F(f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}, + ) + return {gene.pop('gene_id'): gene for gene in genes} + + def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, has_location_filter=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects diff --git a/seqr/utils/gene_utils.py b/seqr/utils/gene_utils.py index 1b2b615715..f7b6152147 100644 --- a/seqr/utils/gene_utils.py +++ b/seqr/utils/gene_utils.py @@ -158,7 +158,7 @@ def _process_result(result, gene): return _get_json_for_models(genes, process_result=_process_result, **kwargs) -def parse_locus_list_items(request_json, genome_version=None, **kwargs): +def parse_locus_list_items(request_json, genome_version=None, get_genes_func=get_genes): raw_items = request_json.get('rawItems') if not raw_items: return None, None, None @@ -196,6 +196,6 @@ def parse_locus_list_items(request_json, genome_version=None, **kwargs): gene_symbols_to_ids = get_gene_ids_for_gene_symbols(gene_symbols, genome_version=genome_version) invalid_items += [symbol for symbol in gene_symbols if not gene_symbols_to_ids.get(symbol)] gene_ids.update({gene_ids[0] for gene_ids in gene_symbols_to_ids.values() if len(gene_ids)}) - genes_by_id = get_genes(list(gene_ids), genome_version=genome_version, **kwargs) if gene_ids else {} + genes_by_id = get_genes_func(list(gene_ids), genome_version=genome_version) if gene_ids else {} invalid_items += [gene_id for gene_id in gene_ids if not genes_by_id.get(gene_id)] return genes_by_id, intervals, invalid_items \ No newline at end of file From 8974db108ca5fd00a2b5e8dd4241563db2551364 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 11:40:54 -0400 Subject: [PATCH 17/23] genome version clean up --- clickhouse_search/search.py | 30 +++++++++++++++++++------- seqr/utils/search/utils.py | 42 +++++++++++-------------------------- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 06ed8f02aa..1d5f12e20a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -37,14 +37,16 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, user, genome_version, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, locus=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): +def get_clickhouse_variants(families, user, genome_version=None, sort=None, sample_data_by_dataset_type=None, encode_genotypes_json=False, inheritance=None, locus=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): + genome_version = genome_version or no_access_project_genome_version or _get_search_genome_version(families) + search['inheritance_filter'] = (inheritance or {}).get('filter') or {} inheritance_mode = None if search['inheritance_filter'].get('genotype') else (inheritance or {}).get('mode') _parse_locus_search(locus or {}, genome_version, search) has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} - has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) - has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) + has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(**search) + has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(**search) has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) sample_data_by_dataset_type = sample_data_by_dataset_type or {} results = [] @@ -119,6 +121,20 @@ def get_clickhouse_variants(families, user, genome_version, sort=None, sample_da return get_sorted_search_results(results, sort, families) +def _get_search_genome_version(families): + genome_versions = families.values_list('project__genome_version', flat=True).distinct() + if len (genome_versions) > 1: + project_versions = families.values('project__genome_version').annotate( + projects=ArrayAgg('project__name', distinct=True) + ) + summary = '; '.join( + sorted([f"{agg['project__genome_version']} - {', '.join(sorted(agg['projects']))}" for agg in project_versions])) + raise InvalidSearchException( + f'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: {summary}') + + return genome_versions[0] + + def _parse_locus_search(locus, genome_version, search): exclude = search.get('exclude') or {} exclude_locations = bool(exclude.get('rawItems')) @@ -697,16 +713,16 @@ def _affected_male_families(sample_data, affected_male_family_guids): } -def _is_x_chrom_only(genome_version, genes=None, intervals=None, **kwargs): +def _is_x_chrom_only(genes=None, intervals=None, **kwargs): if not (genes or intervals): return False - return all('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) and all('X' in interval['chrom'] for interval in (intervals or [])) + return all('X' in interval['chrom'] for interval in list((genes or {}).values()) + (intervals or [])) -def _has_x_chrom(genome_version, genes=None, intervals=None, **kwargs): +def _has_x_chrom(genes=None, intervals=None, **kwargs): if not (genes or intervals): return True - return any('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) or any('X' in interval['chrom'] for interval in (intervals or [])) + return any('X' in interval['chrom'] for interval in list((genes or {}).values()) + (intervals or [])) def _prioritized_gene_sort(gene_ids, families): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f80b1568b7..3c2ba38e0b 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -7,7 +7,7 @@ from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Project, VariantSearchResults +from seqr.models import Sample, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY @@ -20,30 +20,11 @@ MAX_EXPORT_VARIANTS = 1000 -def _get_search_genome_version(search_model): - projects = Project.objects.filter(family__in=search_model.families.all()).values_list('genome_version', 'name').distinct() - project_versions = defaultdict(set) - for genome_version, project_name in projects: - project_versions[genome_version].add(project_name) - - if len(project_versions) > 1: - summary = '; '.join( - [f"{build} - {', '.join(sorted(projects))}" for build, projects in sorted(project_versions.items())]) - raise InvalidSearchException( - f'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: {summary}') - - if not project_versions: - return search_model.variant_search.search.get('no_access_project_genome_version') - - return next(iter(project_versions.keys())) - - def get_single_variant(family, variant_id, user): - genome_version = family.project.genome_version - variants = get_clickhouse_variants([family], user, genome_version, raw_variant_items=variant_id, variant_ids=[variant_id]) + variants = get_clickhouse_variants([family], user, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return format_clickhouse_results(variants, genome_version)[0] + return format_clickhouse_results(variants, family.project.genome_version)[0] def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): @@ -68,10 +49,11 @@ def _get_search_cache_key(search_model, sort=None): def export_variants(search_model, user): - search_results, genome_version = _query_variants(search_model, user) + search_results = _query_variants(search_model, user) total_variants = len(search_results) if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + genome_version = search_model.families.first().project.genome_version return format_clickhouse_export_results(search_results, genome_version) @@ -95,18 +77,18 @@ def _get_previous_search_results(search_model, sort): def query_variants(search_model, sort, page, num_results, user): if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(user): sort = PATHOGENICTY_HGMD_SORT_KEY - all_results, genome_version = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) + all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) + genome_version = search_model.families.first().project.genome_version results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results], genome_version) return results_page, len(all_results) def _query_variants(search_model, user, sort=None): - genome_version = _get_search_genome_version(search_model) previous_search_results = _get_previous_search_results(search_model, sort) or {} if previous_search_results: - return previous_search_results, genome_version + return previous_search_results search = deepcopy(search_model.variant_search.search) families = search_model.families.all() @@ -116,17 +98,17 @@ def _query_variants(search_model, user, sort=None): if exclude_previous_hash: exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - results = get_clickhouse_variants(families, user, genome_version, sort=sort, **search, **exclude_search) + results = get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - return results, genome_version + return results def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - results, _ = _query_variants(previous_search_model, user) + results = _query_variants(previous_search_model, user) exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) for variant in results: @@ -149,7 +131,7 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - results, _ = _query_variants(search_model, user) + results = _query_variants(search_model, user) flat_variants = [ v for variants in results for v in (variants if isinstance(variants, list) else [variants]) ] From 93555ff9efb96953e765f3e5d999ef6c9e5977b2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 11:51:08 -0400 Subject: [PATCH 18/23] more genome version clean ip --- clickhouse_search/search.py | 14 +++++++++++--- seqr/utils/search/utils.py | 8 +++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1d5f12e20a..d9c9f1eb40 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -477,8 +477,12 @@ def get_sorted_search_results(results, sort, families): return sorted_results -def format_clickhouse_export_results(results, genome_version): +def format_clickhouse_export_results(results): formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] + if not formatted_results: + return [] + + genome_version = formatted_results[0]['genomeVersion'] keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) details_by_key = { @@ -505,7 +509,11 @@ def format_clickhouse_export_results(results, genome_version): return formatted_results -def format_clickhouse_results(results, genome_version): +def format_clickhouse_results(results): + if not results: + return [] + + genome_version = (results[0] if isinstance(results[0], list) else results)[0]['genomeVersion'] keys_with_no_details = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant } @@ -876,7 +884,7 @@ def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only variant = results.result_values().first() if variant: - variant = format_clickhouse_results([variant], genome_version)[0] + variant = format_clickhouse_results([variant])[0] return variant def _filter_lookup_entries(entries, affected_only, hom_only): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 3c2ba38e0b..bf3c038799 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -24,7 +24,7 @@ def get_single_variant(family, variant_id, user): variants = get_clickhouse_variants([family], user, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return format_clickhouse_results(variants, family.project.genome_version)[0] + return format_clickhouse_results(variants)[0] def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): @@ -53,8 +53,7 @@ def export_variants(search_model, user): total_variants = len(search_results) if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') - genome_version = search_model.families.first().project.genome_version - return format_clickhouse_export_results(search_results, genome_version) + return format_clickhouse_export_results(search_results) def _get_previous_search_results(search_model, sort): @@ -79,8 +78,7 @@ def query_variants(search_model, sort, page, num_results, user): sort = PATHOGENICTY_HGMD_SORT_KEY all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) - genome_version = search_model.families.first().project.genome_version - results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results], genome_version) + results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results]) return results_page, len(all_results) From 616b40385ff97f84dbf4df98e1fca9d98fa03744 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 14:14:19 -0400 Subject: [PATCH 19/23] use queryset; --- seqr/utils/search/utils.py | 4 ++-- seqr/views/apis/variant_search_api.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index bf3c038799..8df369fd04 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -20,8 +20,8 @@ MAX_EXPORT_VARIANTS = 1000 -def get_single_variant(family, variant_id, user): - variants = get_clickhouse_variants([family], user, raw_variant_items=variant_id, variant_ids=[variant_id]) +def get_single_variant(families, variant_id, user): + variants = get_clickhouse_variants(families, user, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return format_clickhouse_results(variants)[0] diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index b3ff656ae7..09f4ccc04c 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -139,7 +139,7 @@ def query_single_variant_handler(request, variant_id): family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(family, variant_id, request.user) + variant = get_single_variant(families, variant_id, request.user) response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) From c9bcb8c65ba7a52970139d9ee08e93e6a96b8638 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 14:31:43 -0400 Subject: [PATCH 20/23] codacy clean up --- clickhouse_search/search.py | 44 ++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index d9c9f1eb40..8e7ce659d2 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -47,7 +47,6 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(**search) has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(**search) - has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) sample_data_by_dataset_type = sample_data_by_dataset_type or {} results = [] searched_dataset_types = set() @@ -56,20 +55,15 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, - annotate_affected_males=has_x_chrom_comp_het or has_x_linked, has_location_filter=has_location_filter, + annotate_affected_males=has_x_chrom_comp_het or has_x_linked, allow_no_samples=bool(no_access_project_genome_version), inheritance_mode=inheritance_mode, **search, ) except InvalidDatasetTypeException: continue if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: - if len(search.get('genes') or []) != 1: - raise InvalidSearchException('Including external projects is only available when searching for a single gene') - if has_comp_het: - raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - logger.info('Looking up variants in projects with no user access', user) - results += _get_search_results( - entry_qs, variants_qs, **search, **parsed_filters, sample_data=None, skip_entry_fields=True, + results += _get_no_access_search_results( + entry_qs, variants_qs, has_comp_het, user, **search, **parsed_filters, exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), inheritance_mode=inheritance_mode, ) searched_dataset_types.add(dataset_type) @@ -94,14 +88,12 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp ) if has_comp_het: - if not any(parsed_filters.values()): - raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') comp_het_sample_data = sample_data if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: comp_het_sample_data = _no_affected_male_families(sample_data, user) result_q = _get_data_type_comp_het_results_queryset( - entry_qs, variants_qs, comp_het_sample_data, exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), - **search, **parsed_filters, + entry_qs, variants_qs, comp_het_sample_data, parsed_filters, + exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), **search, ) if result_q is not None: dataset_results += _evaluate_results(result_q, is_comp_het=True) @@ -169,7 +161,7 @@ def _get_genes(gene_ids, genome_version): return {gene.pop('gene_id'): gene for gene in genes} -def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, has_location_filter=False, **search_kwargs): +def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects parsed_filters = variants_qs.get_parsed_annotations_filters(**search_kwargs) @@ -183,7 +175,7 @@ def _parse_dataset_type_query(genome_version, dataset_type, families, sample_dat inheritance_mode=inheritance_mode, inheritance_filter=inheritance_filter, annotate_affected_males=annotate_affected_males, - has_location_filter=has_location_filter, + has_location_filter=any(search_kwargs.get(field) for field in ['genes', 'intervals', 'variant_ids']), allow_no_samples=allow_no_samples, ) sample_data_by_dataset_type[dataset_type] = sample_data @@ -218,6 +210,15 @@ def _get_search_results(entry_qs, variants_qs, sample_data, skip_entry_fields=Fa return _evaluate_results(results.result_values(skip_entry_fields=skip_entry_fields)) +def _get_no_access_search_results(entry_qs, variants_qs, has_comp_het, user, **search_kwargs): + if len(search_kwargs.get('genes') or []) != 1: + raise InvalidSearchException('Including external projects is only available when searching for a single gene') + if has_comp_het: + raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') + logger.info('Looking up variants in projects with no user access', user) + return _get_search_results(entry_qs, variants_qs, **search_kwargs, sample_data=None, skip_entry_fields=True) + + def _get_x_linked_male_search_results(entry_qs, variants_qs, dataset_type, user, sample_data, **search_kwargs): affected_male_family_guids = { s['family_guid'] for s in sample_data['samples'] if s['affected'] == AFFECTED and s['sex'] in MALE_SEXES @@ -336,7 +337,10 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d return results -def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): +def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, parsed_filters, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): + if not any(parsed_filters.values()): + raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') + parsed_secondary_filters = {} if annotations_secondary: try: @@ -345,11 +349,11 @@ def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, return None entries = entry_qs.search( - sample_data, **search_kwargs, inheritance_mode=COMPOUND_HET, annotate_carriers=True, + sample_data, **search_kwargs, **parsed_filters, inheritance_mode=COMPOUND_HET, annotate_carriers=True, ) - primary_q = variants_qs.subquery_join(entries).search(**search_kwargs) - secondary_kwargs = {**search_kwargs, **parsed_secondary_filters} + primary_q = variants_qs.subquery_join(entries).search(**search_kwargs, **parsed_filters) + secondary_kwargs = {**search_kwargs, **parsed_filters, **parsed_secondary_filters} secondary_q = variants_qs.subquery_join(entries).search(**secondary_kwargs) return _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, sample_data['num_families'], exclude_key_pairs) @@ -925,7 +929,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: from seqr.utils.search.utils import run_liftover - chrom, pos, ref, alt = variant_id.split('-') + chrom, pos, _ = variant_id.split('-', 2) liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: lifted_id = variant_id.replace(pos, str(liftover_results[1])) From d9d2ee62e7d228f672833d51538c3f7ef4c24590 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 14:55:45 -0400 Subject: [PATCH 21/23] codacy clean up --- clickhouse_search/search.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 8e7ce659d2..3d8947e6ba 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -88,15 +88,11 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp ) if has_comp_het: - comp_het_sample_data = sample_data - if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - comp_het_sample_data = _no_affected_male_families(sample_data, user) - result_q = _get_data_type_comp_het_results_queryset( - entry_qs, variants_qs, comp_het_sample_data, parsed_filters, - exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), **search, + dataset_results += _get_data_type_comp_het_results_queryset( + entry_qs, variants_qs, sample_data, user, parsed_filters, **search, + exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), + is_x_chrom=has_x_chrom_comp_het and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS, ) - if result_q is not None: - dataset_results += _evaluate_results(result_q, is_comp_het=True) if 'samples' not in sample_data: _add_individual_guids(dataset_results, encode_genotypes_json=encode_genotypes_json) @@ -337,7 +333,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d return results -def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, parsed_filters, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): +def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, user, parsed_filters, is_x_chrom=False, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): if not any(parsed_filters.values()): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') @@ -346,7 +342,10 @@ def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, try: parsed_secondary_filters = variants_qs.get_parsed_annotations_filters(annotations=annotations_secondary) except InvalidDatasetTypeException: - return None + return [] + + if is_x_chrom and 'affected_male_family_guids' in sample_data: + sample_data = _no_affected_male_families(sample_data, user) entries = entry_qs.search( sample_data, **search_kwargs, **parsed_filters, inheritance_mode=COMPOUND_HET, annotate_carriers=True, @@ -356,7 +355,8 @@ def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, secondary_kwargs = {**search_kwargs, **parsed_filters, **parsed_secondary_filters} secondary_q = variants_qs.subquery_join(entries).search(**secondary_kwargs) - return _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, sample_data['num_families'], exclude_key_pairs) + result_q = _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, sample_data['num_families'], exclude_key_pairs) + return _evaluate_results(result_q, is_comp_het=True) if result_q is not None else [] def _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, num_families, exclude_key_pairs): From 6b490cc85927f3745f3e76e7e2c9f0cbeb518efe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 30 Mar 2026 15:09:19 -0400 Subject: [PATCH 22/23] copilot feedback --- clickhouse_search/search.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 3d8947e6ba..fd554442b0 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -923,16 +923,16 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff entry_qs, genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) if variant: - _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only) + _add_liftover_genotypes(variant, data_type, affected_only, hom_only) else: lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: from seqr.utils.search.utils import run_liftover - chrom, pos, _ = variant_id.split('-', 2) + chrom, pos, base_id = variant_id.split('-', 2) liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: - lifted_id = variant_id.replace(pos, str(liftover_results[1])) + lifted_id = f'{liftover_results[0]}-{liftover_results[1]}-{base_id}' entry_qs = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) variant = _clickhouse_variant_lookup( entry_qs, lifted_genome_version, data_type, affected_only=affected_only, hom_only=hom_only, @@ -963,11 +963,11 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff return variants -def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only): +def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return - lifted_id = variant_id.replace(str(variant['pos']), str(variant['liftedOverPos'])) + lifted_id = f"{variant['liftedOverChrom']}-{variant['liftedOverPos']}-{variant['ref']}-{variant['alt']}" lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() From 2799a58743acb62264ae97bec35108912f255bc1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 15 Apr 2026 15:57:12 -0400 Subject: [PATCH 23/23] fix variant lookup links in vlm --- vlm/match.py | 2 +- vlm/test_vlm.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index 3d2281a5c8..07cbe57e60 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -63,7 +63,7 @@ def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str pos = lifted.position genome_build = liftover_genome_build genome_build = genome_build.replace('GRCh', '') - return f'{SEQR_BASE_URL}summary_data/variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' + return f'{SEQR_BASE_URL}variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 0381198a41..7889cddbeb 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -52,7 +52,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', 'email': None, } ], @@ -124,7 +124,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=37&variantId=7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=37&variantId=7-143270172-A-G', 'email': None, } ], @@ -195,7 +195,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', 'email': None, } ],