From 0e6c7b5a534d9b806d7b0b38f324e010f224bae8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 11:21:23 -0400 Subject: [PATCH 01/71] no call UI toggle --- .../filters/CustomInheritanceFilter.jsx | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx b/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx index b82e317c49..16fbac3414 100644 --- a/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx +++ b/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx @@ -2,12 +2,13 @@ import React from 'react' import PropTypes from 'prop-types' import { connect } from 'react-redux' import { FormSpy } from 'react-final-form' -import { Table, Header, Popup, Loader } from 'semantic-ui-react' +import { Table, Header, Popup, Loader, Divider } from 'semantic-ui-react' import { loadFamilyDetails } from 'redux/rootReducer' -import { getFamiliesByGuid, getIndividualsByGuid, getFamilyDetailsLoading } from 'redux/selectors' +import { getFamiliesByGuid, getIndividualsByGuid, getFamilyDetailsLoading, getElasticsearchEnabled } from 'redux/selectors' import DataLoader from 'shared/components/DataLoader' -import { Select } from 'shared/components/form/Inputs' +import { helpLabel } from 'shared/components/form/FormHelpers' +import { Select, InlineToggle } from 'shared/components/form/Inputs' import PedigreeIcon from 'shared/components/icons/PedigreeIcon' import PedigreeImagePanel from 'shared/components/panel/view-pedigree-image/PedigreeImagePanel' import { AFFECTED, UNAFFECTED, AFFECTED_OPTIONS } from 'shared/utils/constants' @@ -103,14 +104,45 @@ CustomInheritanceFilterContent.propTypes = { individualsByGuid: PropTypes.object, } -const CustomInheritanceFilter = React.memo(({ load, loading, family, ...props }) => { - if (!family) { - return
- } - return ( +const NO_CALL_FIELD = 'allowNoCall' + +const handleChange = (onChange, value) => update => onChange({ ...value, [NO_CALL_FIELD]: update }) + +const NoCallToggle = ({ name, value, onChange, ...props }) => ( + +) + +NoCallToggle.propTypes = { + name: PropTypes.string.isRequired, + value: PropTypes.object, + onChange: PropTypes.func.isRequired, +} + +const CustomInheritanceFilter = React.memo(({ load, loading, family, esEnabled, individualsByGuid, ...props }) => { + const content = family ? ( - + + ) :
+ if (esEnabled) { + return content + } + return ( +
+ {content} + + +
) }) @@ -118,6 +150,7 @@ const mapStateToProps = (state, ownProps) => ({ family: getFamiliesByGuid(state)[ownProps.familyGuid], individualsByGuid: getIndividualsByGuid(state), loading: !!getFamilyDetailsLoading(state)[ownProps.familyGuid], + esEnabled: getElasticsearchEnabled(state), }) const mapDispatchToProps = { @@ -127,7 +160,9 @@ const mapDispatchToProps = { CustomInheritanceFilter.propTypes = { load: PropTypes.func, family: PropTypes.object, + individualsByGuid: PropTypes.object, loading: PropTypes.bool, + esEnabled: PropTypes.bool, } const ConnectedCustomInheritanceFilter = connect(mapStateToProps, mapDispatchToProps)(CustomInheritanceFilter) From ef7281d91a0e48b6e34a650d0fb96c4af077567a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 12:17:44 -0400 Subject: [PATCH 02/71] implement no call override --- clickhouse_search/managers.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7351b83809..35cf71d64f 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -788,7 +788,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit ) if self._has_clinvar() else None inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, - annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {}, + annotate_carriers, inheritance_filter=inheritance_filter or {}, ) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) @@ -821,11 +821,12 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) - def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected): + def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, inheritance_filter): samples_by_genotype = defaultdict(list) affected_samples = [] unaffected_samples = [] family_missing_type_samples = defaultdict(lambda: defaultdict(list)) + custom_affected = inheritance_filter.get('affected') or {} for sample in sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) @@ -849,15 +850,23 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i 'sampleId': (affected_samples, 'has({value}, {field})'), }) elif samples_by_genotype: - if all(len(self.genotype_lookup[genotype]) == 1 for genotype in samples_by_genotype.keys()): - samples_by_gt = {self.genotype_lookup[genotype][0]: samples for genotype, samples in samples_by_genotype.items()} + genotype_lookup = self.genotype_lookup + if inheritance_filter.get('allowNoCall') and inheritance_mode: + unaffected_genotype = self.INHERITANCE_FILTERS.get(inheritance_mode, {}).get(UNAFFECTED) + if unaffected_genotype and -1 not in genotype_lookup[unaffected_genotype]: + genotype_lookup = {**genotype_lookup, unaffected_genotype: [-1] + genotype_lookup[unaffected_genotype]} + if inheritance_mode == X_LINKED_RECESSIVE and -1 not in genotype_lookup[REF_REF]: + genotype_lookup = {**genotype_lookup, REF_REF: [-1] + genotype_lookup[REF_REF]} + + if all(len(genotype_lookup[genotype]) == 1 for genotype in samples_by_genotype.keys()): + samples_by_gt = {genotype_lookup[genotype][0]: samples for genotype, samples in samples_by_genotype.items()} gt_filter_map = ', '.join([f"{gt}, {samples_by_gt.get(gt, [])}" for gt in [-1, 0, 1, 2]]) gt_filter = (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)') else: genotype_sample_map = ', '.join([f"'{genotype or 'any'}', {samples}" for genotype, samples in samples_by_genotype.items()]) gt_genotypes = defaultdict(list) for genotype in samples_by_genotype.keys(): - for gt in self.genotype_lookup[genotype]: + for gt in genotype_lookup[genotype]: gt_genotypes[gt].append(genotype or 'any') gt_genotype_map = ', '.join([f"{gt}, {gt_genotypes[gt]}" for gt in [-1, 0, 1, 2]]) genotype_maps = f'genotype -> map({genotype_sample_map})[genotype], map({gt_genotype_map})' From 980bde229111282e1e93610eed46ca2301fc3625 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 12:43:25 -0400 Subject: [PATCH 03/71] test no call inheritance override --- clickhouse_search/fixtures/clickhouse_search.json | 2 +- clickhouse_search/managers.py | 1 + clickhouse_search/search_tests.py | 6 ++++++ clickhouse_search/test_utils.py | 4 ++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 3260b68c68..046fba62be 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -681,7 +681,7 @@ ], "sign": 1, "calls": [ - ["HG00733", 1, 99, 0.6530612111091614, 49], + ["HG00733", null, 0, null, null], ["HG00731", 2, 48, 1, 16] ] } diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 35cf71d64f..61cd83203e 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -873,6 +873,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i gt_filter = (genotype_maps, 'has(arrayFlatten(arrayMap({value}[ifNull({field}, -1)])), x.sampleId)') inheritance_q = Q(calls__array_all={'gt': gt_filter}) + # TODO allowNoCall quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8aa7186072..99789b8cce 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -498,6 +498,12 @@ def test_inheritance_filter(self): ], ) + self._set_grch37_search() + self._assert_expected_search([], inheritance_mode=inheritance_mode) + self._assert_expected_search( + [GRCH37_VARIANT], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}, + ) + def test_quality_filter(self): quality_filter = {'vcf_filter': 'pass'} self._assert_expected_search( diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 91c001d888..63b2136857 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -388,12 +388,12 @@ 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1, 'filters': ['VQSRTrancheSNP99.90to99.95'], }, 'I000006_hg00733': { 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', - 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, + 'familyGuid': 'F000002_2', 'numAlt': None, 'dp': None, 'gq': 0, 'ab': None, 'filters': ['VQSRTrancheSNP99.90to99.95'], }, }, 'populations': { - 'seqr': {'ac': 3, 'hom': 1}, + 'seqr': {'ac': 2, 'hom': 1}, 'topmed': {'af': 0.52131897, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, 'exac': {'af': 0.63, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.81987739}, 'gnomad_exomes': {'af': 0.63542193, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.82261163}, From d30ac27172369ebf35f371e5a4feaf77f16e3eb6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 13:05:37 -0400 Subject: [PATCH 04/71] no call override quality --- clickhouse_search/managers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 61cd83203e..1cbd7de4ab 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -827,6 +827,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i unaffected_samples = [] family_missing_type_samples = defaultdict(lambda: defaultdict(list)) custom_affected = inheritance_filter.get('affected') or {} + allow_no_call = inheritance_filter.get('allowNoCall') for sample in sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) @@ -851,7 +852,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i }) elif samples_by_genotype: genotype_lookup = self.genotype_lookup - if inheritance_filter.get('allowNoCall') and inheritance_mode: + if allow_no_call and inheritance_mode: unaffected_genotype = self.INHERITANCE_FILTERS.get(inheritance_mode, {}).get(UNAFFECTED) if unaffected_genotype and -1 not in genotype_lookup[unaffected_genotype]: genotype_lookup = {**genotype_lookup, unaffected_genotype: [-1] + genotype_lookup[unaffected_genotype]} @@ -873,8 +874,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i gt_filter = (genotype_maps, 'has(arrayFlatten(arrayMap({value}[ifNull({field}, -1)])), x.sampleId)') inheritance_q = Q(calls__array_all={'gt': gt_filter}) - # TODO allowNoCall - quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) + quality_q = self._quality_q(quality_filter, allow_no_call, affected_samples, clinvar_override_q) return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples @@ -888,7 +888,7 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty genotype = REF_REF return genotype - def _quality_q(self, quality_filter, affected_samples, clinvar_override_q): + def _quality_q(self, quality_filter, allow_no_call, affected_samples, clinvar_override_q): quality_filter_conditions = {} for field, scale, *filters in self.quality_filters: @@ -898,6 +898,8 @@ def _quality_q(self, quality_filter, affected_samples, clinvar_override_q): value = quality_filter.get(filter_key) if value: or_filters = ['isNull({field})', '{field} >= {value}'] + filters + if allow_no_call: + or_filters.append('isNull(x.gt)') quality_filter_conditions[field] = (value / scale, f'or({", ".join(or_filters)})') if not quality_filter_conditions: From a0db6ed19a6c1d9bc2992fe62292e5d081735d80 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 13:09:59 -0400 Subject: [PATCH 05/71] test no call quality filter --- clickhouse_search/search_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 99789b8cce..1032ce32fe 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -586,7 +586,14 @@ def test_quality_filter(self): [VARIANT2, selected_family_3_variant, MITO_VARIANT1], quality_filter=quality_filter, annotations=annotations, pathogenicity={'clinvar': ['pathogenic']}, cached_variant_fields=cached_variant_fields[1:], ) -# + + self._set_grch37_search() + quality_filter = {'min_gq': 1, 'min_ab': 10} + self._assert_expected_search([], quality_filter=quality_filter, annotations=None, pathogenicity=None) + self._assert_expected_search( + [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True}, + ) + def test_location_search(self): self._assert_expected_search( [MULTI_FAMILY_VARIANT, VARIANT4], **LOCATION_SEARCH, cached_variant_fields=[ From 5e5600d3a285488dad140e78cd564bd6f67ce4dc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 14:16:47 -0400 Subject: [PATCH 06/71] add ui option --- ui/pages/Search/constants.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/pages/Search/constants.js b/ui/pages/Search/constants.js index de484732e3..e87c33b7c6 100644 --- a/ui/pages/Search/constants.js +++ b/ui/pages/Search/constants.js @@ -135,6 +135,7 @@ export const INHERITANCE_FILTER_JSON_OPTIONS = INHERITANCE_FILTER_OPTIONS.map( const CLINVAR_NAME = 'clinvar' const CLIVAR_PATH = 'pathogenic' const CLINVAR_LIKELY_PATH = 'likely_pathogenic' +const CLINVAR_CONFLICTING_NOT_BENIGN = 'conflicting_not_benign' const CLINVAR_UNCERTAIN = 'vus_or_conflicting' const CLINVAR_OPTIONS = [ { @@ -145,6 +146,11 @@ const CLINVAR_OPTIONS = [ text: 'Likely Pathogenic (LP)', value: CLINVAR_LIKELY_PATH, }, + { + description: 'Clinvar variant of with conflicting interpretations, at least one of which is not benign or likely benign', + text: 'Conflicting - Not Benign', + value: CLINVAR_CONFLICTING_NOT_BENIGN, + }, { description: 'Clinvar variant of uncertain significance or variant with conflicting interpretations', text: 'VUS or Conflicting', @@ -214,14 +220,14 @@ export const HGMD_PATHOGENICITY_FILTER_OPTIONS = [ { text: 'Pathogenic/ Likely Path.', value: { - [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH], + [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_NOT_BENIGN], [HGMD_NAME]: [HGMD_DM], }, }, { text: 'Not Benign', value: { - [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_UNCERTAIN], + [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_NOT_BENIGN, CLINVAR_UNCERTAIN], [HGMD_NAME]: HGMD_OPTIONS.map(({ value }) => value), }, }, From decce66f1a3c0c20e0a7e8c34fa4e1a788f428e5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 10:20:17 -0400 Subject: [PATCH 07/71] add server side constants --- seqr/utils/search/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py index ca4ee9d9a9..93f068a3ee 100644 --- a/seqr/utils/search/constants.py +++ b/seqr/utils/search/constants.py @@ -63,7 +63,8 @@ CLINVAR_KEY = 'clinvar' CLINVAR_PATH_FILTER = 'pathogenic' CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic' -CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER} +CLINVAR_CONFLICTING_NOT_BENIGN_FILTER = 'conflicting_not_benign' +CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER, CLINVAR_CONFLICTING_NOT_BENIGN_FILTER} PATH_FREQ_OVERRIDE_CUTOFF = 0.05 CLINVAR_PATH_RANGES = [ (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'), From 3467081d0d428f154931f1bf53c1f7fca3cadb9c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 10:27:20 -0400 Subject: [PATCH 08/71] base class implementation of clinvar filter --- clickhouse_search/managers.py | 58 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8f78c87766..bab167511c 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -42,6 +42,31 @@ def _clinvar_tuple(self): output_field=NamedTupleField(list(self.clinvar_fields.values()), null_if_empty=True, null_empty_arrays=True), ) + @classmethod + def _clinvar_path_q(cls, pathogenicity): + clinvar_path_filters = [ + f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES + ] + return cls._clinvar_filter_q(clinvar_path_filters) if clinvar_path_filters else None + + @classmethod + def _clinvar_filter_q(cls, clinvar_filters): + ranges = [[None, None]] + for path_filter, start, end in CLINVAR_PATH_RANGES: + if path_filter in clinvar_filters: + ranges[-1][1] = end + if ranges[-1][0] is None: + ranges[-1][0] = start + elif ranges[-1] != [None, None]: + ranges.append([None, None]) + ranges = [r for r in ranges if r[0] is not None] + + clinvar_qs = [cls._clinvar_range_q(path_range) for path_range in ranges] + clinvar_q = clinvar_qs[0] + for q in clinvar_qs[1:]: + clinvar_q |= q + return clinvar_q + def _seqr_pop_fields(self, seqr_populations): sample_types = [ sample_type.lower() for sample_type in @@ -591,35 +616,10 @@ def _hgmd_filter(hgmd): return ('{field}__classification__range', (min_class, max_class)) return ('{field}__classification__gt', min_class) - @classmethod - def _clinvar_filter_q(cls, clinvar_filters, _get_range_q=None): - ranges = [[None, None]] - for path_filter, start, end in CLINVAR_PATH_RANGES: - if path_filter in clinvar_filters: - ranges[-1][1] = end - if ranges[-1][0] is None: - ranges[-1][0] = start - elif ranges[-1] != [None, None]: - ranges.append([None, None]) - ranges = [r for r in ranges if r[0] is not None] - - clinvar_qs = [(_get_range_q or cls._clinvar_range_q)(path_range) for path_range in ranges] - clinvar_q = clinvar_qs[0] - for q in clinvar_qs[1:]: - clinvar_q |= q - return clinvar_q - @classmethod def _clinvar_range_q(cls, path_range): return Q(clinvar__0__range=path_range, clinvar_key__isnull=False) - @classmethod - def _clinvar_path_q(cls, pathogenicity, _get_range_q=None): - clinvar_path_filters = [ - f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES - ] - return cls._clinvar_filter_q(clinvar_path_filters, _get_range_q=_get_range_q) if clinvar_path_filters else None - def explode_gene_id(self, gene_id_key): consequence_field = self.GENE_CONSEQUENCE_FIELD if self.has_annotation(self.GENE_CONSEQUENCE_FIELD) else self.transcript_field results = self.annotate( @@ -752,6 +752,10 @@ def result_values(self, sample_data=None): def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') + @classmethod + def _clinvar_range_q(cls, path_range): + return Q(clinvar_join__pathogenicity__range=path_range) + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): project_guids = sample_data['project_guids'] project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=project_guids[0]) @@ -783,9 +787,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_filter = qualityFilter or {} individual_genotype_filter = (inheritance_filter or {}).get('genotype') if inheritance_mode or individual_genotype_filter or quality_filter: - clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( - pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), - ) if self._has_clinvar() else None + clinvar_override_q = self._clinvar_path_q(pathogenicity) if self._has_clinvar() else None inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {}, From 7ebbe1be93c4773148b27304566bb66a4f07d6ee Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 10:57:37 -0400 Subject: [PATCH 09/71] implement conflicting override --- clickhouse_search/managers.py | 24 +++++++++++++++++++----- seqr/utils/search/constants.py | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index bab167511c..c6e34acec6 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -14,7 +14,8 @@ from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ - EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, PATH_FREQ_OVERRIDE_CUTOFF, \ + EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, CLINVAR_VUS_FILTER, \ + CLINVAR_CONFLICTING_NOT_BENIGN_FILTER, PATH_FREQ_OVERRIDE_CUTOFF, \ HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS @@ -62,6 +63,11 @@ def _clinvar_filter_q(cls, clinvar_filters): ranges = [r for r in ranges if r[0] is not None] clinvar_qs = [cls._clinvar_range_q(path_range) for path_range in ranges] + + if CLINVAR_CONFLICTING_NOT_BENIGN_FILTER in clinvar_filters: + max_path = next(end for path_filter, _, end in CLINVAR_PATH_RANGES if path_filter == CLINVAR_VUS_FILTER) + clinvar_qs.append(cls._clinvar_conflicting_path_filter({1: (max_path, "{field} <= '{value}'")})) + clinvar_q = clinvar_qs[0] for q in clinvar_qs[1:]: clinvar_q |= q @@ -616,10 +622,14 @@ def _hgmd_filter(hgmd): return ('{field}__classification__range', (min_class, max_class)) return ('{field}__classification__gt', min_class) - @classmethod - def _clinvar_range_q(cls, path_range): + @staticmethod + def _clinvar_range_q(path_range): return Q(clinvar__0__range=path_range, clinvar_key__isnull=False) + @staticmethod + def _clinvar_conflicting_path_filter(conflicting_filter): + return Q(clinvar__5__array_exists=conflicting_filter, clinvar_key__isnull=False) + def explode_gene_id(self, gene_id_key): consequence_field = self.GENE_CONSEQUENCE_FIELD if self.has_annotation(self.GENE_CONSEQUENCE_FIELD) else self.transcript_field results = self.annotate( @@ -752,10 +762,14 @@ def result_values(self, sample_data=None): def _has_clinvar(self): return hasattr(self.model, 'clinvar_join') - @classmethod - def _clinvar_range_q(cls, path_range): + @staticmethod + def _clinvar_range_q(path_range): return Q(clinvar_join__pathogenicity__range=path_range) + @staticmethod + def _clinvar_conflicting_path_filter(conflicting_filter): + return Q(clinvar_join__conflicting_pathogenicities__array_exists=conflicting_filter) + def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs): project_guids = sample_data['project_guids'] project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=project_guids[0]) diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py index 93f068a3ee..0f8562c558 100644 --- a/seqr/utils/search/constants.py +++ b/seqr/utils/search/constants.py @@ -64,12 +64,13 @@ CLINVAR_PATH_FILTER = 'pathogenic' CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic' CLINVAR_CONFLICTING_NOT_BENIGN_FILTER = 'conflicting_not_benign' +CLINVAR_VUS_FILTER = 'vus_or_conflicting' CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER, CLINVAR_CONFLICTING_NOT_BENIGN_FILTER} PATH_FREQ_OVERRIDE_CUTOFF = 0.05 CLINVAR_PATH_RANGES = [ (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'), (CLINVAR_LIKELY_PATH_FILTER, 'Pathogenic/Likely_pathogenic', 'Likely_risk_allele'), - ('vus_or_conflicting', 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), + (CLINVAR_VUS_FILTER, 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), ('likely_benign', 'Likely_benign', 'Benign/Likely_benign'), ('benign', 'Benign/Likely_benign', 'Benign'), ] From 93f7481df0bc12d5a5e8688e6222fd813afd8dde Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 11:00:12 -0400 Subject: [PATCH 10/71] update saved searches --- seqr/fixtures/variant_searches.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/seqr/fixtures/variant_searches.json b/seqr/fixtures/variant_searches.json index 53529b97e3..8b3298d298 100644 --- a/seqr/fixtures/variant_searches.json +++ b/seqr/fixtures/variant_searches.json @@ -80,7 +80,8 @@ ], "clinvar": [ "pathogenic", - "likely_pathogenic" + "likely_pathogenic", + "conflicting_not_benign" ] }, "qualityFilter": { @@ -175,7 +176,8 @@ ], "clinvar": [ "pathogenic", - "likely_pathogenic" + "likely_pathogenic", + "conflicting_not_benign" ] }, "qualityFilter": { @@ -321,6 +323,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", + "conflicting_not_benign", "vus_or_conflicting" ] }, @@ -422,6 +425,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", + "conflicting_not_benign", "vus_or_conflicting" ] }, From 37a5eb0d9fea8afd6af396fce4d9caa7e7a82aed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 12:32:07 -0400 Subject: [PATCH 11/71] unit tests --- .../fixtures/clickhouse_search.json | 2 +- clickhouse_search/search_tests.py | 20 ++++++++++++++----- clickhouse_search/test_utils.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 3260b68c68..3b89605e6d 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -48,7 +48,7 @@ "populations": [ [35805, 0.29499999, 121372, 0.41530353, 0, 24061, 5872], [72672, 0.28899795, 251462, 0.4116475, 0, 11567], - [0, 0, 0, 0, 0, 0], + [927, 0.03444932, 26912, 0.04027665, 0, 0], [65154, 0.246152, 264690, 47604, 8775] ], "sorted_transcript_consequences": [ diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 5869c0924d..9ff251e3bd 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -571,14 +571,22 @@ def test_quality_filter(self): {}, ] + quality_filter['min_gq'] = 50 + self._assert_expected_search( + [VARIANT1, selected_family_3_variant, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter, + annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']}, + cached_variant_fields=cached_variant_fields[:1] + cached_variant_fields[2:], + ) + self._assert_expected_search( [VARIANT1, VARIANT2, selected_family_3_variant, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter, - annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']}, cached_variant_fields=cached_variant_fields, + annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'conflicting_not_benign', 'vus_or_conflicting']}, + cached_variant_fields=cached_variant_fields, ) self._assert_expected_search( [VARIANT2, selected_family_3_variant, MITO_VARIANT1], quality_filter=quality_filter, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic']}, cached_variant_fields=cached_variant_fields[1:], + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_not_benign']}, cached_variant_fields=cached_variant_fields[1:], ) # def test_location_search(self): @@ -790,14 +798,14 @@ def test_frequency_filter(self): annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT4, MITO_VARIANT1], + [VARIANT1, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.01, 'hh': 10}, 'gnomad_mito': {'af': 0.01}}, annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting']}, ) self._assert_expected_search( [VARIANT2, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.01}, 'gnomad_mito': {'af': 0.01}}, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'vus_or_conflicting']}, + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_not_benign', 'vus_or_conflicting']}, ) def test_annotations_filter(self): @@ -809,6 +817,8 @@ def test_annotations_filter(self): [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, ) + self._assert_expected_search([VARIANT2], pathogenicity={'clinvar': ['conflicting_not_benign']}) + exclude = {'clinvar': pathogenicity['clinvar'][1:]} pathogenicity['clinvar'] = pathogenicity['clinvar'][:1] snv_38_only_annotations = {'SCREEN': ['CTCF-only', 'DNase-only'], 'UTRAnnotator': ['5_prime_UTR_stop_codon_loss_variant']} @@ -1238,7 +1248,7 @@ def test_sort(self): ) self._assert_expected_search( - [VARIANT2, MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT1, MITO_VARIANT3, VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], + [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT1, VARIANT2, MITO_VARIANT3, VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='gnomad', ) diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 91c001d888..6e4e5e16e3 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -128,7 +128,7 @@ 'topmed': {'af': 0.246152, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, 'exac': {'af': 0.29499999, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.41530353}, 'gnomad_exomes': {'af': 0.28899795, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.4116475}, - 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0}, + 'gnomad_genomes': {'af': 0.03444932, 'ac': 927, 'an': 26912, 'hom': 0, 'hemi': 0, 'filter_af': 0.04027665}, }, 'predictions': { 'cadd': 20.9, From d1fa127d109815b58e6d98178b55a8bc07a8f1ef Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 14:28:11 -0400 Subject: [PATCH 12/71] add ui filter --- ui/shared/utils/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 74991660a0..8ff670b583 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1505,7 +1505,7 @@ const REVERSE_PRED_COLOR_MAP = [...PRED_COLOR_MAP].reverse() export const ORDERED_PREDICTOR_FIELDS = [ { field: 'cadd', group: CODING_IN_SILICO_GROUP, thresholds: [0.151, 22.8, 25.3, 28.1, undefined], min: 1, max: 99, fieldTitle: 'CADD', requiresCitation: true }, { field: 'revel', group: MISSENSE_IN_SILICO_GROUP, thresholds: [0.0161, 0.291, 0.644, 0.773, 0.932], fieldTitle: 'REVEL', requiresCitation: true }, - { field: 'alphamissense', fieldTitle: 'AlphaMissense', displayOnly: true }, + { field: 'alphamissense', fieldTitle: 'AlphaMissense', group: MISSENSE_IN_SILICO_GROUP }, { field: 'vest', thresholds: [undefined, 0.45, 0.764, 0.861, 0.965], fieldTitle: 'VEST', requiresCitation: true }, { field: 'mpc', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, undefined, 1.36, 1.828, undefined], max: 5, fieldTitle: 'MPC' }, { From 5ca3196ab177c8d1796c20be1cb8cc43f55c9139 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 14:58:24 -0400 Subject: [PATCH 13/71] implement alphamissense filter --- clickhouse_search/managers.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8f78c87766..3ea203d88d 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -393,7 +393,12 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): return results def _filter_in_silico(self, results, in_silico=None, **kwargs): + require_score = in_silico.get('requireScore', False) + in_silico_q = None + if (in_silico or {}).get('alphamissense'): + in_silico_q = self._alphamissense_q(in_silico['alphamissense'], require_score) + for score, value in (in_silico or {}).items(): score_q = self._get_in_silico_score_q(score, value) if in_silico_q is None: @@ -401,7 +406,7 @@ def _filter_in_silico(self, results, in_silico=None, **kwargs): elif score_q: in_silico_q |= score_q - if in_silico_q and not in_silico.get('requireScore', False): + if in_silico_q and not require_score: in_silico_q |= Q(**{f'predictions__{score}__isnull': True for score in in_silico.keys() if score in self.prediction_fields}) if in_silico_q: @@ -421,6 +426,14 @@ def _get_in_silico_score_q(self, score, value): except ValueError: return Q(**{score_column: value}) + def _alphamissense_q(self, value, require_score): + if not ('alphamissensePathogenicity' in self.transcript_fields and value): + return None + q = Q(**{f'{self.transcript_field}__array_exists': {'alphamissensePathogenicity': (value, '{value} <= {field}')}}) + if not require_score: + q |= Q(**{f'{self.transcript_field}__array_all': {'alphamissensePathogenicity': (None, 'isNull({field})')}}) + return q + def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, genes=None, intervals=None, exclude_locations=False, padded_interval_end=None, **kwargs): transcript_field = self.transcript_field if self.model.GENOTYPE_OVERRIDE_FIELDS: From d2c581911694f639aa47c8efee3bf1e72bdc4b6a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 15:00:40 -0400 Subject: [PATCH 14/71] score filter hover --- ui/shared/utils/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 8ff670b583..4195f8e4b7 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1505,7 +1505,7 @@ const REVERSE_PRED_COLOR_MAP = [...PRED_COLOR_MAP].reverse() export const ORDERED_PREDICTOR_FIELDS = [ { field: 'cadd', group: CODING_IN_SILICO_GROUP, thresholds: [0.151, 22.8, 25.3, 28.1, undefined], min: 1, max: 99, fieldTitle: 'CADD', requiresCitation: true }, { field: 'revel', group: MISSENSE_IN_SILICO_GROUP, thresholds: [0.0161, 0.291, 0.644, 0.773, 0.932], fieldTitle: 'REVEL', requiresCitation: true }, - { field: 'alphamissense', fieldTitle: 'AlphaMissense', group: MISSENSE_IN_SILICO_GROUP }, + { field: 'alphamissense', fieldTitle: 'AlphaMissense', group: MISSENSE_IN_SILICO_GROUP, thresholds: [0.34, 0.34, 0.564, 0.564] }, { field: 'vest', thresholds: [undefined, 0.45, 0.764, 0.861, 0.965], fieldTitle: 'VEST', requiresCitation: true }, { field: 'mpc', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, undefined, 1.36, 1.828, undefined], max: 5, fieldTitle: 'MPC' }, { From b54743bf141293e2260ca2621206f82a9662ed94 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 15:02:23 -0400 Subject: [PATCH 15/71] clean up --- clickhouse_search/managers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 3ea203d88d..7d81b7367f 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -393,13 +393,14 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): return results def _filter_in_silico(self, results, in_silico=None, **kwargs): + in_silico = in_silico or {} require_score = in_silico.get('requireScore', False) in_silico_q = None - if (in_silico or {}).get('alphamissense'): + if in_silico.get('alphamissense'): in_silico_q = self._alphamissense_q(in_silico['alphamissense'], require_score) - for score, value in (in_silico or {}).items(): + for score, value in in_silico.items(): score_q = self._get_in_silico_score_q(score, value) if in_silico_q is None: in_silico_q = score_q From 0fa57ab867344fbc06e3d077888ac7aa1de3ef52 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 15:12:34 -0400 Subject: [PATCH 16/71] unit tests --- clickhouse_search/fixtures/clickhouse_search.json | 2 +- clickhouse_search/fixtures/clickhouse_transcripts.json | 2 +- clickhouse_search/search_tests.py | 8 +++++++- clickhouse_search/test_utils.py | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 3260b68c68..ff6e3f42b2 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -124,7 +124,7 @@ ], "sorted_transcript_consequences": [ [null, null, ["splice_donor_variant"], true, null, "ENSG00000097046"], - [null, 1, ["missense_variant"], false, null, "ENSG00000097046"], + [0.1, 1, ["missense_variant"], false, null, "ENSG00000097046"], [null, null, ["missense_variant"], false, null, "ENSG00000097046"] ], "sorted_motif_feature_consequences": [ diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json index e7a3352963..27894fd71a 100644 --- a/clickhouse_search/fixtures/clickhouse_transcripts.json +++ b/clickhouse_search/fixtures/clickhouse_transcripts.json @@ -113,7 +113,7 @@ ], null] ], [ - [null], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046", + [0.1], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046", "ENST00000350997.12:c.375+139G>A", null, [1, 11], [null, []], "missense_variant", null, "NM_013402.7", "NM_013402.7", [false], "ENST00000350997", 1, [null, null, null, [ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 5869c0924d..673197e8e6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -1188,6 +1188,12 @@ def test_in_silico_filter(self): [VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'requireScore': True}, ) + self._assert_expected_search( + [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + in_silico={'alphamissense': 0.5}, + ) + self._assert_expected_search([VARIANT2], in_silico={'alphamissense': 0.5, 'requireScore': True}) + sv_in_silico = {'strvctvre': 0.1, 'requireScore': True} self._assert_expected_search( [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], in_silico=sv_in_silico, @@ -1267,7 +1273,7 @@ def test_sort(self): ) self._assert_expected_search( - [VARIANT2, VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='alphamissense', + [VARIANT2, VARIANT4, VARIANT1, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='alphamissense', ) sort = 'in_omim' diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 91c001d888..cfd50d200b 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -298,7 +298,7 @@ 'transcripts': { 'ENSG00000097046': [ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000466716.5:c.-264+1G>A', 'hgvsp': None, 'transcriptId': 'ENST00000466716', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': ['5UTR_SPLICE']}, 'spliceregion': {'extended_intronic_splice_region_variant': True}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'splice_donor_variant', 'consequenceTerms': ['splice_donor_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1}, + {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': 0.1}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1}, {'aminoAcids': 'T/I', 'canonical': None, 'codons': 'aCc/aTc', 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'transcriptId': 'ENST00000257261', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 12}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001281501.1', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 2}, ], }, @@ -1250,7 +1250,7 @@ 'fiveutrConsequence': None, 'geneId': 'ENSG00000097046', }, { - 'alphamissensePathogenicity': None, + 'alphamissensePathogenicity': 0.1, 'canonical': 1, 'consequenceTerms': ['missense_variant'], 'extendedIntronicSpliceRegionVariant': False, From a0cf1b2d8e847b15e306716cde11e3ded6eb4b1c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 15:55:34 -0400 Subject: [PATCH 17/71] add exclude search toggle to ui --- .../VariantSearchFormPanelConfigs.jsx | 27 +++++++++++++++++++ ui/pages/Search/selectors.js | 11 ++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx index 9493a7936e..02f3e5d956 100644 --- a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx +++ b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx @@ -1,4 +1,5 @@ import React from 'react' +import PropTypes from 'prop-types' import styled from 'styled-components' import { Form, Grid, Header, List, Loader, Table } from 'semantic-ui-react' @@ -529,6 +530,24 @@ export const QUALITY_PANEL = { fieldProps: { control: LazyLabeledSlider, format: val => val || null }, } +const handleChange = (onChange, value) => shouldExclude => onChange({ ...value, shouldExclude }) + +const ExcludeSearchToggle = ({ name, value, onChange, ...props }) => ( + +) + +ExcludeSearchToggle.propTypes = { + name: PropTypes.string, + value: PropTypes.object, + onChange: PropTypes.func, +} + const ES_EXCLUDE_FIELDS = [ { ...BASE_LOCUS_FIELD, @@ -537,6 +556,14 @@ const ES_EXCLUDE_FIELDS = [ }, ] const EXCLUDE_FIELDS = [ + { + name: 'previousSearch', + label: 'Exclude Previous Search Results', + labelHelp: 'Exclude any results returned by the current search from the next search results', + color: 'grey', + component: ExcludeSearchToggle, + width: 8, + }, { ...CLINVAR_FIELD, ...PATHOGENICITY_FIELD_PROPS, diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 52c59c9183..9c78a127c9 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -130,11 +130,18 @@ const getIntitialProjectFamilies = createProjectFamiliesSelector( ) export const getIntitialSearch = createSelector( + getCurrentSearchHash, getCurrentSearchParams, getIntitialProjectFamilies, - (searchParams, projectFamilies) => { + (searchHash, searchParams, projectFamilies) => { if (searchParams) { - return searchParams + return { + ...searchParams, + search: { + ...(searchParams.search || {}), + exclude: { ...(searchParams.search || {}).exclude || {}, previousSearch: { searchHash } }, + }, + } } return projectFamilies ? { projectFamilies: [projectFamilies] } : null From a32d96f187c4eb81cde0a909f38fefd813baad8e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 16:13:27 -0400 Subject: [PATCH 18/71] first pass exclude results --- clickhouse_search/managers.py | 5 ++++- seqr/utils/search/utils.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 8f78c87766..eed679c4a3 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -702,9 +702,12 @@ def clinvar_model(self): def genome_version(self): return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] - def search(self, sample_data, freqs=None, annotations=None, **kwargs): + def search(self, sample_data, freqs=None, annotations=None, exclude_keys=None, **kwargs): entries = self.filter_locus(**kwargs) + if exclude_keys: + entries = entries.exclude(key__in=exclude_keys) + entries = self._join_annotations(entries) is_sv_class = 'cn' in self.call_fields diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 02d0c5508f..de44f1fe54 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -7,7 +7,7 @@ from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Individual, Project +from seqr.models import Sample, Individual, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ @@ -293,6 +293,10 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if duplicates: raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') + exclude_previous = exclude.pop('previousSearch', None) + if (exclude_previous or {}).get('shouldExclude'): + parsed_search['exclude_keys'] = backend_specific_call(lambda *args: None, _get_clickhouse_search_keys)(exclude_previous['searchHash']) + for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} @@ -323,6 +327,14 @@ def _query_variants(search_model, user, previous_search_results, genome_version, return variant_results, previous_search_results.get('total_results') +def _get_clickhouse_search_keys(search_hash): + previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) + cached_results = _get_any_sort_cached_results(previous_search_model) + if cached_results: + return {v['key'] for variants in cached_results['all_results'] for v in (variants if isinstance(variants, list) else [variants])} + raise NotImplementedError # TODO + + def get_variant_query_gene_counts(search_model, user): return backend_specific_call( _get_es_variant_query_gene_counts, From 586d60d6d325be14f4bf7a7a92fe8bc444ee013d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 16:46:46 -0400 Subject: [PATCH 19/71] handle comp hets and multi data type --- clickhouse_search/search.py | 25 +++++++++++++++---------- seqr/utils/search/utils.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 061d2c63b0..4a4fca0d62 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -35,6 +35,8 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno family_guid = None inheritance_mode = search.get('inheritance_mode') has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} + exclude_keys = search.pop('exclude_keys', None) or {} + exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} for dataset_type, sample_data in sample_data_by_dataset_type.items(): logger.info(f'Loading {dataset_type} data for {len(set(sample_data["family_guids"]))} families', user) @@ -45,13 +47,13 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno dataset_results = [] if inheritance_mode != COMPOUND_HET: - result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=is_multi_project, **search) + result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=is_multi_project, exclude_keys=exclude_keys.get(dataset_type), **search) dataset_results += _evaluate_results(result_q) if has_comp_het: comp_het_sample_data = sample_data if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(genome_version, **search): comp_het_sample_data = _no_affected_male_families(sample_data, user) - result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, skip_individual_guid=is_multi_project, **search) + result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), skip_individual_guid=is_multi_project, **search) dataset_results += _evaluate_results(result_q, is_comp_het=True) if is_multi_project: @@ -61,7 +63,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, **search) + results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, **search) cache_results = get_clickhouse_cache_results(results, sort, family_guid) previous_search_results.update(cache_results) @@ -84,7 +86,7 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -125,7 +127,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b sv_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][sv_dataset_type] sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations) - result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families)) + result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families), exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}')) dataset_results = _evaluate_results(result_q, is_comp_het=True) if skip_individual_guid: _add_individual_guids(dataset_results, sv_sample_data) @@ -134,7 +136,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b return results -def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, annotations=None, annotations_secondary=None, inheritance_mode=None, exclude_key_pairs=None, **search_kwargs): entries = entry_cls.objects.search( sample_data, **search_kwargs, inheritance_mode=COMPOUND_HET, annotations=annotations, annotate_carriers=True, ) @@ -144,10 +146,10 @@ def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_ annotations=annotations_secondary or annotations, **search_kwargs, ) - return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(set(sample_data['family_guids']))) + return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(set(sample_data['family_guids'])), exclude_key_pairs) -def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_families): +def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_families, exclude_key_pairs): results = annotations_cls.objects.search_compound_hets(primary_q, secondary_q) if results.has_annotation('primary_carriers') and results.has_annotation('secondary_carriers'): @@ -207,9 +209,12 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ **genotype_expressions, ) - return results.annotate( + pair_results = results.annotate( pair_key=ArraySort(Array('primary_key', 'secondary_key')), - ).distinct('pair_key').values_list( + ).distinct('pair_key') + if exclude_key_pairs: + pair_results = pair_results.exclude(pair_key__in=exclude_key_pairs) + return pair_results.values_list( 'pair_key', _result_as_tuple(results, 'primary_'), _result_as_tuple(results, 'secondary_'), diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index de44f1fe54..99c82a33b4 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -295,7 +295,9 @@ def _query_variants(search_model, user, previous_search_results, genome_version, exclude_previous = exclude.pop('previousSearch', None) if (exclude_previous or {}).get('shouldExclude'): - parsed_search['exclude_keys'] = backend_specific_call(lambda *args: None, _get_clickhouse_search_keys)(exclude_previous['searchHash']) + parsed_search.update(backend_specific_call( + lambda *args: {}, _get_clickhouse_exclude_keys, + )(exclude_previous['searchHash'], user, genome_version)) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): @@ -327,12 +329,34 @@ def _query_variants(search_model, user, previous_search_results, genome_version, return variant_results, previous_search_results.get('total_results') -def _get_clickhouse_search_keys(search_hash): +def _get_clickhouse_exclude_keys(search_hash, user, genome_version): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) cached_results = _get_any_sort_cached_results(previous_search_model) if cached_results: - return {v['key'] for variants in cached_results['all_results'] for v in (variants if isinstance(variants, list) else [variants])} - raise NotImplementedError # TODO + results = cached_results['all_results'] + else: + results, _ = _query_variants(previous_search_model, user, {}, genome_version) + exclude_keys = defaultdict(list) + exclude_key_pairs = defaultdict(list) + for variant in results: + if isinstance(variant, list): + dt1, key1 = _parse_variant_key(variant[0]) + dt2, key2 = _parse_variant_key(variant[1]) + dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) + exclude_key_pairs[dataset_type].append(sorted([key1, key2])) + else: + dataset_type, key = _parse_variant_key(variant) + exclude_keys[dataset_type].append(key) + return {'exclude_keys': exclude_keys, 'exclude_key_pairs': exclude_key_pairs} + + +def _parse_variant_key(variant): + parsed_variant_id = parse_variant_id(variant['variantId']) + dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] + if dataset_type == Sample.DATASET_TYPE_SV_CALLS: + sample_type = next(g['sampleType'] for g in variant['genotypes'].values()) + dataset_type = f'{dataset_type}_{sample_type}' + return dataset_type, variant['key'] def get_variant_query_gene_counts(search_model, user): From f403bdf35ec68dd611555db22e1e6c6a0f49fecb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 19 Sep 2025 16:52:22 -0400 Subject: [PATCH 20/71] share utility method --- seqr/utils/search/utils.py | 22 ++++++++++------------ seqr/views/utils/variant_utils.py | 11 ++--------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 99c82a33b4..d9b12803f0 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -340,23 +340,21 @@ def _get_clickhouse_exclude_keys(search_hash, user, genome_version): exclude_key_pairs = defaultdict(list) for variant in results: if isinstance(variant, list): - dt1, key1 = _parse_variant_key(variant[0]) - dt2, key2 = _parse_variant_key(variant[1]) + dt1= variant_dataset_type(variant[0]) + dt2 = variant_dataset_type(variant[1]) dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) - exclude_key_pairs[dataset_type].append(sorted([key1, key2])) + exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']])) else: - dataset_type, key = _parse_variant_key(variant) - exclude_keys[dataset_type].append(key) + dataset_type = variant_dataset_type(variant) + exclude_keys[dataset_type].append(variant['key']) return {'exclude_keys': exclude_keys, 'exclude_key_pairs': exclude_key_pairs} -def _parse_variant_key(variant): - parsed_variant_id = parse_variant_id(variant['variantId']) - dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - sample_type = next(g['sampleType'] for g in variant['genotypes'].values()) - dataset_type = f'{dataset_type}_{sample_type}' - return dataset_type, variant['key'] +def variant_dataset_type(variant): + if not parse_variant_id(variant['variantId']): + sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES + return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS def get_variant_query_gene_counts(search_model, user): diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 1371b142de..c16327b842 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -16,7 +16,7 @@ from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids -from seqr.utils.search.utils import backend_specific_call, parse_variant_id +from seqr.utils.search.utils import backend_specific_call, variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos @@ -94,7 +94,7 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): update_json = { 'key': variant_json['key'], 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': _dataset_type(variant_id, variant_json), + 'dataset_type': variant_dataset_type(variant_json), } else: update_json = {'saved_variant_json': variant_json} @@ -108,13 +108,6 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): }, update_json -def _dataset_type(variant_id, variant): - if not parse_variant_id(variant_id): - sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES - return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS - - def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=False): all_family_ids = {family_id for family_id, _ in family_variant_data.keys()} all_variant_ids = {variant_id for _, variant_id in family_variant_data.keys()} From 9af6f899840e78b769d85bd80bcda330b7163095 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 10:26:51 -0400 Subject: [PATCH 21/71] todo --- ui/pages/Search/selectors.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 9c78a127c9..727f4eaeeb 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -135,6 +135,7 @@ export const getIntitialSearch = createSelector( getIntitialProjectFamilies, (searchHash, searchParams, projectFamilies) => { if (searchParams) { + // TODO does not work when search is edited - use form spy? return { ...searchParams, search: { From 469b9394db05379775ad5aa78323887beb45ebf6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 10:33:24 -0400 Subject: [PATCH 22/71] override up to likely path --- clickhouse_search/managers.py | 8 ++++---- clickhouse_search/search_tests.py | 8 ++++---- seqr/fixtures/variant_searches.json | 8 ++++---- seqr/utils/search/constants.py | 7 +++---- ui/pages/Search/constants.js | 12 ++++++------ 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index c6e34acec6..131d1635e8 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -14,8 +14,8 @@ from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ - EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, CLINVAR_VUS_FILTER, \ - CLINVAR_CONFLICTING_NOT_BENIGN_FILTER, PATH_FREQ_OVERRIDE_CUTOFF, \ + EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, CLINVAR_LIKELY_PATH_FILTER, \ + CLINVAR_CONFLICTING_P_LP, PATH_FREQ_OVERRIDE_CUTOFF, \ HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS @@ -64,8 +64,8 @@ def _clinvar_filter_q(cls, clinvar_filters): clinvar_qs = [cls._clinvar_range_q(path_range) for path_range in ranges] - if CLINVAR_CONFLICTING_NOT_BENIGN_FILTER in clinvar_filters: - max_path = next(end for path_filter, _, end in CLINVAR_PATH_RANGES if path_filter == CLINVAR_VUS_FILTER) + if CLINVAR_CONFLICTING_P_LP in clinvar_filters: + max_path = next(end for path_filter, _, end in CLINVAR_PATH_RANGES if path_filter == CLINVAR_LIKELY_PATH_FILTER) clinvar_qs.append(cls._clinvar_conflicting_path_filter({1: (max_path, "{field} <= '{value}'")})) clinvar_q = clinvar_qs[0] diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 9ff251e3bd..87486f16ad 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -580,13 +580,13 @@ def test_quality_filter(self): self._assert_expected_search( [VARIANT1, VARIANT2, selected_family_3_variant, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter, - annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'conflicting_not_benign', 'vus_or_conflicting']}, + annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'conflicting_p_lp', 'vus_or_conflicting']}, cached_variant_fields=cached_variant_fields, ) self._assert_expected_search( [VARIANT2, selected_family_3_variant, MITO_VARIANT1], quality_filter=quality_filter, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_not_benign']}, cached_variant_fields=cached_variant_fields[1:], + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp']}, cached_variant_fields=cached_variant_fields[1:], ) # def test_location_search(self): @@ -805,7 +805,7 @@ def test_frequency_filter(self): self._assert_expected_search( [VARIANT2, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.01}, 'gnomad_mito': {'af': 0.01}}, - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_not_benign', 'vus_or_conflicting']}, + annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus_or_conflicting']}, ) def test_annotations_filter(self): @@ -817,7 +817,7 @@ def test_annotations_filter(self): [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, ) - self._assert_expected_search([VARIANT2], pathogenicity={'clinvar': ['conflicting_not_benign']}) + self._assert_expected_search([VARIANT2], pathogenicity={'clinvar': ['conflicting_p_lp']}) exclude = {'clinvar': pathogenicity['clinvar'][1:]} pathogenicity['clinvar'] = pathogenicity['clinvar'][:1] diff --git a/seqr/fixtures/variant_searches.json b/seqr/fixtures/variant_searches.json index 8b3298d298..117c0fb012 100644 --- a/seqr/fixtures/variant_searches.json +++ b/seqr/fixtures/variant_searches.json @@ -81,7 +81,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", - "conflicting_not_benign" + "conflicting_p_lp" ] }, "qualityFilter": { @@ -177,7 +177,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", - "conflicting_not_benign" + "conflicting_p_lp" ] }, "qualityFilter": { @@ -323,7 +323,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", - "conflicting_not_benign", + "conflicting_p_lp", "vus_or_conflicting" ] }, @@ -425,7 +425,7 @@ "clinvar": [ "pathogenic", "likely_pathogenic", - "conflicting_not_benign", + "conflicting_p_lp", "vus_or_conflicting" ] }, diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py index 0f8562c558..c276793263 100644 --- a/seqr/utils/search/constants.py +++ b/seqr/utils/search/constants.py @@ -63,14 +63,13 @@ CLINVAR_KEY = 'clinvar' CLINVAR_PATH_FILTER = 'pathogenic' CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic' -CLINVAR_CONFLICTING_NOT_BENIGN_FILTER = 'conflicting_not_benign' -CLINVAR_VUS_FILTER = 'vus_or_conflicting' -CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER, CLINVAR_CONFLICTING_NOT_BENIGN_FILTER} +CLINVAR_CONFLICTING_P_LP = 'conflicting_p_lp' +CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER, CLINVAR_CONFLICTING_P_LP} PATH_FREQ_OVERRIDE_CUTOFF = 0.05 CLINVAR_PATH_RANGES = [ (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'), (CLINVAR_LIKELY_PATH_FILTER, 'Pathogenic/Likely_pathogenic', 'Likely_risk_allele'), - (CLINVAR_VUS_FILTER, 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), + ('vus_or_conflicting', 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), ('likely_benign', 'Likely_benign', 'Benign/Likely_benign'), ('benign', 'Benign/Likely_benign', 'Benign'), ] diff --git a/ui/pages/Search/constants.js b/ui/pages/Search/constants.js index e87c33b7c6..511f52ce36 100644 --- a/ui/pages/Search/constants.js +++ b/ui/pages/Search/constants.js @@ -135,7 +135,7 @@ export const INHERITANCE_FILTER_JSON_OPTIONS = INHERITANCE_FILTER_OPTIONS.map( const CLINVAR_NAME = 'clinvar' const CLIVAR_PATH = 'pathogenic' const CLINVAR_LIKELY_PATH = 'likely_pathogenic' -const CLINVAR_CONFLICTING_NOT_BENIGN = 'conflicting_not_benign' +const CLINVAR_CONFLICTING_P_LP = 'conflicting_p_lp' const CLINVAR_UNCERTAIN = 'vus_or_conflicting' const CLINVAR_OPTIONS = [ { @@ -147,9 +147,9 @@ const CLINVAR_OPTIONS = [ value: CLINVAR_LIKELY_PATH, }, { - description: 'Clinvar variant of with conflicting interpretations, at least one of which is not benign or likely benign', - text: 'Conflicting - Not Benign', - value: CLINVAR_CONFLICTING_NOT_BENIGN, + description: 'Clinvar variant of with conflicting interpretations, at least one of which is Pathogenic (P) or Likely Pathogenic (LP)', + text: 'Conflicting with P/LP', + value: CLINVAR_CONFLICTING_P_LP, }, { description: 'Clinvar variant of uncertain significance or variant with conflicting interpretations', @@ -220,14 +220,14 @@ export const HGMD_PATHOGENICITY_FILTER_OPTIONS = [ { text: 'Pathogenic/ Likely Path.', value: { - [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_NOT_BENIGN], + [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_P_LP], [HGMD_NAME]: [HGMD_DM], }, }, { text: 'Not Benign', value: { - [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_NOT_BENIGN, CLINVAR_UNCERTAIN], + [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_P_LP, CLINVAR_UNCERTAIN], [HGMD_NAME]: HGMD_OPTIONS.map(({ value }) => value), }, }, From 2d8deec5dd2eaca4d7adacad08e2ef30d96fcdd4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 10:47:20 -0400 Subject: [PATCH 23/71] fix test --- seqr/views/apis/variant_search_api_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index c20bb87f08..34d8291325 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -528,7 +528,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0.03444932', '0.28899795', '0.246152', '20.9', '0.197', '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1.0', @@ -562,7 +562,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro ['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',], - ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0', + ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0.03444932', '0.28899795', '0.246152', '20.9', '0.197', '2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C', 'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', 'HG00731', '2', '', '99', '1.0', From 5ba8d76758489e1d6a5216ae28def2e9b1e379b8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 11:25:31 -0400 Subject: [PATCH 24/71] properly set search has in ui search --- .../VariantSearchFormPanelConfigs.jsx | 24 +++++++------------ ui/pages/Search/selectors.js | 13 +++------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx index 02f3e5d956..bc349932d3 100644 --- a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx +++ b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx @@ -1,5 +1,5 @@ import React from 'react' -import PropTypes from 'prop-types' +import { FormSpy } from 'react-final-form' import styled from 'styled-components' import { Form, Grid, Header, List, Loader, Table } from 'semantic-ui-react' @@ -530,24 +530,16 @@ export const QUALITY_PANEL = { fieldProps: { control: LazyLabeledSlider, format: val => val || null }, } -const handleChange = (onChange, value) => shouldExclude => onChange({ ...value, shouldExclude }) +const SUBSCRIPTION = { values: true } -const ExcludeSearchToggle = ({ name, value, onChange, ...props }) => ( - +const ExcludeSearchToggle = props => ( + + {({ values }) => ( + + )} + ) -ExcludeSearchToggle.propTypes = { - name: PropTypes.string, - value: PropTypes.object, - onChange: PropTypes.func, -} - const ES_EXCLUDE_FIELDS = [ { ...BASE_LOCUS_FIELD, diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 727f4eaeeb..14278ecabb 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -133,19 +133,12 @@ export const getIntitialSearch = createSelector( getCurrentSearchHash, getCurrentSearchParams, getIntitialProjectFamilies, - (searchHash, searchParams, projectFamilies) => { + (previousSearchHash, searchParams, projectFamilies) => { if (searchParams) { - // TODO does not work when search is edited - use form spy? - return { - ...searchParams, - search: { - ...(searchParams.search || {}), - exclude: { ...(searchParams.search || {}).exclude || {}, previousSearch: { searchHash } }, - }, - } + return { ...searchParams, previousSearchHash } } - return projectFamilies ? { projectFamilies: [projectFamilies] } : null + return projectFamilies ? { projectFamilies: [projectFamilies], previousSearchHash } : null }, ) From 8081ddb22cee66be59820f3bf87498e1f63eedac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 11:37:40 -0400 Subject: [PATCH 25/71] use updated search model --- seqr/utils/search/utils.py | 6 +++--- seqr/views/apis/variant_search_api.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index d9b12803f0..2430abc1db 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -293,11 +293,11 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if duplicates: raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') - exclude_previous = exclude.pop('previousSearch', None) - if (exclude_previous or {}).get('shouldExclude'): + exclude_previous_hash = exclude.pop('previousSearchHash', None) + if exclude_previous_hash: parsed_search.update(backend_specific_call( lambda *args: {}, _get_clickhouse_exclude_keys, - )(exclude_previous['searchHash'], user, genome_version)) + )(exclude_previous_hash, user, genome_version)) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index a52b3cbb1a..1030601460 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -113,6 +113,8 @@ def _get_or_create_results_model(search_hash, search_context, user): ).distinct() search_dict = search_context.get('search', {}) + if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).pop('previousSearch', False): + search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] search_model = VariantSearch.objects.filter(search=search_dict).filter( Q(created_by=user) | Q(name__isnull=False)).first() if not search_model: From 3a40c054ec6a6e8223ad401cf18a49215ead9d6d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 14:34:05 -0400 Subject: [PATCH 26/71] fix comp het pair filter --- clickhouse_search/backend/functions.py | 6 ++++++ seqr/views/apis/variant_search_api.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 0e61e7b6ab..ab1835fde6 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -7,6 +7,12 @@ class Array(Func): function = 'array' + def _resolve_output_field(self): + output_field = super()._resolve_output_field() + if not isinstance(output_field, ArrayField): + output_field = ArrayField(base_field=output_field) + return output_field + class ArrayConcat(Func): function = 'arrayConcat' diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 1030601460..01691142e2 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -113,7 +113,7 @@ def _get_or_create_results_model(search_hash, search_context, user): ).distinct() search_dict = search_context.get('search', {}) - if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).pop('previousSearch', False): + if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] search_model = VariantSearch.objects.filter(search=search_dict).filter( Q(created_by=user) | Q(name__isnull=False)).first() From cc2f41109ce21cd290c088a8487620fc89d267a6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 14:47:15 -0400 Subject: [PATCH 27/71] test search model creation --- seqr/views/apis/variant_search_api_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index c20bb87f08..be1e9eea55 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -641,16 +641,20 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro mock_error_logger.assert_not_called() # Test no results + VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete() mock_get_variants.side_effect = _get_empty_es_variants response = self.client.post(url, content_type='application/json', data=json.dumps({ - 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH + 'projectFamilies': PROJECT_FAMILIES, 'search': {**SEARCH, 'exclude': {'previousSearch': True}}, 'previousSearchHash': 'abc1234', })) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertDictEqual(response_json, { 'searchedVariants': [], 'search': { - 'search': SEARCH, + 'search': {**SEARCH, 'exclude': { + 'previousSearch': True, + 'previousSearchHash': 'abc1234', + }}, 'projectFamilies': PROJECT_FAMILIES, 'totalResults': 0, } From e41ca89abf98a0d22f23c3a1ea9d2f8573d10494 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 15:21:02 -0400 Subject: [PATCH 28/71] test previous does nothing for es search --- seqr/utils/search/search_utils_tests.py | 17 +++++++++++++++++ seqr/utils/search/utils.py | 1 + 2 files changed, 18 insertions(+) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index ae6b57e4c4..281d89dbdf 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -466,6 +466,23 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'], ) + self.set_cache(None) + self.search_model.search = { + 'inheritance': {'mode': 'any_affected'}, + 'exclude': {'previousSearch': True, 'previousSearchHash': 'abc1234', 'clinvar': ['benign']}, + } + previous_search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}}) + previous_results_model = VariantSearchResults.objects.create(variant_search=previous_search_model, search_hash='abc1234') + previous_results_model.families.set(self.families) + query_variants(self.results_model, user=self.user) + self._test_exclude_previous_search( + mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, + inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], + ) + + def _test_exclude_previous_search(self, *args, **kwargs): + self._test_expected_search_call(*args, **kwargs) + def test_cached_query_variants(self): self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS}) variants, total = query_variants(self.results_model, user=self.user) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2430abc1db..0f0c3ef982 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -293,6 +293,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if duplicates: raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') + exclude.pop('previousSearch', None) exclude_previous_hash = exclude.pop('previousSearchHash', None) if exclude_previous_hash: parsed_search.update(backend_specific_call( From 2c0514399da5195d706e2f32906c76008edce5b9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 16:13:35 -0400 Subject: [PATCH 29/71] test exclude cached previous results --- seqr/utils/search/search_utils_tests.py | 51 ++++++++++++++++++++----- seqr/utils/search/utils.py | 10 ++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 281d89dbdf..229c43eba1 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -279,9 +279,9 @@ def test_invalid_search_query_variants(self): self._test_invalid_search_params(query_variants) - def _test_expected_search_call(self, mock_get_variants, results_cache, search_fields=None, has_gene_search=False, + def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False, rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', - dataset_type=None, secondary_dataset_type=None, omitted_sample_guids=None, + dataset_type=None, secondary_dataset_type=None, exclude_keys=None, exclude_key_pairs=None, exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): genes = intervals = None has_included_gene_search = has_gene_search and not exclude_locations @@ -315,8 +315,16 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi expected_search['annotations'] = annotations if annotations_secondary is not None: expected_search['annotations_secondary'] = annotations_secondary + if exclude_keys is not None: + expected_search['exclude_keys'] = exclude_keys + if exclude_key_pairs is not None: + expected_search['exclude_key_pairs'] = exclude_key_pairs - mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs) + return (mock.ANY, expected_search, self.user, results_cache, '37'), kwargs, genes + + def _test_expected_search_call(self, mock_get_variants, *args, has_gene_search=False, exclude_locations=False, omitted_sample_guids=None, **kwargs): + call_args, call_kwargs, genes = self._get_expected_search_call(*args, has_gene_search=has_gene_search, exclude_locations=exclude_locations, **kwargs) + mock_get_variants.assert_called_with(*call_args, **call_kwargs) self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) if genes: @@ -341,16 +349,17 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid def test_query_variants(self, mock_get_variants): + parsed_variants = [{**v, 'key': (i+1) * 1000 } for i, v in enumerate(PARSED_VARIANTS)] def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = PARSED_VARIANTS + previous_search_results['all_results'] =parsed_variants previous_search_results['total_results'] = 5 - return PARSED_VARIANTS + return parsed_variants mock_get_variants.side_effect = _mock_get_variants variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, PARSED_VARIANTS) + self.assertListEqual(variants, parsed_variants) self.assertEqual(total, 5) - results_cache = {'all_results': PARSED_VARIANTS, 'total_results': 5} + results_cache = {'all_results': parsed_variants, 'total_results': 5} self.assert_cached_results(results_cache) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, @@ -467,6 +476,7 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v ) self.set_cache(None) + mock_get_variants.reset_mock() self.search_model.search = { 'inheritance': {'mode': 'any_affected'}, 'exclude': {'previousSearch': True, 'previousSearchHash': 'abc1234', 'clinvar': ['benign']}, @@ -480,8 +490,9 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'], ) - def _test_exclude_previous_search(self, *args, **kwargs): - self._test_expected_search_call(*args, **kwargs) + def _test_exclude_previous_search(self, mock_get_variants, *args, num_searches=1, **kwargs): + self._test_expected_search_call(mock_get_variants, *args, **kwargs) + self.assertEqual(mock_get_variants.call_count, num_searches) def test_cached_query_variants(self): self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS}) @@ -643,6 +654,28 @@ def _assert_expected_get_single_variant_call(self, mock_call, variant_id, expect def test_query_variants(self, mock_call): super().test_query_variants(mock_call) + def _test_exclude_previous_search(self, mock_get_variants, *args, **kwargs): + super()._test_exclude_previous_search( + mock_get_variants, *args, **kwargs, + exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, num_searches=2, + ) + call_args, call_kwargs, _ = self._get_expected_search_call(*args, inheritance_mode='de_novo', sort=None, num_results=100) + mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)]) + + # Test when previous results are cached + mock_get_variants.reset_mock() + self.mock_redis.get.side_effect = [ + None, + json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), + ] + self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + query_variants(self.results_model, user=self.user) + super()._test_exclude_previous_search( + mock_get_variants, *args, **kwargs, num_searches=1, + exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]}, + exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]}, + ) + def test_cached_query_variants(self): Project.objects.filter(id=1).update(genome_version='38') super().test_cached_query_variants() diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0f0c3ef982..e7612f49eb 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -333,10 +333,10 @@ def _query_variants(search_model, user, previous_search_results, genome_version, def _get_clickhouse_exclude_keys(search_hash, user, genome_version): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) cached_results = _get_any_sort_cached_results(previous_search_model) - if cached_results: - results = cached_results['all_results'] - else: - results, _ = _query_variants(previous_search_model, user, {}, genome_version) + if not cached_results: + cached_results = {} + _query_variants(previous_search_model, user, cached_results, genome_version) + results = cached_results['all_results'] exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) for variant in results: @@ -348,7 +348,7 @@ def _get_clickhouse_exclude_keys(search_hash, user, genome_version): else: dataset_type = variant_dataset_type(variant) exclude_keys[dataset_type].append(variant['key']) - return {'exclude_keys': exclude_keys, 'exclude_key_pairs': exclude_key_pairs} + return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)} def variant_dataset_type(variant): From 45b4f19fad1287cca6eb2ba1a3bed7261071387b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 22 Sep 2025 16:19:57 -0400 Subject: [PATCH 30/71] enable all clickhouse data management pages --- ui/pages/DataManagement/DataManagement.jsx | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ui/pages/DataManagement/DataManagement.jsx b/ui/pages/DataManagement/DataManagement.jsx index 600e9044ac..df0828186f 100644 --- a/ui/pages/DataManagement/DataManagement.jsx +++ b/ui/pages/DataManagement/DataManagement.jsx @@ -44,13 +44,9 @@ const ES_DATA_MANAGEMENT_PAGES = [ ...DATA_MANAGEMENT_PAGES, ] -const LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ +const CLICKHOUSE_DATA_MANAGEMENT_PAGES = [ ...DATA_MANAGEMENT_PAGES, { path: 'pipeline_status', component: () => }, -] - -const AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ - ...DATA_MANAGEMENT_PAGES, ...TRIGGER_SEARCH_DATA_UPDATE_PAGES, ] @@ -58,10 +54,7 @@ const dataManagementPages = (user, elasticsearchEnabled) => { if (!user.isDataManager) { return PM_DATA_MANAGEMENT_PAGES } - if (elasticsearchEnabled) { - return ES_DATA_MANAGEMENT_PAGES - } - return user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES + return elasticsearchEnabled ? ES_DATA_MANAGEMENT_PAGES : CLICKHOUSE_DATA_MANAGEMENT_PAGES } const DataManagement = ({ match, user, pages }) => ( From 65d6036bf48d50013fcdff23153c32f7a3ce011d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 11:35:24 -0400 Subject: [PATCH 31/71] clearer trigger delete ui --- .../components/TriggerSearchDataUpdatePages.jsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index 6a0e6fdfaa..9bc97d1237 100644 --- a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -46,25 +46,25 @@ const FAMILY_FIELDS = [ DATASET_TYPE_FIELD, ] -const TriggerSearchDataUpdateForm = ({ path, fields }) => ( +const TriggerSearchDataUpdateForm = ({ entity, fields }) => ( ) TriggerSearchDataUpdateForm.propTypes = { - path: PropTypes.string, + entity: PropTypes.string, fields: PropTypes.arrayOf(PropTypes.object), } const TriggerDeleteProjects = () => ( - + ) const TriggerDeleteFamilies = () => ( - + ) export default [ From 81742977254d2fe999de9c968b361320c7d04828 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 11:52:17 -0400 Subject: [PATCH 32/71] disable search samples on trigger delete --- seqr/views/apis/data_manager_api.py | 16 ++++++++++------ seqr/views/apis/data_manager_api_tests.py | 10 +++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 4c7713148c..c3d7f18a36 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -425,7 +425,8 @@ def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') project = Project.objects.get(guid=project_guid) - return _trigger_data_update(delete_clickhouse_project, request_json, project) + samples = Sample.objects.filter(individual__family__project=project) + return _trigger_data_update(delete_clickhouse_project, request.user, samples, request_json, project) @data_manager_required @@ -434,14 +435,17 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(delete_clickhouse_family, request_json, project, family_guid) + samples = Sample.objects.filter(individual__family__guid=family_guid) + return _trigger_data_update(delete_clickhouse_family, request.user, samples, request_json, project, family_guid) -def _trigger_data_update(clickhouse_func, request_json, project, *args): +def _trigger_data_update(clickhouse_func, user, samples, request_json, project, *args): dataset_type = request_json.get('datasetType') - sample_types = Sample.objects.filter( - individual__family__project=project, dataset_type=dataset_type, is_active=True, - ).values_list('sample_type', flat=True).distinct() if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] + samples = samples.filter(dataset_type=dataset_type, is_active=True) + sample_types = list( + samples.values_list('sample_type', flat=True).distinct() + ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] + Sample.bulk_update(user=user, update_json={'is_active': False}, queryset=samples) info = [] for sample_type in sample_types: info.append(clickhouse_func(project, *args, dataset_type=dataset_type, sample_type=sample_type)) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index a9e1818bd3..ec792eb0eb 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -14,7 +14,7 @@ from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses -from seqr.models import Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization +from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' @@ -2069,12 +2069,20 @@ def _assert_expected_delete_project(self, response): 22: (0, 3, 0, 1), }) + project_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True) + self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0) + self.assertEqual(project_samples.count(), 4) + def _assert_expected_delete_family(self, response): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { 'info': ['Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg'], }) + family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) + self.assertEqual(family_samples.filter(dataset_type='SV').count(), 0) + self.assertEqual(family_samples.count(),4) + def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', From a092d68887aa2d1d7aa97bb6c502b574cbc894bf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 11:56:03 -0400 Subject: [PATCH 33/71] better info message --- seqr/views/apis/data_manager_api.py | 4 ++-- seqr/views/apis/data_manager_api_tests.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index c3d7f18a36..9df12df900 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -445,8 +445,8 @@ def _trigger_data_update(clickhouse_func, user, samples, request_json, project, sample_types = list( samples.values_list('sample_type', flat=True).distinct() ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] - Sample.bulk_update(user=user, update_json={'is_active': False}, queryset=samples) - info = [] + updated = Sample.bulk_update(user=user, update_json={'is_active': False}, queryset=samples) + info = [f'Deactivated search for {len(updated)} individuals'] for sample_type in sample_types: info.append(clickhouse_func(project, *args, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index ec792eb0eb..854350bd16 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -2053,7 +2053,10 @@ def _test_validate_dataset_type(self, url): def _assert_expected_delete_project(self, response): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { - 'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de'], + 'info': [ + 'Deactivated search for 7 individuals', + 'Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de', + ], }) self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) @@ -2076,7 +2079,10 @@ def _assert_expected_delete_project(self, response): def _assert_expected_delete_family(self, response): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { - 'info': ['Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg'], + 'info': [ + 'Deactivated search for 3 individuals', + 'Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg', + ], }) family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) From 33e6fbd659c1c7e41edb60e4bf9ee48a80f76418 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:01:13 -0400 Subject: [PATCH 34/71] remove duplicate manage command --- .../commands/deactivate_project_search.py | 16 +----- .../tests/deactivate_project_search_tests.py | 50 +++---------------- 2 files changed, 10 insertions(+), 56 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 8d8fd2f417..df48683bea 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -1,8 +1,7 @@ from django.core.management.base import BaseCommand, CommandError -from clickhouse_search.search import delete_clickhouse_project from seqr.models import Project, Sample -from seqr.utils.search.utils import backend_specific_call +from seqr.utils.search.utils import es_only import logging logger = logging.getLogger(__name__) @@ -12,6 +11,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('project') + @es_only def handle(self, *args, **options): project = Project.objects.get(guid=options['project']) @@ -21,15 +21,3 @@ def handle(self, *args, **options): updated = Sample.bulk_update(user=None, update_json={'is_active': False}, individual__family__project=project) logger.info(f'Deactivated {len(updated)} samples') - - if updated: - dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() - for dataset_type, sample_type in dataset_types: - backend_specific_call( - lambda *args, **kwargs: True, self._delete_clickhouse_project, - )(project, dataset_type, sample_type) - - @staticmethod - def _delete_clickhouse_project(project, dataset_type, sample_type): - info = delete_clickhouse_project(project, dataset_type, sample_type) - logger.info(info) diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index 2a35dc9a4c..87c74a2f61 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -15,9 +15,8 @@ VARIANT_ID = '21-3343353-GAGA-G' -class DeactivateProjectSearchTest(object): - - DELETE_SUCCESS_LOGS = [] +class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase): + fixtures = ['users', '1kg_project'] @responses.activate @mock.patch('seqr.management.commands.deactivate_project_search.input') @@ -47,54 +46,21 @@ def test_command(self, mock_input): 'updateType': 'bulk_update'}, }), ('Deactivated 11 samples', None), - ] + self.DELETE_SUCCESS_LOGS) + ]) active_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True) self.assertEqual(active_samples.count(), 0) - self._assert_expected_delete() # Re-running has no effect self.reset_logs() call_command('deactivate_project_search', PROJECT_GUID) self.assert_json_logs(user=None, expected=[('Deactivated 0 samples', None)]) - def _assert_expected_delete(self): - pass - -class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest): - fixtures = ['users', '1kg_project'] - -class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase, DeactivateProjectSearchTest): +class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase): fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search'] - DELETE_SUCCESS_LOGS = [ - ('Deleted all MITO search data for project 1kg project nåme with uniçøde', None), - ('Deleted all SNV_INDEL search data for project 1kg project nåme with uniçøde', None), - ('Deleted all GCNV search data for project 1kg project nåme with uniçøde', None), - ] - - def _assert_expected_delete(self): - self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(EntriesMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(ProjectGtStatsMito.objects.filter(project_guid=PROJECT_GUID).count(), 0) - self.assertEqual(EntriesGcnv.objects.filter(project_guid=PROJECT_GUID).count(), 0) - - updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) - self.assertDictEqual(updated_seqr_pops_by_key, { - 1: (2, 2, 1, 1), - 2: (1, 1, 0, 0), - 3: (0, 0, 0, 0), - 4: (0, 0, 0, 0), - 5: (1, 1, 0, 0), - 6: (0, 0, 0, 0), - 22: (0, 3, 0, 1), - }) - - updated_mito_pops_by_key = dict(AnnotationsMito.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) - self.assertDictEqual(updated_mito_pops_by_key, { - 6: (0, 0, 0, 0), - 7: (0, 0, 0, 0), - 8: (0, 0, 0, 0), - }) + def test_command(self): + with self.assertRaises(ValueError) as e: + call_command('deactivate_project_search', 'foo') + self.assertEqual(str(e.exception), 'handle is disabled without the elasticsearch backend') From 3d0c5e1a08c7bf55538e6397a4bb1b7cba9533c2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:31:04 -0400 Subject: [PATCH 35/71] unit test clickhouse --- clickhouse_search/search_tests.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f27b800f74..50460594cc 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -504,6 +504,34 @@ def test_inheritance_filter(self): [GRCH37_VARIANT], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}, ) + def test_exclude_previous_search_results(self): + VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234') + self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], + ]})] + self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + + self._assert_expected_search( + [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], + inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, + exclude={'previousSearch': True, 'previousSearchHash': 'abc1234'}, cached_variant_fields=[ + [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], + [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], + {}, {}, + ], + ) + + self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ + [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, + ]})] + self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + self._assert_expected_search( + [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], + inheritance_mode='recessive', cached_variant_fields=[ + {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], + ], + ) + def test_quality_filter(self): quality_filter = {'vcf_filter': 'pass'} self._assert_expected_search( From 84e7e5785244f07b8ebbbd0842f4defbd12e969e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:36:17 -0400 Subject: [PATCH 36/71] fix js test --- ui/pages/Search/selectors.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/Search/selectors.test.js b/ui/pages/Search/selectors.test.js index e261f012d2..78b77f1bc7 100644 --- a/ui/pages/Search/selectors.test.js +++ b/ui/pages/Search/selectors.test.js @@ -20,7 +20,7 @@ test('getIntitialSearch', () => { expect(getIntitialSearch(NO_SEARCH_STATE, { match: { params: {} } })).toEqual(null) - expect(getIntitialSearch(STATE, { match: { params: { searchHash: SEARCH_HASH } } })).toEqual(SEARCH) + expect(getIntitialSearch(STATE, { match: { params: { searchHash: SEARCH_HASH } } })).toEqual({ ...SEARCH, previousSearchHash: SEARCH_HASH }) expect(getIntitialSearch( NO_SEARCH_STATE, { match: { params: { projectGuid: PROJECT_GUID } } }) From d2469d3623ae5539b6bd5b0f3252031308daf55e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:37:21 -0400 Subject: [PATCH 37/71] codacy --- clickhouse_search/search_tests.py | 4 ++-- seqr/utils/search/search_utils_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 50460594cc..20e3d350d2 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -509,7 +509,7 @@ def test_exclude_previous_search_results(self): self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3], ]})] - self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], @@ -524,7 +524,7 @@ def test_exclude_previous_search_results(self): self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, ]})] - self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', cached_variant_fields=[ diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 229c43eba1..17373d8f4d 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -668,7 +668,7 @@ def _test_exclude_previous_search(self, mock_get_variants, *args, **kwargs): None, json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}), ] - self.mock_redis.keys.side_effect = [[], [f'search_results__abc1234__gnomad']] + self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']] query_variants(self.results_model, user=self.user) super()._test_exclude_previous_search( mock_get_variants, *args, **kwargs, num_searches=1, From ff967982a722c49602ec88efc8f8b3171121baa4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:42:13 -0400 Subject: [PATCH 38/71] fix test --- seqr/views/utils/variant_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index c16327b842..d70cb50099 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -89,7 +89,8 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): ref = variant_json.get('ref') alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - variant_id = variant_json.get('variantId', variant_id) + if not variant_json.get('variantId'): + variant_json['variantId'] = variant_id if variant_json.get('key'): update_json = { 'key': variant_json['key'], From 7e1d821ff04532250d8c911efded2b8d7065ff77 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:46:40 -0400 Subject: [PATCH 39/71] clean up imports --- clickhouse_search/search.py | 1 - seqr/management/commands/reload_saved_variant_json.py | 2 +- seqr/management/tests/reload_saved_variant_json_tests.py | 2 +- .../tests/transfer_families_to_different_project_tests.py | 1 - seqr/utils/search/utils.py | 2 +- seqr/views/utils/test_utils.py | 2 -- 6 files changed, 3 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 061d2c63b0..8981f74ce6 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -18,7 +18,6 @@ from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from settings import CLICKHOUSE_SERVICE_HOSTNAME logger = SeqrLogger(__name__) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index ce3e0edbab..7c9f14efc7 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -1,5 +1,5 @@ import logging -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db.models.query_utils import Q from seqr.models import Project from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index ae5983fa0f..b451ca0b96 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -1,7 +1,7 @@ #-*- coding: utf-8 -*- import mock -from django.core.management import call_command, CommandError +from django.core.management import call_command from django.test import TestCase PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index f696bbc7be..1ea732e2ac 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -1,4 +1,3 @@ -import responses from django.core.management import call_command from seqr.models import Family, VariantTagType, VariantTag, Sample diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 02d0c5508f..3bb80dac32 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -17,7 +17,7 @@ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items -from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS +from seqr.utils.xpos_utils import get_xpos, format_chrom logger = SeqrLogger(__name__) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 58c764391f..03e319d088 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -1,7 +1,6 @@ # Utilities used for unit and integration tests. from collections import defaultdict from copy import deepcopy -from datetime import datetime from django.contrib.auth.models import User, Group from django.core.management import call_command from django.db import connections, transaction @@ -12,7 +11,6 @@ import logging import mock import re -import requests import responses from urllib.parse import quote_plus, urlparse From 58a20285badf226aae713249653009181e997d2c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:51:50 -0400 Subject: [PATCH 40/71] clean up --- seqr/utils/search/add_data_utils.py | 1 - seqr/views/apis/data_manager_api_tests.py | 2 +- seqr/views/utils/test_utils.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index f77ba0a074..93540e0ca7 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -182,7 +182,6 @@ def get_loading_samples_validator(vcf_samples: list[str], loaded_individual_ids: def validate_expected_samples(record_family_ids, previous_loaded_individuals, sample_type): errors = [] - nonlocal loaded_sample_types if loaded_sample_types is not None: if sample_type: loaded_sample_types.append(sample_type) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index a9e1818bd3..009836de51 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1377,7 +1377,7 @@ def test_get_loaded_projects(self): self._assert_expected_airtable_errors(url) - def _assert_expected_pm_access(self, get_response, mock_current_gene_version=None): + def _assert_expected_pm_access(self, get_response): response = get_response() self.assertEqual(response.status_code, 200) self.login_data_manager_user() diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 03e319d088..80462c1a7a 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -45,8 +45,6 @@ class AuthenticationTestCase(TestCase): no_policy_user = None def setUp(self): - patcher = mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', self.CLICKHOUSE_HOSTNAME) - patcher.start() patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) patcher.start() self.addCleanup(patcher.stop) From 233785117a5fd03d1ee0617729490ea401b6b11a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 12:57:10 -0400 Subject: [PATCH 41/71] fix syntax --- seqr/views/utils/variant_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index d70cb50099..734c6cf2ce 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -89,13 +89,12 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): ref = variant_json.get('ref') alt = variant_json.get('alt') var_length = variant_json['end'] - variant_json['pos'] if variant_json.get('end') is not None else len(ref) - 1 - if not variant_json.get('variantId'): - variant_json['variantId'] = variant_id + variant_id = variant_json.get('variantId', variant_id) if variant_json.get('key'): update_json = { 'key': variant_json['key'], 'genotypes': variant_json.get('genotypes', {}), - 'dataset_type': variant_dataset_type(variant_json), + 'dataset_type': variant_dataset_type({'variantId': variant_id, **variant_json}), } else: update_json = {'saved_variant_json': variant_json} From 3ac7f03c5875e80bb515819e085f1ac0384a8a4c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 13:58:12 -0400 Subject: [PATCH 42/71] fix mocks --- clickhouse_search/search_tests.py | 1 - seqr/management/tests/reload_saved_variant_json_tests.py | 1 - seqr/utils/search/search_utils_tests.py | 2 -- seqr/views/status_tests.py | 1 - 4 files changed, 5 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f27b800f74..ab069b1241 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -24,7 +24,6 @@ from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'localhost') class ClickhouseSearchTests(DifferentDbTransactionSupportMixin, SearchTestHelper, TestCase): databases = '__all__' fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index b451ca0b96..51b9f3e5c5 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -88,7 +88,6 @@ def test_with_param_command(self, mock_get_variants, mock_logger): mock_logger.error.assert_called_with('Error reloading variants in 1kg project n\xe5me with uni\xe7\xf8de: Database error.') -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') class ClickhouseReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index ae6b57e4c4..7bc1f6682e 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -504,7 +504,6 @@ def _mock_get_gene_counts(families, search, user, previous_search_results, genom ) -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): databases = ['default', 'reference_data'] @@ -595,7 +594,6 @@ def _test_invalid_no_location_search_params(self): pass -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): databases = '__all__' fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts'] diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index 33518adef3..f2cbc40e18 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -8,7 +8,6 @@ from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.views.status.redis.StrictRedis') @mock.patch('seqr.views.status.logger') class ElasticsearchStatusTest(TestCase): From a51b93412cb18c8ee7005b59954e44521c48b12f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 14:04:59 -0400 Subject: [PATCH 43/71] helper function --- clickhouse_search/managers.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 02a52a78cd..eb1930b212 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -813,20 +813,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit entries = entries.annotate(carriers=self._carriers_expression(unaffected_samples)) if multi_sample_type_families: - if gt_filter: - inheritance_q |= multi_sample_type_family_q - entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples) - elif inheritance_q is not None: - entries = entries.annotate(passes_inheritance=inheritance_q) - inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q - else: - entries = entries.annotate(passes_inheritance=Value(True)) - - if quality_q is None: - entries = entries.annotate(passes_quality=Value(True)) - else: - entries = entries.annotate(passes_quality=quality_q) - quality_q = Q(passes_quality=True) | multi_sample_type_family_q + entries, inheritance_q, quality_q = self._get_multi_sample_type_family_call_qs( + entries, multi_sample_type_family_q, inheritance_q, quality_q, gt_filter, family_missing_type_samples, + ) if inheritance_q is not None: entries = entries.filter(inheritance_q) @@ -932,6 +921,24 @@ def _quality_q(self, quality_filter, allow_no_call, affected_samples, clinvar_ov return quality_q + def _get_multi_sample_type_family_call_qs(self, entries, multi_sample_type_family_q, inheritance_q, quality_q, gt_filter, family_missing_type_samples): + if gt_filter: + inheritance_q |= multi_sample_type_family_q + entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples) + elif inheritance_q is not None: + entries = entries.annotate(passes_inheritance=inheritance_q) + inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q + else: + entries = entries.annotate(passes_inheritance=Value(True)) + + if quality_q is None: + entries = entries.annotate(passes_quality=Value(True)) + else: + entries = entries.annotate(passes_quality=quality_q) + quality_q = Q(passes_quality=True) | multi_sample_type_family_q + + return entries, inheritance_q, quality_q + @staticmethod def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples): entries = entries.annotate(failed_family_samples= ArrayMap( From 4907e89db57e520d371922d936d527a66b245276 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 14:15:16 -0400 Subject: [PATCH 44/71] remove ui option for skip validation on data loading --- seqr/utils/search/add_data_utils.py | 3 +-- seqr/views/apis/data_manager_api.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 16 ++++++++-------- ui/pages/DataManagement/components/LoadData.jsx | 6 ------ 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index f77ba0a074..7bee729a64 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -67,7 +67,7 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N def trigger_data_loading(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str, genome_version: str, data_path: str, user: User, raise_error: bool = False, skip_expect_tdr_metrics: bool = True, - skip_validation: bool = False, skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None, + skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None, success_message: str = None, error_message: str = None, success_slack_channel: str = SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL): variables = { 'projects_to_run': sorted([p.guid for p in projects]) if projects else None, @@ -77,7 +77,6 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam 'sample_type': sample_type, } bool_variables = { - 'skip_validation': skip_validation, 'skip_check_sex_and_relatedness': skip_check_sex_and_relatedness, 'skip_expect_tdr_metrics': skip_expect_tdr_metrics, } diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 4c7713148c..b8a55c0c70 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -353,7 +353,7 @@ def load_data(request): success = trigger_data_loading( projects_by_guid.values(), individual_ids, sample_type, dataset_type, request_json['genomeVersion'], - _callset_path(request_json), user=request.user, skip_validation=request_json.get('skipValidation', False), + _callset_path(request_json), user=request.user, skip_check_sex_and_relatedness=request_json.get('skipSRChecks', False), vcf_sample_id_map=vcf_sample_id_map, raise_error=is_local, skip_expect_tdr_metrics=is_local, success_message=success_message, error_message=error_message, ) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index a9e1818bd3..84cce3e6ee 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1403,7 +1403,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp' body = {**self.REQUEST_BODY, 'projects': [ json.dumps(option) for option in self.PROJECT_OPTIONS + [{'projectGuid': 'R0005_not_project'}] - ], 'vcfSamples': self.VCF_SAMPLES, 'skipValidation': True} + ], 'vcfSamples': self.VCF_SAMPLES, 'skipSRChecks': True} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'The following projects are invalid: R0005_not_project'}) @@ -1419,7 +1419,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m ) self.assertDictEqual(response.json(), {'success': True}) - self._assert_expected_load_data_requests(sample_type='WES', skip_validation=True) + self._assert_expected_load_data_requests(sample_type='WES', skip_check_sex_and_relatedness=True) self._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, 'SNV_INDEL', sample_type='WES', has_remap=bool(self.MOCK_AIRTABLE_KEY)) variables = { @@ -1431,7 +1431,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m 'reference_genome': 'GRCh38', 'callset_path': f'{self.TRIGGER_CALLSET_DIR}/callset.vcf', 'sample_type': 'WES', - 'skip_validation': True, + 'skip_check_sex_and_relatedness': True, } if self.SKIP_TDR: variables['skip_expect_tdr_metrics'] = True @@ -1445,8 +1445,8 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m responses.calls.reset() self.reset_logs() - del body['skipValidation'] - del variables['skip_validation'] + del body['skipSRChecks'] + del variables['skip_check_sex_and_relatedness'] body.update({'datasetType': 'SV', 'filePath': f'{self.CALLSET_DIR}/sv_callset.vcf'}) self._trigger_error(url, body, variables, mock_open, mock_gzip_open, mock_mkdir) @@ -1489,7 +1489,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m }) self.assertEqual(len(responses.calls), 0) - def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_validation=False): + def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_check_sex_and_relatedness=False): projects = [PROJECT_GUID, NON_ANALYST_PROJECT_GUID] if skip_project: projects = projects[1:] @@ -1502,8 +1502,8 @@ def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_t } if self.SKIP_TDR: body['skip_expect_tdr_metrics'] = True - if skip_validation: - body['skip_validation'] = True + if skip_check_sex_and_relatedness: + body['skip_check_sex_and_relatedness'] = True self.assertDictEqual(json.loads(responses.calls[-1].request.body), body) def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_mkdir): diff --git a/ui/pages/DataManagement/components/LoadData.jsx b/ui/pages/DataManagement/components/LoadData.jsx index 2048f9b986..5cbb9ead40 100644 --- a/ui/pages/DataManagement/components/LoadData.jsx +++ b/ui/pages/DataManagement/components/LoadData.jsx @@ -54,12 +54,6 @@ const FILE_PATH_FIELD = { } const CALLSET_PAGE_FIELDS = [ - { - name: 'skipValidation', - label: 'Skip Callset Validation', - component: InlineToggle, - asFormInput: true, - }, { ...GENOME_VERSION_FIELD, component: ButtonRadioGroup, From 934fa1011a7e5550d793d2df7efc5fdabf74744f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 15:21:45 -0400 Subject: [PATCH 45/71] flag places where need update --- matchmaker/views/matchmaker_api.py | 2 +- seqr/views/apis/family_api.py | 2 +- seqr/views/apis/saved_variant_api.py | 2 +- seqr/views/apis/summary_data_api.py | 2 +- seqr/views/utils/anvil_metadata_utils.py | 2 +- seqr/views/utils/project_context_utils.py | 2 +- seqr/views/utils/variant_utils.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index f3a7dfffe4..17f0450d3e 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -58,7 +58,7 @@ def get_individual_mme_matches(request, submission_guid): ) variants = response_json['savedVariantsByGuid'].values() - backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)( + backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)( # TODO variants, project.genome_version, request.user, ) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index e26037621c..5059f80ada 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -61,7 +61,7 @@ def family_page_data(request, family_guid): 'xpos', 'xpos_end', *additional_fields, svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'), ) - gene_ids = backend_specific_call( + gene_ids = backend_specific_call( # TODO _variants_gene_ids, _clickhouse_variants_gene_ids, )(discovery_variants, project.genome_version, request.user) discovery_variant_intervals = [dict(zip( diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 8a75fb697b..ffa07ea469 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -127,7 +127,7 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) + gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) # TODO create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 1b0f7f621b..707bfb9737 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -115,7 +115,7 @@ def saved_variants_page(request, tag): saved_variant_models = saved_variant_models.filter(family__project__guid__in=get_project_guids_user_can_view(request.user)) if gene: - gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( + gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( # TODO update lambda *args: Q(), _saved_variant_with_clickhouse_gene_q, )(saved_variant_models, gene, is_all_tags) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index ec867c784d..b4185f21cc 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -346,7 +346,7 @@ def _get_parsed_saved_discovery_variants_by_family( variant_json_fields.append('clinvar') variants = [] gene_ids = set() - variant_json_by_guid = backend_specific_call( + variant_json_by_guid = backend_specific_call( # TODO potential update _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, )(project_saved_variants, include_clinvar) for variant in project_saved_variants: diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 546aa2fb2c..f1ed20a860 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -125,7 +125,7 @@ def families_discovery_tags(families, genome_version, project=None): varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, ) try: - discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)( + discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)( # TODO discovery_variants, genome_version=genome_version, ) except Exception as e: diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 1371b142de..0feea871e1 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -147,7 +147,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use ).annotate(saved_variant_ids=ArrayAgg('saved_variants__id', ordering='id')) } - variant_genes_by_id = backend_specific_call( + variant_genes_by_id = backend_specific_call( # TODO _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, )(saved_variant_map.values(), genome_version) From 69648521d443bfdc6d6ec0d5567a7f63e6c8303e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 16:37:25 -0400 Subject: [PATCH 46/71] store gene ids on saved variant model --- seqr/migrations/0081_savedvariant_gene_ids.py | 63 +++++++++++++++++++ seqr/models.py | 1 + 2 files changed, 64 insertions(+) create mode 100644 seqr/migrations/0081_savedvariant_gene_ids.py diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py new file mode 100644 index 0000000000..a76ee4ec21 --- /dev/null +++ b/seqr/migrations/0081_savedvariant_gene_ids.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.24 on 2025-09-25 19:26 + +from collections import defaultdict +import django.contrib.postgres.fields +from django.db import migrations, models + +from clickhouse_search.backend.functions import ArrayMap +from clickhouse_search.search import get_annotations_queryset + +BATCH_SIZE = 10000 + +class JsonKeys(models.Func): + function = 'jsonb_object_keys' + + +def populate_variant_gene_ids(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + variants = SavedVariant.objects.using(db_alias).filter(key__isnull=True) + for variant in variants: + variant.gene_ids = list(variant.saved_variant_json.get('transcripts', {}).keys()) + num_updated = SavedVariant.objects.using(db_alias).bulk_update(variants, ['gene_ids']) + if num_updated: + print(f'Populated gene_ids for {num_updated} variants') + for dataset_type in ['SNV_INDEL', 'MITO', 'SV_WGS', 'SV_WES']: + for genome_version in ['37', '38']: + variants = SavedVariant.objects.using(db_alias).filter( + key__isnull=False, family__project__genome_version=genome_version, dataset_type=dataset_type, + ) + if not variants.exists(): + continue + variants_by_key = defaultdict(list) + for v in variants: + variants_by_key[v.key].append(v) + all_keys = sorted(variants_by_key.keys()) + print(f'Populating gene_ids for {len(all_keys)} {dataset_type} ({genome_version}) clickhouse variants') + for i in range(0, len(all_keys), BATCH_SIZE): + batch_keys = all_keys[i:i + BATCH_SIZE] + clickhouse_qs = get_annotations_queryset(genome_version, dataset_type, batch_keys) + gene_ids_expr = ArrayMap(clickhouse_qs.transcript_field, mapped_expression='x.geneId') + to_update = [] + for key, gene_ids in clickhouse_qs.values_list('key', gene_ids_expr): + for variant in variants_by_key[key]: + variant.gene_ids = gene_ids + to_update.append(variant) + num_updated = SavedVariant.objects.using(db_alias).bulk_update(to_update, ['gene_ids']) + print(f'Updated {num_updated} variants') + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0080_savedvariant_dataset_type_savedvariant_genotypes_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='savedvariant', + name='gene_ids', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=20), blank=True, null=True, size=None), + ), + migrations.RunPython(populate_variant_gene_ids, reverse_code=migrations.RunPython.noop), + ] diff --git a/seqr/models.py b/seqr/models.py index 4f4951e90d..ea45964737 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -841,6 +841,7 @@ class SavedVariant(ModelWithGUID): selected_main_transcript_id = models.CharField(max_length=20, null=True) saved_variant_json = models.JSONField(default=dict) genotypes = models.JSONField(default=dict) + gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True) acmg_classification = models.JSONField(null=True) # ACMG based classification From 781ac854f911f42b0849db0d5d836707b317631d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 17:00:33 -0400 Subject: [PATCH 47/71] sort gene transcripts --- seqr/migrations/0081_savedvariant_gene_ids.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py index a76ee4ec21..496fbf2092 100644 --- a/seqr/migrations/0081_savedvariant_gene_ids.py +++ b/seqr/migrations/0081_savedvariant_gene_ids.py @@ -18,7 +18,7 @@ def populate_variant_gene_ids(apps, schema_editor): db_alias = schema_editor.connection.alias variants = SavedVariant.objects.using(db_alias).filter(key__isnull=True) for variant in variants: - variant.gene_ids = list(variant.saved_variant_json.get('transcripts', {}).keys()) + variant.gene_ids = sorted(variant.saved_variant_json.get('transcripts', {}).keys(), key=lambda gene_id: _transcript_sort(gene_id, variant.saved_variant_json)) num_updated = SavedVariant.objects.using(db_alias).bulk_update(variants, ['gene_ids']) if num_updated: print(f'Populated gene_ids for {num_updated} variants') @@ -47,6 +47,13 @@ def populate_variant_gene_ids(apps, schema_editor): print(f'Updated {num_updated} variants') +def _transcript_sort(gene_id, saved_variant_json): + gene_transcripts = saved_variant_json['transcripts'][gene_id] + main_transcript_id = saved_variant_json.get('mainTranscriptId') + is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) + return (is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) + + class Migration(migrations.Migration): dependencies = [ From a72dbfc5d9fcf67ad2ac33b2562b1b3268b52f8f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 17:11:08 -0400 Subject: [PATCH 48/71] set gene ids for new saved variants --- seqr/views/utils/variant_utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 0feea871e1..ee14b962c9 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -98,6 +98,8 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): } else: update_json = {'saved_variant_json': variant_json} + if 'transcripts' in variant_json: + update_json['gene_ids'] = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) return { 'xpos': xpos, 'xpos_end': xpos + var_length, @@ -115,6 +117,13 @@ def _dataset_type(variant_id, variant): return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS +def _transcript_sort(gene_id, saved_variant_json): + gene_transcripts = saved_variant_json['transcripts'][gene_id] + main_transcript_id = saved_variant_json.get('mainTranscriptId') + is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) + return (is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) + + def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=False): all_family_ids = {family_id for family_id, _ in family_variant_data.keys()} all_variant_ids = {variant_id for _, variant_id in family_variant_data.keys()} @@ -129,7 +138,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) if load_new_variant_data else backend_specific_call( - lambda o, _: o, _get_clickhouse_variant_keys, + lambda o, _: o, _get_clickhouse_variant_keys, # TODO need to have gene ids when new variants created (or update after) )({k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): From e2329bb577e2b83006253d1d2eb0e61248fede7e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 25 Sep 2025 18:02:56 -0400 Subject: [PATCH 49/71] update fixtures --- seqr/fixtures/1kg_project.json | 6 ++++ seqr/fixtures/report_variants.json | 5 +++ seqr/migrations/0081_savedvariant_gene_ids.py | 2 +- seqr/views/apis/saved_variant_api_tests.py | 35 +++++++++++++++---- seqr/views/utils/variant_utils.py | 2 +- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 4a9e17b50b..bd77c0dfee 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1844,6 +1844,7 @@ "ref": "GAGA", "alt": "G", "variant_id": "21-3343353-GAGA-G", + "gene_ids": ["ENSG00000135953"], "saved_variant_json": { "variantId": "21-3343353-GAGA-G", "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, @@ -1895,6 +1896,7 @@ "numAlt": 1 } }, + "gene_ids": ["ENSG00000240361", "ENSG00000135953"], "saved_variant_json": { "clinvar": { "pathogenicity": "Uncertain_significance", @@ -2006,6 +2008,7 @@ "ref": "G", "alt": "A", "variant_id": "1-46859832-G-A", + "gene_ids": ["ENSG00000197530"], "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -2046,6 +2049,7 @@ "ref": "G", "alt": "CA", "variant_id": "1-1562437-G-CA", + "gene_ids": ["ENSG00000197530"], "saved_variant_json": { "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, "liftedOverGenomeVersion": "38", @@ -2136,6 +2140,7 @@ "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1}, "I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1} }, + "gene_ids": ["ENSG00000240361", "ENSG00000135953"], "saved_variant_json": { "clinvar": { "pathogenicity": "Uncertain_significance", @@ -2190,6 +2195,7 @@ "ref": null, "alt": null, "variant_id": "prefix_19107_DEL", + "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], "saved_variant_json": { "liftedOverGenomeVersion": null, "pos": 249045487, diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 9ff4fdd18d..0d64249d1c 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -12,6 +12,7 @@ "ref": "C", "alt": "T", "variant_id": "19-1912634-C-T", + "gene_ids": ["ENSG00000240361"], "saved_variant_json": { "liftedOverGenomeVersion": "37", "pos": 1912634, @@ -57,6 +58,7 @@ "ref": "G", "alt": "T", "variant_id": "19-1912633-G-T", + "gene_ids": ["ENSG00000135953"], "saved_variant_json": { "liftedOverGenomeVersion": "37", "pos": 1912633, @@ -105,6 +107,7 @@ "ref": "G", "alt": "C", "variant_id": "19-1912632-G-C", + "gene_ids": ["ENSG00000135953"], "saved_variant_json": { "pos": 1912632, "end": 1912632, @@ -145,6 +148,7 @@ "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]} }, + "gene_ids": ["ENSG00000198727"], "saved_variant_json": { "variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783, "genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783, @@ -178,6 +182,7 @@ "genotypes": { "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, + "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], "saved_variant_json": { "liftedOverGenomeVersion": null, "pos": 249045487, diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py index 496fbf2092..89942141e4 100644 --- a/seqr/migrations/0081_savedvariant_gene_ids.py +++ b/seqr/migrations/0081_savedvariant_gene_ids.py @@ -51,7 +51,7 @@ def _transcript_sort(gene_id, saved_variant_json): gene_transcripts = saved_variant_json['transcripts'][gene_id] main_transcript_id = saved_variant_json.get('mainTranscriptId') is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) - return (is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) + return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) class Migration(migrations.Migration): diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 9c13be3624..c4fbb5c38c 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -95,7 +95,7 @@ 'chrom': '2', 'genotypes': {}, 'genomeVersion': '37', - 'mainTranscriptId': None, + 'mainTranscriptId': 'ENST00000450625', 'populations': { 'seqr': {'ac': 0, 'hom': 0}, 'exac': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': 0.0, 'hemi': 0, 'het': 0, 'hom': 0}, @@ -121,7 +121,25 @@ 'splice_ai_consequence': None, 'vest': None}, 'ref': 'AAAG', - 'transcripts': {}, + 'transcripts': { + 'ENSG00000177000': [ + {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { + 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', + 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, + }}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426925.5:c.-677A>C', 'hgvsp': None, 'transcriptId': 'ENST00000426925', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 11}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 2, 'utrannotator': { + 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 8,'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', + 'fiveutrAnnotation': {'type': None, 'KozakContext': 'TCAATGC', 'KozakStrength': 'Weak', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 588, 'FrameWithCDS': 'inFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None, 'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, + }}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000412806.1:c.138+1722A>C', 'hgvsp': None, 'transcriptId': 'ENST00000412806', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3},'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 3}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000459627.1:n.298A>C', 'hgvsp': None, 'transcriptId': 'ENST00000459627', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 3}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 4}, + {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000465894.6:n.33A>C', 'hgvsp': None, 'transcriptId': 'ENST00000465894', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 5}, + ], + 'ENSG00000277258': [ + {'aminoAcids': 'L/F', 'canonical': None, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000277258', 'hgvsc': 'ENST00000450625.1:c.156A>C', 'hgvsp': 'ENSP00000389484.1:p.Leu52Phe', 'transcriptId': 'ENST00000450625', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant', 'NMD_transcript_variant'], 'transcriptRank': 6}, + ] + }, 'familyGuids': ['F000001_1', 'F000002_2'], 'variantId': '2-61413835-AAAG-A', 'CAID': None, @@ -385,7 +403,7 @@ def test_create_saved_variant(self): saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') variant_json = {'xpos': 2061413835} variant_json.update(create_variant_json) - self._assert_created_variant(saved_variant, variant_json) + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258', 'ENSG00000177000']) variant_json.update({ 'variantGuid': variant_guid, @@ -394,6 +412,8 @@ def test_create_saved_variant(self): 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], + 'transcripts': mock.ANY, + 'mainTranscriptId': mock.ANY, }) response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][variant_guid] @@ -415,9 +435,10 @@ def test_create_saved_variant(self): def _format_create_variant(self, variant_json, **kwargs): return variant_json - def _assert_created_variant(self, saved_variant, variant_json, **kwargs): + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, **kwargs): for field in ['xpos', 'ref', 'alt']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) + self.assertEqual(saved_variant.gene_ids, gene_ids or []) def test_create_saved_sv_variant(self): # SVs are only supported on build 38 @@ -465,7 +486,7 @@ def test_create_saved_sv_variant(self): saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1') variant_json.update({'xpos': 2061413835}) - self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES') + self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', gene_ids=['ENSG00000240361']) self.assertEqual(saved_variant.xpos_end, 2061414175) variant_json.update({ @@ -1030,7 +1051,7 @@ class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): SAVED_VARIANT_DETAIL_FIELDS = SAVED_VARIANT_DETAIL_FIELDS def _assert_created_variant(self, saved_variant, variant_json, **kwargs): - super()._assert_created_variant(saved_variant, variant_json) + super()._assert_created_variant(saved_variant, variant_json, **kwargs) self.assertDictEqual(variant_json, saved_variant.saved_variant_json) @@ -1121,7 +1142,7 @@ def _format_create_variant(self, variant_json, key=123, **kwargs): return {**variant_json, 'key': key} def _assert_created_variant(self, saved_variant, variant_json, dataset_type='SNV_INDEL', **kwargs): - super()._assert_created_variant(saved_variant, variant_json) + super()._assert_created_variant(saved_variant, variant_json, **kwargs) self.assertEqual(variant_json['key'], saved_variant.key) self.assertEqual(dataset_type, saved_variant.dataset_type) self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index ee14b962c9..d9c382447e 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -121,7 +121,7 @@ def _transcript_sort(gene_id, saved_variant_json): gene_transcripts = saved_variant_json['transcripts'][gene_id] main_transcript_id = saved_variant_json.get('mainTranscriptId') is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) - return (is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) + return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100) def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=False): From 494c1ff70e68827e396fa15874b1dc1a38ac514b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 10:26:48 -0400 Subject: [PATCH 50/71] fix migration output field --- seqr/migrations/0081_savedvariant_gene_ids.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py index 89942141e4..f41d49cfe7 100644 --- a/seqr/migrations/0081_savedvariant_gene_ids.py +++ b/seqr/migrations/0081_savedvariant_gene_ids.py @@ -37,7 +37,10 @@ def populate_variant_gene_ids(apps, schema_editor): for i in range(0, len(all_keys), BATCH_SIZE): batch_keys = all_keys[i:i + BATCH_SIZE] clickhouse_qs = get_annotations_queryset(genome_version, dataset_type, batch_keys) - gene_ids_expr = ArrayMap(clickhouse_qs.transcript_field, mapped_expression='x.geneId') + gene_ids_expr = ArrayMap( + clickhouse_qs.transcript_field, mapped_expression='x.geneId', + output_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=20)), + ) to_update = [] for key, gene_ids in clickhouse_qs.values_list('key', gene_ids_expr): for variant in variants_by_key[key]: From e732f35602223d60fc86042114727cc250d20013 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 10:33:41 -0400 Subject: [PATCH 51/71] update saved variant gene filter --- clickhouse_search/search.py | 15 ----------- seqr/views/apis/summary_data_api.py | 42 +++-------------------------- 2 files changed, 3 insertions(+), 54 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 061d2c63b0..f5517594ea 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -646,21 +646,6 @@ def get_clickhouse_genes(genome_version, dataset_type, keys): )['gene_ids'] -def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): - results = get_annotations_queryset(genome_version, dataset_type, keys) - return list(results.filter( - **{f'{results.transcript_field}__array_exists': {'geneId': (f"'{gene_id}'",)}}, - ).values_list('key', flat=True)) - - -def get_all_clickhouse_keys_for_gene(gene_model, genome_version): - entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - return list(entry_cls.objects.filter_locus(require_gene_filter=True, genes={gene_model.gene_id: { - 'id': gene_model.id, - **{f'{field}Grch{genome_version}': getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}}, - }).values_list('key', flat=True).distinct()) - - def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False): key_lookup_class = KEY_LOOKUP_CLASS_MAP[genome_version][dataset_type] lookup = {} diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 707bfb9737..4a4940fc73 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -1,25 +1,22 @@ from collections import defaultdict from datetime import datetime -from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import PermissionDenied from django.core.mail.message import EmailMessage from django.contrib.auth.models import User -from django.db.models import CharField, F, Q, Value +from django.db.models import CharField, F, Value from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf import json -from clickhouse_search.search import get_clickhouse_keys_for_gene, get_all_clickhouse_keys_for_gene from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission -from reference_data.models import HumanPhenotypeOntology, GeneInfo, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 +from reference_data.models import HumanPhenotypeOntology from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy, Sample from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_utils import create_json_response from seqr.utils.logging_utils import SeqrLogger from seqr.utils.xpos_utils import get_chrom_pos @@ -115,11 +112,7 @@ def saved_variants_page(request, tag): saved_variant_models = saved_variant_models.filter(family__project__guid__in=get_project_guids_user_can_view(request.user)) if gene: - gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( # TODO update - lambda *args: Q(), - _saved_variant_with_clickhouse_gene_q, - )(saved_variant_models, gene, is_all_tags) - saved_variant_models = saved_variant_models.filter(gene_filter) + saved_variant_models = saved_variant_models.filter(gene_ids__overlap=[gene]) elif saved_variant_models.count() > MAX_SAVED_VARIANTS: return create_json_response({'error': 'Select a gene to filter variants'}, status=400) @@ -131,35 +124,6 @@ def saved_variants_page(request, tag): return create_json_response(response_json) -def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id, is_all_tags): - key_qs = [] - if is_all_tags: - gene_model = GeneInfo.objects.get(gene_id=gene_id) - for genome_version in [GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38]: - gene_keys = get_all_clickhouse_keys_for_gene(gene_model, genome_version) - if gene_keys: - key_qs.append( - Q(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, family__project__genome_version=genome_version, key__in=gene_keys) - ) - saved_variant_models = saved_variant_models.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - - search_type_keys = saved_variant_models.filter(key__isnull=False).values( - 'dataset_type', genome_version=F('family__project__genome_version'), - ).annotate(keys=ArrayAgg('key', distinct=True)) - for agg in search_type_keys: - gene_keys = get_clickhouse_keys_for_gene(gene_id, **agg) - if gene_keys: - q = Q(dataset_type=agg['dataset_type'], family__project__genome_version=agg['genome_version'], key__in=gene_keys) - key_qs.append(q) - - if not key_qs: - return Q() - has_key_q = key_qs[0] - for q in key_qs[1:]: - has_key_q |= q - return has_key_q - - @login_and_policies_required def hpo_summary_data(request, hpo_id): data = Individual.objects.filter( From 737cb207fc4588fcabb8974f0428a18faa7e6b18 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 11:11:53 -0400 Subject: [PATCH 52/71] clean up family discovery genes --- seqr/views/utils/project_context_utils.py | 62 +++++++++++--------- ui/pages/Project/components/FamilyPage.jsx | 6 +- ui/pages/Project/reducers.js | 2 +- ui/shared/components/panel/family/Family.jsx | 2 +- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index f1ed20a860..45c440dbd4 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -118,25 +118,29 @@ def add_child_ids(response): def families_discovery_tags(families, genome_version, project=None): - families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} + families_by_guid = {f['familyGuid']: dict(discoveryGeneIds=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} discovery_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ) + ).annotate(family_guid=F('family__guid')) + + family_discovery_genes = list(discovery_variants.filter( + Q(gene_ids__len=1) | (Q(selected_main_transcript_id__isnull=True) & ~Q(dataset_type__startswith='SV')) + ).values_list('family_guid', F('gene_ids__0'))) try: - discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)( # TODO - discovery_variants, genome_version=genome_version, + backend_specific_call(_get_no_key_selected_transcript_gene_id, _get_clickhouse_selected_transcript_gene_id)( + family_discovery_genes, + discovery_variants.filter(selected_main_transcript_id__isnull=False, gene_ids__len__gt=1), + genome_version=genome_version, ) except Exception as e: logger.error(f'Error loading discovery genes from clickhouse: {e}', None) - discovery_tags = [] gene_ids = set() - for tag in discovery_tags: - tag['transcripts'] = tag.get('transcripts') or {} - gene_ids.update(list(tag['transcripts'].keys())) - families_by_guid[tag.pop('family_guid')]['discoveryTags'].append(tag) + for family_guid, gene_id in family_discovery_genes: + gene_ids.add(gene_id) + families_by_guid[family_guid]['discoveryGeneIds'].append(gene_id) return { 'familiesByGuid': families_by_guid, @@ -144,37 +148,39 @@ def families_discovery_tags(families, genome_version, project=None): } -def _get_no_key_tags(discovery_variants, **kwargs): - return discovery_variants.values( - family_guid=F('family__guid'), selectedMainTranscriptId=F('selected_main_transcript_id'), - transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), - ) +def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): + return next(( + gene_id for gene_id, gene_transcripts in transcripts.items() + if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts) + ), None) + + +def _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants, **kwargs): + family_discovery_genes += [ + (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts)) + for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values( + 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts', + ) + ] -def _get_clickhouse_tags(discovery_variants, genome_version): - discovery_tags = list(_get_no_key_tags(discovery_variants.filter(key__isnull=True))) +def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version): + _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants.filter(key__isnull=True)) tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( keys=ArrayAgg('key', distinct=True), - tags=ArrayAgg(JSONObject(key='key', family_guid='family__guid', selectedMainTranscriptId='selected_main_transcript_id')), + variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')), ) - for dataset_type, keys, tags in tags_by_dataset_type.values_list('dataset_type', 'keys', 'tags'): + for dataset_type, keys, variants in tags_by_dataset_type.values_list('dataset_type', 'keys', 'variants'): if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: transcripts_by_key = get_transcripts_by_key(genome_version, keys) else: qs = get_annotations_queryset(genome_version, dataset_type, keys) transcripts_by_key = dict(qs.values_list('key', qs.transcript_field)) - for tag in tags: - key = tag.pop('key') - tag['transcripts'] = transcripts_by_key[key] - tag['mainTranscriptId'] = next(( - t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts - if t.get('transcriptRank') == 0 - ), None) - discovery_tags.append(tag) - - return discovery_tags + for v in variants: + gene_id = _get_transcripts_selected_gene(v['selected_main_transcript_id'], transcripts_by_key[v['key']]) + family_discovery_genes.append((v['family_guid'], gene_id)) MME_TAG_NAME = 'MME Submission' diff --git a/ui/pages/Project/components/FamilyPage.jsx b/ui/pages/Project/components/FamilyPage.jsx index 71c584612d..b81f57f106 100644 --- a/ui/pages/Project/components/FamilyPage.jsx +++ b/ui/pages/Project/components/FamilyPage.jsx @@ -40,8 +40,8 @@ SearchLink.propTypes = { } const DiscoveryGenes = React.memo(({ family, genesById }) => { - const discoveryGenes = (family.discoveryTags || []).map( - tag => (genesById[getVariantMainGeneId(tag)] || {}).geneSymbol, + const discoveryGenes = (family.discoveryGeneIds || []).map( + geneId => (genesById[geneId] || {}).geneSymbol, ).filter(val => val) return discoveryGenes.length > 0 ? ( @@ -59,7 +59,7 @@ DiscoveryGenes.propTypes = { const BaseVariantDetail = ( { project, family, hasActiveVariantSample, compact, genesById, tagTypeCounts, load, loading }, ) => ( - + (dispatch, getState) => { export const loadProjectLocusLists = () => loadCurrentProjectChildEntities('locus lists', REQUEST_LOCUS_LISTS, RECEIVE_LOCUS_LISTS) export const loadFamilyVariantSummary = familyGuid => loadFamilyData( - familyGuid, 'discoveryTags', 'variant_tag_summary', REQUEST_FAMILY_VARIANT_SUMMARY, + familyGuid, 'discoveryGeneIds', 'variant_tag_summary', REQUEST_FAMILY_VARIANT_SUMMARY, ) export const loadSavedVariants = ({ familyGuids, variantGuid, tag }) => (dispatch, getState) => { diff --git a/ui/shared/components/panel/family/Family.jsx b/ui/shared/components/panel/family/Family.jsx index 659a8f5739..eba08fa0b8 100644 --- a/ui/shared/components/panel/family/Family.jsx +++ b/ui/shared/components/panel/family/Family.jsx @@ -129,7 +129,7 @@ const FAMILY_FIELD_RENDER_LOOKUP = { [FAMILY_FIELD_DISCOVERY_MONDO_ID]: { ...MONDO_FIELD, internal: true, - canEditFamily: ({ discoveryTags }) => discoveryTags?.length > 0, + canEditFamily: ({ discoveryGeneIds }) => discoveryGeneIds?.length > 0, }, [FAMILY_FIELD_OMIM_NUMBERS]: { canEditFamily: ({ postDiscoveryOmimOptions }) => Object.keys(postDiscoveryOmimOptions || {}).length > 0, From 47e8f9f6b56cc67a0668d7dae247a51113a23f73 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 11:24:43 -0400 Subject: [PATCH 53/71] fix tests --- seqr/views/apis/family_api_tests.py | 8 ++--- seqr/views/apis/project_api_tests.py | 43 ++++++----------------- seqr/views/utils/project_context_utils.py | 4 +-- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 19c9b98f24..96b0feaeac 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -231,12 +231,8 @@ def test_family_variant_tag_summary(self): self.assertSetEqual(set(response_json.keys()), response_keys) family = response_json['familiesByGuid'][FAMILY_GUID] - self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryTags'}) - self.assertListEqual(family['discoveryTags'], [{ - 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, - 'mainTranscriptId': 'ENST00000258436', - 'selectedMainTranscriptId': None, - }]) + self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryGeneIds'}) + self.assertListEqual(family['discoveryGeneIds'], ['ENSG00000135953']) project = response_json['projectsByGuid'][PROJECT_GUID] self.assertSetEqual(set(project.keys()), {'variantTagTypes', 'variantFunctionalTagTypes'}) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 81d8c13f87..80ba7b1c14 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -41,20 +41,11 @@ {'id': 'recSgwrXNkmlIB5eM', 'fields': {'Status': 'Available in Seqr'}}, ]} -DISCOVERY_TAG = { - 'mainTranscriptId': None, - 'selectedMainTranscriptId': 'ENST00000371839', - 'transcripts': { - 'ENSG00000135953': mock.ANY, - 'ENSG00000240361': mock.ANY, - }, -} class ProjectAPITest(object): CREATE_PROJECT_JSON = WORKSPACE_CREATE_PROJECT_JSON REQUIRED_FIELDS = ['name', 'genomeVersion', 'workspaceNamespace', 'workspaceName'] AIRTABLE_TRACKING_URL = f'{MOCK_AIRTABLE_URL}/appUelDNM3BnWaR7M/AnVIL%20Seqr%20Loading%20Requests%20Tracking' - DISCOVERY_TAG = DISCOVERY_TAG @mock.patch('seqr.views.utils.airtable_utils.logger') @mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_URL', MOCK_AIRTABLE_URL) @@ -387,8 +378,7 @@ def test_project_families(self): empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) self._check_empty_project(empty_url, response_keys) - gene_ids = self._assert_expected_project_families(url, response_keys) - self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'}) + self._assert_expected_project_families(url, response_keys) def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False): response = self.client.get(url) @@ -401,7 +391,7 @@ def _assert_expected_project_families(self, url, response_keys, no_discovery_tag family_3 = response_json['familiesByGuid']['F000003_3'] empty_family = response_json['familiesByGuid']['F000013_13'] family_fields = { - 'individualGuids', 'discoveryTags', 'caseReviewStatuses', 'caseReviewStatusLastModified', 'hasRequiredMetadata', + 'individualGuids', 'discoveryGeneIds', 'caseReviewStatuses', 'caseReviewStatusLastModified', 'hasRequiredMetadata', 'parents', 'hasPhenotypePrioritization', 'hasRna', 'externalData', } family_fields.update(SUMMARY_FAMILY_FIELDS) @@ -434,22 +424,17 @@ def _assert_expected_project_families(self, url, response_keys, no_discovery_tag self.assertListEqual(empty_family['externalData'], []) - self.assertListEqual(family_3['discoveryTags'], []) - self.assertListEqual(empty_family['discoveryTags'], []) - family_1_tags = [] if no_discovery_tags else [{ - 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, - 'mainTranscriptId': 'ENST00000258436', - 'selectedMainTranscriptId': None, - }] - self.assertListEqual(family_1['discoveryTags'], family_1_tags) - family_2_tags = [] if no_discovery_tags else [self.DISCOVERY_TAG] - self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], family_2_tags) + self.assertListEqual(family_3['discoveryGeneIds'], []) + self.assertListEqual(empty_family['discoveryGeneIds'], []) + self.assertListEqual(family_1['discoveryGeneIds'], ['ENSG00000135953']) + family_2_tags = [] if no_discovery_tags else ['ENSG00000135953'] + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryGeneIds'], family_2_tags) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ - len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families + len(response_json['familiesByGuid'][family_guid]['discoveryGeneIds']) for family_guid in no_discovery_families }, {0}) - return set(response_json['genesById'].keys()) + self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953'}) def test_project_individuals(self): url = reverse(project_individuals, args=[PROJECT_GUID]) @@ -722,7 +707,6 @@ class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest): PROJECT_COLLABORATORS = ANVIL_COLLABORATORS PROJECT_COLLABORATOR_GROUPS = None HAS_EMPTY_PROJECT = False - DISCOVERY_TAG = {**DISCOVERY_TAG, 'mainTranscriptId': 'ENST00000505820'} def test_create_and_delete_project(self, *args, **kwargs): super(AnvilProjectAPITest, self).test_create_and_delete_project(*args, **kwargs) @@ -798,20 +782,15 @@ def test_project_collaborators(self): self.assertEqual(self.mock_get_ws_access_level.call_count, 5) def _assert_expected_project_families(self, *args, **kwargs): - gene_ids = super()._assert_expected_project_families(*args, **kwargs) + super()._assert_expected_project_families(*args, **kwargs) # Test success when clickhouse is unavailable self.reset_logs() connections['clickhouse'].close() - self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}} - no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True) - self.assertSetEqual(no_clickhouse_gene_ids, set()) + super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True) self.assert_json_logs(None, [ ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { 'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', }), ]) - - return gene_ids - diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 45c440dbd4..8e96d02f1a 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -150,7 +150,7 @@ def families_discovery_tags(families, genome_version, project=None): def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): return next(( - gene_id for gene_id, gene_transcripts in transcripts.items() + gene_id for gene_id, gene_transcripts in (transcripts or {}).items() if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts) ), None) @@ -158,7 +158,7 @@ def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts): def _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants, **kwargs): family_discovery_genes += [ (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts)) - for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values( + for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values_list( 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts', ) ] From 79d65e8a7cb790723cc247e8a9fa561334f65e92 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 11:58:40 -0400 Subject: [PATCH 54/71] remove clickhosue call from matchmaker summary --- matchmaker/views/matchmaker_api.py | 48 ++++++++-------------- matchmaker/views/matchmaker_api_tests.py | 8 ++-- ui/pages/Project/components/FamilyPage.jsx | 2 +- ui/pages/Project/selectors.js | 4 +- 4 files changed, 22 insertions(+), 40 deletions(-) diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 17f0450d3e..601546a59f 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -2,10 +2,9 @@ import requests from datetime import datetime from django.core.mail.message import EmailMessage -from django.db.models import prefetch_related_objects, Q, F, Value, CharField +from django.db.models import prefetch_related_objects, Q, Value, CharField from django.db.models.functions import Coalesce -from clickhouse_search.search import get_transcripts_by_key from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \ MatchmakerIncomingQuery from matchmaker.matchmaker_utils import get_mme_genes_phenotypes_for_results, parse_mme_patient, \ @@ -15,12 +14,11 @@ from seqr.utils.communication_utils import safe_post_to_slack from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.utils import backend_specific_call from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response -from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants_with_tags, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants, \ get_json_for_matchmaker_submission, get_json_for_matchmaker_submissions from seqr.views.utils.permissions_utils import check_mme_permissions, check_project_permissions, analyst_required, \ has_project_permissions, login_and_policies_required, get_project_and_check_permissions @@ -50,44 +48,30 @@ def get_individual_mme_matches(request, submission_guid): results = MatchmakerResult.objects.filter(submission=submission) - response_json = get_json_for_saved_variants_with_tags( - SavedVariant.objects.filter(family=submission.individual.family), genome_version=project.genome_version, additional_values={ - 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), - 'transcripts': F('saved_variant_json__transcripts'), - }, additional_model_fields=backend_specific_call([], ['key', 'genotypes']), - ) + hpo_terms_by_id = get_hpo_terms_by_id( + {feature['id'] for feature in (submission.features or []) if feature.get('id')}) + phenotypes = parse_mme_features(submission.features, hpo_terms_by_id) + response_json = _get_submission_detail_response(submission, phenotypes) - variants = response_json['savedVariantsByGuid'].values() - backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)( # TODO - variants, project.genome_version, request.user, - ) + gene_variants = response_json['mmeSubmissionsByGuid'][submission.guid]['geneVariants'] + gene_ids = {gv['geneId'] for gv in gene_variants} + variant_guids = {gv['variantGuid'] for gv in gene_variants} - gene_ids = set() + variants = get_json_for_saved_variants( + SavedVariant.objects.filter(guid__in=variant_guids), genome_version=project.genome_version, additional_values={ + 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + }, + ) + response_json['savedVariantsByGuid'] = {} for variant in variants: chrom, pos = get_chrom_pos(variant['xpos']) variant.update({'chrom': chrom, 'pos': pos}) - gene_ids.update(list(variant.get('transcripts', {}).keys())) - - hpo_terms_by_id = get_hpo_terms_by_id( - {feature['id'] for feature in (submission.features or []) if feature.get('id')}) - phenotypes = parse_mme_features(submission.features, hpo_terms_by_id) - response_json.update(_get_submission_detail_response(submission, phenotypes)) + response_json['savedVariantsByGuid'][variant['variantGuid']] = variant return _parse_mme_results( submission, results, request.user, additional_genes=gene_ids, response_json=response_json) -def _add_clickhouse_transcripts(variants, genome_version, user): - keys = {variant['key'] for variant in variants} - try: - transcripts_by_key = get_transcripts_by_key(genome_version, keys) - except Exception as e: - logger.error(f'Error loading transcripts from clickhouse: {e}', user) - transcripts_by_key = {} - for variant in variants: - variant['transcripts'] = transcripts_by_key.get(variant['key'], {}) - - @login_and_policies_required def get_mme_nodes(request): return create_json_response({'mmeNodes': list(MME_NODES_BY_NAME.keys())}) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index bea9a7694c..43ef6a8c8c 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -190,9 +190,7 @@ def test_get_individual_mme_matches(self): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), { - 'mmeResultsByGuid', 'genesById', 'savedVariantsByGuid', 'variantTagsByGuid', - 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'mmeContactNotes', - 'mmeSubmissionsByGuid', + 'mmeResultsByGuid', 'genesById', 'savedVariantsByGuid', 'mmeContactNotes', 'mmeSubmissionsByGuid', }) self.assertSetEqual( @@ -228,9 +226,9 @@ def test_get_individual_mme_matches(self): ) self.assertSetEqual( set(response_json['savedVariantsByGuid'].keys()), - {'SV0000001_2103343353_r0390_100', 'SV0059957_11562437_f019313_1', 'SV0059956_11560662_f019313_1'} + {'SV0000001_2103343353_r0390_100'} ) - saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'transcripts', 'genomeVersion', 'noteGuids', 'functionalDataGuids', 'tagGuids'} + saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'genomeVersion'} self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields) self.assertDictEqual(response_json['mmeContactNotes'], {}) diff --git a/ui/pages/Project/components/FamilyPage.jsx b/ui/pages/Project/components/FamilyPage.jsx index b81f57f106..76a35f7757 100644 --- a/ui/pages/Project/components/FamilyPage.jsx +++ b/ui/pages/Project/components/FamilyPage.jsx @@ -12,7 +12,7 @@ import { getGenesById, getHasActiveSearchSampleByFamily, } from 'redux/selectors' -import { FAMILY_DETAIL_FIELDS, getVariantMainGeneId } from 'shared/utils/constants' +import { FAMILY_DETAIL_FIELDS } from 'shared/utils/constants' import { Error404 } from 'shared/components/page/Errors' import Family from 'shared/components/panel/family/Family' import FamilyReads from 'shared/components/panel/family/FamilyReads' diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 84ff29ea22..c849ee0526 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -212,7 +212,7 @@ export const getTaggedVariantsByFamily = createSelector( getGenesById, getVariantTagsByGuid, (savedVariants, genesById, variantTagsByGuid) => Object.values(savedVariants).filter( - variant => variant.tagGuids.length, + variant => variant.tagGuids?.length, ).reduce((acc, variant) => { const { familyGuids, ...variantDetail } = variant variantDetail.tags = variant.tagGuids.map(tagGuid => variantTagsByGuid[tagGuid]) @@ -299,7 +299,7 @@ export const getIndividualTaggedVariants = createSelector( const { familyGuid } = individualsByGuid[individualGuid] return Object.values(taggedVariants[familyGuid] || []).reduce((acc, variant) => { const variantDetail = { - ...variant.genotypes[individualGuid], + ...(variant.genotypes || {})[individualGuid], ...variant, } return [...acc, ...variant.genes.filter(gene => gene).map(gene => ({ From 12c561df7ca413fcaf1ec5d40478ad8df504d586 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 12:06:59 -0400 Subject: [PATCH 55/71] clean up all discovery genes --- clickhouse_search/search.py | 9 +-------- seqr/views/apis/family_api.py | 30 +++-------------------------- seqr/views/apis/family_api_tests.py | 17 +--------------- 3 files changed, 5 insertions(+), 51 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f5517594ea..a7502e0fa8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -9,7 +9,7 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ - ArrayDistinct, ArrayMap + ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP @@ -639,13 +639,6 @@ def get_clickhouse_annotations(genome_version, dataset_type, keys): return format_clickhouse_results(results, genome_version) -def get_clickhouse_genes(genome_version, dataset_type, keys): - results = get_annotations_queryset(genome_version, dataset_type, keys) - return results.aggregate( - gene_ids=ArrayDistinct(GroupArrayArray(ArrayMap(results.transcript_field, mapped_expression='x.geneId')), output_field=ArrayField(StringField())), - )['gene_ids'] - - def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False): key_lookup_class = KEY_LOOKUP_CLASS_MAP[genome_version][dataset_type] lookup = {} diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 5059f80ada..8b23536b52 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -8,7 +8,6 @@ from django.db.models import Count, Max, Q, F from django.db.models.fields.files import ImageFieldFile -from clickhouse_search.search import get_clickhouse_genes from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display @@ -58,12 +57,10 @@ def family_page_data(request, family_guid): additional_fields = backend_specific_call([],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( - 'xpos', 'xpos_end', *additional_fields, - svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'), + 'xpos', 'xpos_end', 'gene_ids', *additional_fields, + svType=F('saved_variant_json__svType') ) - gene_ids = backend_specific_call( # TODO - _variants_gene_ids, _clickhouse_variants_gene_ids, - )(discovery_variants, project.genome_version, request.user) + gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] @@ -106,27 +103,6 @@ def family_page_data(request, family_guid): return create_json_response(response) -def _variants_gene_ids(variants, *args, **kwargs): - return {gene_id for variant in variants for gene_id in (variant['transcripts'] or {}).keys()} - - -def _clickhouse_variants_gene_ids(variants, genome_version, user): - keys_by_dataset_type = defaultdict(set) - no_key_variants = [] - for v in variants: - if v['key']: - keys_by_dataset_type[v['dataset_type']].add(v['key']) - else: - no_key_variants.append(v) - gene_ids = _variants_gene_ids(no_key_variants) - for dataset_type, keys in keys_by_dataset_type.items(): - try: - gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) - except Exception as e: - logger.error(f'Error loading genes from clickhouse: {e}', user) - return gene_ids - - def _intervals_overlap(interval1, interval2): return interval1['chrom'] == interval2['chrom'] and ( (interval2['start'] <= interval1['start'] <= interval2['end']) or diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 96b0feaeac..c75c101669 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -199,10 +199,8 @@ def test_family_page_data(self): 'end': 14409, }]}, } - self._assert_expected_family2_data(omim_options, url.replace(FAMILY_GUID, FAMILY_GUID2), response_keys, family_fields) - def _assert_expected_family2_data(self, omim_options, family_2_url, response_keys, family_fields): - response = self.client.get(family_2_url) + response = self.client.get(url.replace(FAMILY_GUID, FAMILY_GUID2)) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), response_keys) @@ -704,16 +702,3 @@ class AnvilFamilyAPITest(AnvilAuthenticationTestCase, FamilyAPITest): EXTERNAL_ANVIL_CAN_DELETE = True - def _assert_expected_family2_data(self, omim_options, *args, **kwargs): - super()._assert_expected_family2_data(omim_options, *args, **kwargs) - - # test graceful handling when clickhouse is down - self.reset_logs() - connections['clickhouse'].close() - super()._assert_expected_family2_data({}, *args, **kwargs) - self.assert_json_logs(self.analyst_user, [ - ("Error loading genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { - 'severity': 'ERROR', - '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', - }), - ]) From 699709cbcba1934af7872ecb7c5a5aa66c948335 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 12:18:02 -0400 Subject: [PATCH 56/71] minimize clickhosue calls for gene notes --- seqr/fixtures/1kg_project.json | 2 +- seqr/views/apis/saved_variant_api.py | 24 +++++++++--------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index bd77c0dfee..16d334cba4 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -2195,7 +2195,7 @@ "ref": null, "alt": null, "variant_id": "prefix_19107_DEL", - "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], + "gene_ids": ["ENSG00000135953", "ENSG00000223972", "ENSG00000240361"], "saved_variant_json": { "liftedOverGenomeVersion": null, "pos": 249045487, diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index ffa07ea469..eeaf16240b 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -5,7 +5,7 @@ from django.db.models import Q, F from clickhouse_search.backend.functions import ArrayFilter, ArrayMap -from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset +from clickhouse_search.search import get_transcripts_queryset from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json @@ -137,26 +137,20 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): def _variant_gene_id(variant, genome_version): - main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') - if main_transcript_id: + if variant.selected_main_transcript_id and len(variant.gene_ids) > 1: return next( gene_id for gene_id, transcripts in variant.saved_variant_json['transcripts'].items() - if any(t['transcriptId'] == main_transcript_id for t in transcripts)) - return next(gene_id for gene_id in sorted(variant.saved_variant_json['transcripts'])) + if any(t['transcriptId'] == variant.selected_main_transcript_id for t in transcripts)) + return variant.gene_ids[0] def _clickhouse_variant_gene_id(variant, genome_version): - if not variant.key: + if not (variant.key and variant.selected_main_transcript_id) or len(variant.gene_ids) == 1: return _variant_gene_id(variant, genome_version) - if variant.selected_main_transcript_id: - qs = get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap( - ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]), - mapped_expression='x.geneId', output_field=ArrayField(StringField()), - )).annotate(gene_id=F('gene_ids__0')) - else: - annotations = get_annotations_queryset(genome_version, variant.dataset_type, [variant.key]) - qs = annotations.annotate(gene_id=F(f'{annotations.transcript_field}__0__geneId')) - return qs.values_list('gene_id', flat=True)[0] + return get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap( + ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]), + mapped_expression='x.geneId', output_field=ArrayField(StringField()), + )).values_list(F('gene_ids__0'), flat=True)[0] @login_and_policies_required From 24d7e4a73a2fc7994f1fe48c606ef4f36cc05c6f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 12:40:43 -0400 Subject: [PATCH 57/71] report cleanup --- seqr/views/utils/anvil_metadata_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index b4185f21cc..5a862915bb 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -346,7 +346,7 @@ def _get_parsed_saved_discovery_variants_by_family( variant_json_fields.append('clinvar') variants = [] gene_ids = set() - variant_json_by_guid = backend_specific_call( # TODO potential update + variant_json_by_guid = backend_specific_call( _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, )(project_saved_variants, include_clinvar) for variant in project_saved_variants: @@ -411,7 +411,7 @@ def _get_variant_json_by_guid(saved_variants, *args, **kwargs): for v in saved_variants: main_transcript = _get_variant_main_transcript(v) gene_id = main_transcript.get('geneId') - gene_ids = [gene_id] if gene_id else v.saved_variant_json.get('transcripts', {}).keys() + gene_ids = [gene_id] if gene_id else v.gene_ids variant_json_by_guid[v.guid] = { **v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids, } @@ -426,7 +426,7 @@ def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar): for v in saved_variants: if v.key: end_chrom, end = get_chrom_pos(v.xpos_end) - variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end} + variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids} variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id)) else: no_key_variants.append(v) @@ -442,7 +442,7 @@ def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar): def _set_clickhouse_sv_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar): annotations = get_annotations_queryset(genome_version, dataset_type, key_map.keys()).values( - 'key', svType=F('sv_type'), gene_ids=ArrayMap('sorted_gene_consequences', mapped_expression='x.geneId', output_field=ArrayField(StringField())), + 'key', svType=F('sv_type'), ) defaults = {'CAID': None, 'gene_id': None, 'main_transcript': {}} if include_clinvar: @@ -486,8 +486,9 @@ def _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset **anns, 'main_transcript': main_transcript, 'gene_id': gene_id, - 'gene_ids': [gene_id] if gene_id else [transcript['geneId'] for transcript in all_transcripts], }) + if gene_id: + variant_json_by_guid[guid]['gene_ids'] = [gene_id] def _get_main_transcripts_expression(field_name, qs, selected_transcripts): From 1611f366d2089ae554c59e2296f59f4a1a12d211 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 13:01:19 -0400 Subject: [PATCH 58/71] set gene ids for bulk aip --- seqr/views/utils/anvil_metadata_utils.py | 3 +- seqr/views/utils/variant_utils.py | 40 +++++++----------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 5a862915bb..02b4af65a6 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,7 +1,6 @@ from collections import defaultdict from datetime import datetime -from clickhouse_backend.models import ArrayField, StringField from django.db.models import F, Q, Value, CharField, Aggregate from django.db.models.functions import Replace from django.contrib.auth.models import User @@ -9,7 +8,7 @@ import requests from typing import Callable, Iterable -from clickhouse_search.backend.functions import ArrayMap, ArrayFilter +from clickhouse_search.backend.functions import ArrayFilter from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index d9c382447e..4e806f6b7f 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -100,6 +100,8 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,): update_json = {'saved_variant_json': variant_json} if 'transcripts' in variant_json: update_json['gene_ids'] = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json)) + elif 'gene_ids' in variant_json: + update_json['gene_ids'] = variant_json['gene_ids'] return { 'xpos': xpos, 'xpos_end': xpos + var_length, @@ -156,15 +158,11 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use ).annotate(saved_variant_ids=ArrayAgg('saved_variants__id', ordering='id')) } - variant_genes_by_id = backend_specific_call( # TODO - _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, - )(saved_variant_map.values(), genome_version) - update_tags = [] num_new = 0 for key, variant in family_variant_data.items(): updated_tag = _set_updated_tags( - key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user, variant_genes_by_id, + key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user, ) if updated_tag: update_tags.append(updated_tag) @@ -178,7 +176,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_var_ids: list[str], saved_variant_map: dict[tuple[int, str], SavedVariant], existing_tags: dict[tuple[int, ...], VariantTag], - tag_type: VariantTagType, user: User, variant_genes_by_id: dict[str, list[str]]): + tag_type: VariantTagType, user: User): variant = saved_variant_map[key] existing_tag = existing_tags.get(tuple([variant.id])) updated_tag = None @@ -196,11 +194,11 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v VariantTag, {'variant_tag_type': tag_type, 'metadata': json.dumps(metadata)}, user) tag.saved_variants.add(variant) - variant_genes = variant_genes_by_id.get(key[1], set()) + variant_genes = set(variant.gene_ids or []) support_vars = [] for support_id in support_var_ids: support_v = saved_variant_map[(key[0], support_id)] - if variant_genes.intersection(variant_genes_by_id.get(support_id, set())): + if variant_genes.intersection(set(support_v.gene_ids)): support_vars.append(support_v) if support_vars: variants = [variant] + support_vars @@ -213,27 +211,6 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v return updated_tag -def _get_saved_variant_genes(variant_models, *args, **kwargs): - return {v.variant_id: set(v.saved_variant_json['transcripts'].keys()) for v in variant_models} - - -def _get_clickhouse_saved_variant_genes(variant_models, genome_version): - variant_genes_by_id = {} - key_id_map = {} - for v in variant_models: - if v.key: - key_id_map[v.key] = v.variant_id - else: - variant_genes_by_id[v.variant_id] = set(v.saved_variant_json['transcripts'].keys()) - qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, key_id_map.keys()) - variant_genes_by_id.update({ - key_id_map[key]: set(gene_ids) for key, gene_ids in qs.values_list( - 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())), - ) - }) - return variant_genes_by_id - - def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: User, genome_version: str) -> dict[tuple[int, str], dict]: family_ids = set() variant_families = defaultdict(list) @@ -285,12 +262,17 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples, ) + qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys()) + gene_ids_by_key = dict(qs.values_list( + 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())), + )) for key, genotypes in genotype_keys.items(): variant_id = variant_key_map[key] chrom, pos, ref, alt = variant_id.split('-') variants.append({ 'key': key, 'variantId': variant_id, 'chrom': chrom, 'pos': int(pos), 'ref': ref, 'alt': alt, 'genotypes': genotypes, 'familyGuids': sorted({g['familyGuid'] for g in genotypes.values()}), + 'gene_ids': gene_ids_by_key.get(key), }) return variants From 683875956fa4419e87ad4eac82bdb101c3e6a2c1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 13:12:25 -0400 Subject: [PATCH 59/71] set gene ids on bulk create gregor --- seqr/views/apis/individual_api_tests.py | 5 ++++- seqr/views/apis/summary_data_api_tests.py | 1 + seqr/views/utils/variant_utils.py | 16 +++++++++++----- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 0d97ea3e43..f75f5f1e95 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1146,7 +1146,7 @@ def test_import_gregor_metadata(self, mock_subprocess): ).order_by('family_id', 'variant_id').distinct().values( 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json__genomeVersion', 'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId', - 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes' + 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes', 'gene_ids', ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1162,6 +1162,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, + 'gene_ids': ['ENSG00000240361', 'ENSG00000135953'], }) self.assertEqual(len(saved_variants[0]['genotypes']), 2) self.assertDictEqual(saved_variants[1], { @@ -1179,6 +1180,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'key': None, 'dataset_type': None, 'genotypes': {}, + 'gene_ids': ['ENSG00000240361'], }) new_family_genotypes = {new_family_individual_guid: {'numAlt': 2}} self.assertDictEqual(saved_variants[2], { @@ -1194,6 +1196,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, + 'gene_ids': ['ENSG00000135953'], }) variant_tags = VariantTag.objects.filter(variant_tag_type__name='GREGoR Finding') diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 7df3cad33e..92fecee045 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -628,6 +628,7 @@ def _assert_expected_new_saved_variant(self, new_saved_variant): self.assertEqual(new_saved_variant.family_id, 2) self.assertEqual(new_saved_variant.ref, 'GAGA') self.assertEqual(new_saved_variant.alt, 'G') + self.assertListEqual(new_saved_variant.gene_ids, ['ENSG00000135953', 'ENSG00000228198']) def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False): self.assertEqual(response.status_code, 200) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 4e806f6b7f..ca52869588 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -140,7 +140,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) if load_new_variant_data else backend_specific_call( - lambda o, _: o, _get_clickhouse_variant_keys, # TODO need to have gene ids when new variants created (or update after) + lambda o, _: o, _get_clickhouse_variant_keys, )({k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): @@ -262,10 +262,7 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples, ) - qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys()) - gene_ids_by_key = dict(qs.values_list( - 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())), - )) + gene_ids_by_key = _get_gene_ids_by_key(genome_version, variant_key_map.keys()) for key, genotypes in genotype_keys.items(): variant_id = variant_key_map[key] chrom, pos, ref, alt = variant_id.split('-') @@ -277,12 +274,21 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, return variants +def _get_gene_ids_by_key(genome_version, keys): + qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys) + return dict(qs.values_list( + 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())), + )) + + def _get_clickhouse_variant_keys(variant_data: dict[tuple[int, str], dict], genome_version: str) -> dict[tuple[int, str], dict]: variant_ids = {key[1] for key in variant_data} variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, list(variant_ids)) + gene_ids_by_key = _get_gene_ids_by_key(genome_version, variant_key_map.values()) for (_, variant_id), variant in variant_data.items(): if variant_id in variant_key_map: variant['key'] = variant_key_map[variant_id] + variant['gene_ids'] = gene_ids_by_key.get(variant['key']) return variant_data From e91f91954b6a51cc437c9855f9fb9ed75e649efa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 13:22:31 -0400 Subject: [PATCH 60/71] cleanup --- seqr/views/apis/saved_variant_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index eeaf16240b..6fa1fd5b29 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -127,7 +127,7 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) # TODO + gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], From 52c94b78eaab315ebe30b262b857f0a6eee857dd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 13:30:53 -0400 Subject: [PATCH 61/71] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3045cb08ca..289ff71d1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # _seqr_ Changes ## dev +* Store gene ids on SavedVariant model (REQUIRES DB MIGRATION) ## 9/18/25 -# Deprecate hail backend support +* Deprecate hail backend support ## 8/7/25 * Deprecate support for lifting existing projects from GRCH37 to GRCh38 From 1707321e2a3e6ab798698cdfd8ce2a8d2ded85cd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 15:05:08 -0400 Subject: [PATCH 62/71] distict for migration --- seqr/migrations/0081_savedvariant_gene_ids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py index f41d49cfe7..2d3092b515 100644 --- a/seqr/migrations/0081_savedvariant_gene_ids.py +++ b/seqr/migrations/0081_savedvariant_gene_ids.py @@ -4,7 +4,7 @@ import django.contrib.postgres.fields from django.db import migrations, models -from clickhouse_search.backend.functions import ArrayMap +from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap from clickhouse_search.search import get_annotations_queryset BATCH_SIZE = 10000 @@ -37,10 +37,10 @@ def populate_variant_gene_ids(apps, schema_editor): for i in range(0, len(all_keys), BATCH_SIZE): batch_keys = all_keys[i:i + BATCH_SIZE] clickhouse_qs = get_annotations_queryset(genome_version, dataset_type, batch_keys) - gene_ids_expr = ArrayMap( + gene_ids_expr = ArrayDistinct(ArrayMap( clickhouse_qs.transcript_field, mapped_expression='x.geneId', output_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=20)), - ) + )) to_update = [] for key, gene_ids in clickhouse_qs.values_list('key', gene_ids_expr): for variant in variants_by_key[key]: From af0f51b90ec4434e474968aa2c2049c48fc74fbe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 15:47:22 -0400 Subject: [PATCH 63/71] fail file iter when error occurs --- seqr/utils/file_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/utils/file_utils.py b/seqr/utils/file_utils.py index 23afc08a7a..5120cee0f3 100644 --- a/seqr/utils/file_utils.py +++ b/seqr/utils/file_utils.py @@ -82,7 +82,7 @@ def file_iter(file_path, byte_range=None, raw_content=False, user=None, **kwargs def _google_bucket_file_iter(gs_path, byte_range=None, raw_content=False, user=None, **kwargs): """Iterate over lines in the given file""" range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else '' - process = _run_gsutil_command( + process = run_gsutil_with_wait( 'cat{}'.format(range_arg), gs_path, gunzip=gs_path.endswith("gz") and not raw_content, user=user, **kwargs) for line in process.stdout: if not raw_content: @@ -109,8 +109,8 @@ def _get_gs_file_list(gs_path, user, check_subfolders, allow_missing): return [line for line in all_lines if is_google_bucket_file_path(line)] -def run_gsutil_with_wait(command, gs_path, user=None): - process = _run_gsutil_command(command, gs_path, user=user) +def run_gsutil_with_wait(command, gs_path, user=None, **kwargs): + process = _run_gsutil_command(command, gs_path, user=user, **kwargs) if process.wait() != 0: errors = [line.decode('utf-8').strip() for line in process.stdout] raise Exception('Run command failed: ' + ' '.join(errors)) From de904a3da8987aebebaf832c03507b8cbbe76c82 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 16:15:41 -0400 Subject: [PATCH 64/71] update tests --- .../check_for_new_samples_from_pipeline_tests.py | 1 + .../tests/update_individuals_sample_qc_tests.py | 1 + seqr/views/apis/anvil_workspace_api_tests.py | 1 + seqr/views/apis/data_manager_api_tests.py | 2 ++ seqr/views/apis/individual_api_tests.py | 14 +++++++++++++- seqr/views/utils/file_utils_tests.py | 1 + 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index c3ba7986c9..f2a7c6b8ce 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -338,6 +338,7 @@ def mock_opened_file(index): m = mock.MagicMock() + m.wait.return_value = 0 m.stdout = [json.dumps(OPENED_RUN_JSON_FILES[index]).encode()] return m diff --git a/seqr/management/tests/update_individuals_sample_qc_tests.py b/seqr/management/tests/update_individuals_sample_qc_tests.py index 361b1b5698..4ef0c87b96 100644 --- a/seqr/management/tests/update_individuals_sample_qc_tests.py +++ b/seqr/management/tests/update_individuals_sample_qc_tests.py @@ -99,6 +99,7 @@ def test_command(self): # Test 'sample_qc' not in metadata self.mock_ls_process.communicate.return_value = b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-02-24/metadata.json\n', b'' self.mock_metadata_file.stdout = [json.dumps({}).encode()] + self.mock_metadata_file.wait.return_value = 0 self.mock_subprocess.side_effect = [self.mock_ls_process, self.mock_metadata_file] with self.assertRaises(CommandError): diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 4618e275ea..73cd7381d9 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -427,6 +427,7 @@ def test_validate_anvil_vcf(self, mock_subprocess, mock_file_logger, mock_utils_ mock_subprocess.side_effect = [mock_file_exist_or_list_subproc, mock_get_header_subproc] mock_file_exist_or_list_subproc.communicate.return_value = b'gs://test_bucket/test_path-001.vcf.gz\ngs://test_bucket/test_path-102.vcf.gz\n', None mock_get_header_subproc.stdout = BASIC_META + INFO_META + FORMAT_META + HEADER_LINE + DATA_LINES + mock_get_header_subproc.wait.return_value = 0 response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_SHARDED_DATA_PATH)) self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), {'fullDataPath': 'gs://test_bucket/test_path-*.vcf.gz', 'vcfSamples': ['HG00735', 'NA19675_1', 'NA19679']}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 72b73d34ac..0988473e9d 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -823,6 +823,7 @@ def _test_update_rna_seq(self, data_type, mock_open, mock_subprocess, mock_load_ mock_does_file_exist.wait.return_value = 0 mock_file_iter = mock.MagicMock() def _set_file_iter_stdout(rows): + mock_file_iter.wait.return_value = 0 mock_file_iter.stdout = [('\t'.join([str(col) for col in row]) + '\n').encode() for row in rows] mock_subprocess.side_effect = [mock_does_file_exist, mock_file_iter, mock_does_file_exist] @@ -1757,6 +1758,7 @@ def setUp(self): self.mock_does_file_exist = mock.MagicMock() self.mock_file_iter = mock.MagicMock() self.mock_file_iter.stdout = [] + self.mock_file_iter.wait.return_value = 0 self.mock_subprocess.side_effect = [self.mock_does_file_exist, self.mock_file_iter] self.addCleanup(patcher.stop) patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack') diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 0d97ea3e43..2f2ce640c7 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1018,7 +1018,7 @@ def test_import_gregor_metadata(self, mock_subprocess): 'OR4G11P', '', '', '', 'Heterozygous', '', 'unknown', 'Broad_NA20889_1_248367227', '', 'Candidate', 'IRIDA syndrome', 'MONDO:0008788', 'Autosomal dominant', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '', ]) - self._set_metadata_file_iter(mock_subprocess, genetic_findings_table) + mock_subprocess.return_value.stdout.__iter__.return_value = [b'Bucket is a requester pays bucket but no user project provided'] url = reverse(import_gregor_metadata, args=[PM_REQUIRED_PROJECT_GUID]) self.check_pm_login(url) @@ -1028,6 +1028,13 @@ def test_import_gregor_metadata(self, mock_subprocess): 'sampleType': 'exome', } response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'Run command failed: Bucket is a requester pays bucket but no user project provided'}) + + mock_subprocess.reset_mock() + self._set_metadata_file_iter(mock_subprocess, genetic_findings_table) + mock_subprocess.return_value.wait.return_value = 0 + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), { @@ -1224,14 +1231,19 @@ def test_import_gregor_metadata(self, mock_subprocess): mock_subprocess.assert_has_calls([ mock.call('gsutil cat gs://test_bucket/data_tables/experiment_dna_short_read.tsv', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), mock.call('gsutil cat gs://test_bucket/data_tables/experiment.tsv', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), mock.call('gsutil cat gs://test_bucket/data_tables/participant.tsv', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), mock.call('gsutil cat gs://test_bucket/data_tables/phenotype.tsv', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), mock.call('gsutil cat gs://test_bucket/data_tables/genetic_findings.tsv', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), ]) diff --git a/seqr/views/utils/file_utils_tests.py b/seqr/views/utils/file_utils_tests.py index 544183691d..1c4c2f15df 100644 --- a/seqr/views/utils/file_utils_tests.py +++ b/seqr/views/utils/file_utils_tests.py @@ -154,6 +154,7 @@ def test_temp_file_upload(self, *args, **kwargs): mock.call(f'gsutil mv {self._temp_file_path()} {gs_file}', stdout=-1, stderr=-2, shell=True), # nosec mock.call().wait(), mock.call(f'gsutil cat {gs_file} | gunzip -c -q - ', stdout=-1, stderr=-2, shell=True), # nosec + mock.call().wait(), mock.call().stdout.__iter__(), ]) From ce7122a01145a71822ece6303a6d4224de793caa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 16:43:25 -0400 Subject: [PATCH 65/71] fix test --- .../management/tests/set_saved_variant_key_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index be47bf5bb5..02d9cfaa95 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -26,6 +26,7 @@ def setUpTestData(cls): @mock.patch('seqr.utils.file_utils.subprocess.Popen') def test_command(self, mock_subprocess): mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + mock_subprocess.return_value.wait.return_value = 0 call_command('set_saved_variant_key') self.assert_json_logs(user=None, expected=[ @@ -106,6 +107,7 @@ class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest): @mock.patch('seqr.utils.file_utils.subprocess.Popen') def test_command(self, mock_subprocess): mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + mock_subprocess.return_value.wait.return_value = 0 call_command('set_saved_variant_key') self.assert_json_logs(user=None, expected=[ From 2fa6d1d1e36cc9e1be57dde10081dbd5426d683b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 17:03:13 -0400 Subject: [PATCH 66/71] clean up sv id mapping for set key --- .../fixtures/clickhouse_saved_variants.json | 8 +- .../commands/set_saved_variant_key.py | 2072 +---------------- .../tests/set_saved_variant_key_tests.py | 25 +- 3 files changed, 12 insertions(+), 2093 deletions(-) diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 6a7e7706e7..cc94a9fa12 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -213,7 +213,7 @@ "pk": 101, "fields": { "key": 101, - "variant_id": "suffix_19107_DEL" + "variant_id": "suffix_19107_DEL_013746" } }, { "model": "clickhouse_search.annotationssv", @@ -225,7 +225,7 @@ "end_chrom": null, "pos": 249045487, "end": 249045898, - "variant_id": "suffix_19107_DEL", + "variant_id": "suffix_19107_DEL_013746", "lifted_over_chrom": null, "lifted_over_pos": null, "rg37_locus_end": [null, null], @@ -250,7 +250,7 @@ "pk": 111, "fields": { "key": 111, - "variant_id": "prefix_70191_DEL" + "variant_id": "prefix_19107_DEL" } }, { "model": "clickhouse_search.annotationsgcnv", @@ -261,7 +261,7 @@ "chrom": "1", "pos": 249045487, "end": 249045898, - "variant_id": "prefix_70191_DEL", + "variant_id": "prefix_19107_DEL", "lifted_over_chrom": "2", "lifted_over_pos": null, "rg37_locus_end": [null, null], diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 07df47e39b..6b429e54b1 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -9,2046 +9,12 @@ from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import SavedVariant, Sample -from seqr.utils.file_utils import file_iter from seqr.utils.search.utils import parse_variant_id logger = logging.getLogger(__name__) BATCH_SIZE = 10000 -GCNV_CALLSET_PATH = 'gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz' -SV_ID_UPDATE_MAP = { - 'WGS': { - 'CMG.phase1_CMG_DEL_chr10_2038': 'phase2_DEL_chr10_4611', - 'CMG.phase1_CMG_DEL_chr11_2963': 'phase2_DEL_chr11_5789', - 'CMG.phase1_CMG_DEL_chr13_153': 'phase2_DEL_chr13_378', - 'cohort_2911.chr1.final_cleanup_BND_chr1_1167': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1376', - 'cohort_2911.chr1.final_cleanup_BND_chr1_1017': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1208', - 'cohort_2911.chr1.final_cleanup_BND_chr1_2837': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_3326', - 'cohort_2911.chr1.final_cleanup_DEL_chr1_12237': 'phase2_DEL_chr1_9347', - 'cohort_2911.chr1.final_cleanup_DEL_chr1_2953': 'phase2_DEL_chr1_2503', - 'cohort_2911.chrX.final_cleanup_CPX_chrX_20': 'cohort_2911.chrX.final_cleanup_CPX_chrX_19', - 'phase2_DUP_chr1_1164': 'phase4_all_batches.chr1.final_cleanup_DUP_chr1_1666', - 'phase2_CPX_chr1_73': 'phase4_all_batches.chr1.final_cleanup_CPX_chr1_85', - 'cohort_2911.chr1.final_cleanup_DEL_chr1_3844': 'phase2_DEL_chr1_3148', - 'cohort_2911.chr1.final_cleanup_DEL_chr1_4181': 'phase2_DEL_chr1_3389', - 'cohort_2911.chr1.final_cleanup_DEL_chr1_6414': 'phase2_DEL_chr1_5030', - 'cohort_2911.chr10.final_cleanup_DEL_chr10_6275': 'phase2_DEL_chr10_4600', - 'cohort_2911.chr10.final_cleanup_DEL_chr10_7793': 'phase2_DEL_chr10_5695', - 'cohort_2911.chr10.final_cleanup_DUP_chr10_1491': 'phase2_DUP_chr10_1234', - 'cohort_2911.chr10.final_cleanup_DUP_chr10_3659': 'phase2_DUP_chr10_3132', - 'cohort_2911.chr11.final_cleanup_DUP_chr11_3003': 'phase2_DUP_chr11_2757', - 'cohort_2911.chr12.final_cleanup_DEL_chr12_244': 'phase2_DEL_chr12_207', - 'cohort_2911.chr12.final_cleanup_DEL_chr12_5405': 'phase2_DEL_chr12_4072', - 'cohort_2911.chr12.final_cleanup_INS_chr12_42': 'phase4_all_batches.chr12.final_cleanup_INS_chr12_61', - 'cohort_2911.chr14.final_cleanup_DEL_chr14_1513': 'phase2_DEL_chr14_1140', - 'cohort_2911.chr14.final_cleanup_DEL_chr14_2595': 'phase2_DEL_chr14_1998', - 'cohort_2911.chr15.final_cleanup_DEL_chr15_3281': 'phase2_DEL_chr15_2486', - 'cohort_2911.chr16.final_cleanup_DEL_chr16_5308': 'phase2_DEL_chr16_4074', - 'cohort_2911.chr17.final_cleanup_BND_chr17_209': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_250', - 'cohort_2911.chr16.final_cleanup_BND_chr16_635': 'phase4_all_batches.chr16.final_cleanup_BND_chr16_744', - 'cohort_2911.chr17.final_cleanup_CPX_chr17_79': 'phase2_CPX_chr17_46', - 'cohort_2911.chr17.final_cleanup_DEL_chr17_306': 'phase2_DEL_chr17_271', - 'cohort_2911.chr17.final_cleanup_DEL_chr17_3161': 'phase2_DEL_chr17_2627', - 'cohort_2911.chr17.final_cleanup_DEL_chr17_4111': 'phase2_DEL_chr17_3390', - 'cohort_2911.chr17.final_cleanup_DEL_chr17_5157': 'phase2_DEL_chr17_4242', - 'cohort_2911.chr18.final_cleanup_DEL_chr18_3519': 'phase2_DEL_chr18_2585', - 'cohort_2911.chr19.final_cleanup_BND_chr19_162': 'phase4_all_batches.chr19.final_cleanup_BND_chr19_185', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_3395': 'phase2_DEL_chrX_2020', - 'cohort_2911.chr19.final_cleanup_DEL_chr19_1825': 'phase2_DEL_chr19_1567', - 'cohort_2911.chr19.final_cleanup_DEL_chr19_3787': 'phase2_DEL_chr19_3203', - 'cohort_2911.chr19.final_cleanup_DEL_chr19_807': 'phase2_DEL_chr19_692', - 'cohort_2911.chr19.final_cleanup_DEL_chr19_971': 'phase2_DEL_chr19_832', - 'cohort_2911.chr2.final_cleanup_DEL_chr2_7485': 'phase2_DEL_chr2_5601', - 'cohort_2911.chr20.final_cleanup_DEL_chr20_37': 'phase2_DEL_chr20_25', - 'cohort_2911.chr20.final_cleanup_INV_chr20_6': 'phase2_INV_chr20_3', - 'cohort_2911.chr22.final_cleanup_DEL_chr22_1440': 'phase2_DEL_chr22_1210', - 'cohort_2911.chr3.final_cleanup_DUP_chr3_566': 'phase2_DUP_chr3_522', - 'cohort_2911.chr4.final_cleanup_DEL_chr4_5267': 'phase2_DEL_chr4_3794', - 'cohort_2911.chr6.final_cleanup_DEL_chr6_9094': 'phase2_DEL_chr6_6868', - 'cohort_2911.chr7.final_cleanup_DEL_chr7_1540': 'phase2_DEL_chr7_1311', - 'cohort_2911.chr8.final_cleanup_DEL_chr8_75': 'phase2_DEL_chr8_67', - 'cohort_2911.chr8.final_cleanup_DEL_chr8_7934': 'phase2_DEL_chr8_5754', - 'cohort_2911.chr9.final_cleanup_DEL_chr9_6983': 'phase2_DEL_chr9_5130', - 'cohort_2911.chrX.final_cleanup_BND_chrX_166': 'phase4_all_batches.chrX.final_cleanup_BND_chrX_227', - 'cohort_2911.chrX.final_cleanup_CPX_chrX_106': 'cohort_2911.chrX.final_cleanup_CPX_chrX_105', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1861': 'phase2_DEL_chrX_1151', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1922': 'phase2_DEL_chrX_1184', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1936': 'phase2_DEL_chrX_1194', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1974': 'phase2_DEL_chrX_1222', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_2607': 'phase2_DEL_chrX_1550', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_2882': 'phase2_DEL_chrX_1715', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_3450': 'phase2_DEL_chrX_2061', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_6616': 'phase2_DEL_chrX_3880', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_7868': 'phase2_DEL_chrX_4582', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_7872': 'phase2_DEL_chrX_4585', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_7882': 'phase2_DEL_chrX_4594', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_1102': 'phase2_DUP_chrX_1000', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_1791': 'phase2_DUP_chrX_1595', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_2682': 'phase2_DUP_chrX_2394', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_3233': 'phase2_DUP_chrX_2875', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_3564': 'phase2_DUP_chrX_3160', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_3599': 'phase2_DUP_chrX_3192', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_3621': 'phase2_DUP_chrX_3211', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_600': 'phase2_DUP_chrX_554', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_656': 'phase2_DUP_chrX_606', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_818': 'phase2_DUP_chrX_743', - 'phase2_BND_chr17_485': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_710', - 'phase2_BND_chr1_1262': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1669', - 'phase2_BND_chr1_19': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_25', - 'phase2_BND_chr4_1666': 'phase4_all_batches.chr4.final_cleanup_BND_chr4_2264', - 'phase2_BND_chr5_1124': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1476', - 'phase2_DUP_chr11_942': 'phase4_all_batches.chr11.final_cleanup_DUP_chr11_1405', - 'cohort_2911.chr19.final_cleanup_DEL_chr19_3909': 'phase2_DEL_chr19_3310', - 'cohort_2911.chr19.final_cleanup_DUP_chr19_1707': 'phase2_DUP_chr19_1571', - 'cohort_2911.chr2.final_cleanup_DUP_chr2_3291': 'phase2_DUP_chr2_2862', - 'cohort_2911.chr3.final_cleanup_DEL_chr3_8602': 'phase2_DEL_chr3_6419', - 'cohort_2911.chr3.final_cleanup_BND_chr3_2956': 'phase4_all_batches.chr3.final_cleanup_BND_chr3_3398', - 'cohort_2911.chr6.final_cleanup_DEL_chr6_7703': 'phase2_DEL_chr6_5832', - 'cohort_2911.chrX.final_cleanup_CPX_chrX_69': 'phase4_all_batches.chrX.final_cleanup_CPX_chrX_63', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1879': 'phase2_DEL_chrX_1160', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_3396': 'phase2_DEL_chrX_2020', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_6041': 'phase2_DEL_chrX_3547', - 'phase2_CTX_chr3_2': 'phase4_all_batches.chr3.final_cleanup_CTX_chr3_3', - 'phase2_DEL_chr11_3941': 'phase4_all_batches.chr11.final_cleanup_DEL_chr11_6231', - 'phase2_DEL_chr16_453': 'phase2_DEL_chr16_452', - 'phase2_DEL_chr16_516': 'phase2_DEL_chr16_517', - 'phase2_DEL_chr20_2752': 'phase4_all_batches.chr20.final_cleanup_DEL_chr20_4722', - 'phase2_DUP_chr16_1292': 'phase4_all_batches.chr16.final_cleanup_DUP_chr16_1975', - 'cohort_2911.chr2.final_cleanup_DEL_chr2_10322': 'phase2_DEL_chr2_7665', - 'cohort_2911.chr9.final_cleanup_DEL_chr9_2715': 'phase2_DEL_chr9_2002', - 'cohort_2911.chrX.final_cleanup_DEL_chrX_1905': 'phase2_DEL_chrX_1175', - 'cohort_2911.chrX.final_cleanup_DUP_chrX_1533': 'phase4_all_batches.chrX.final_cleanup_DUP_chrX_2546', - 'phase2_CPX_chr1_28': 'cohort_2911.chr1.final_cleanup_CPX_chr1_49', - 'phase2_DEL_chr18_2435': 'cohort_2911.chr18.final_cleanup_DEL_chr18_3308', - 'phase2_DUP_chr8_4806': 'phase2_DUP_chr8_4807', - 'cohort_2911.chr2.final_cleanup_BND_chr2_3079': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_3571', - 'cohort_2911.chr2.final_cleanup_BND_chr2_166': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_192', - 'phase2_BND_chr13_369': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_478', - 'phase2_BND_chr7_230': 'phase4_all_batches.chr7.final_cleanup_BND_chr7_300', - 'phase2_BND_chr8_238': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_320', - 'phase2_BND_chr8_53': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_72', - 'phase2_BND_chr9_489': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_663', - 'cohort_2911.chr5.final_cleanup_BND_chr5_1116': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1251', - 'cohort_2911.chr9.final_cleanup_BND_chr9_764': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_849', - 'phase2_BND_chr13_368': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_479', - }, - 'WES': { - 'R4_variant_7334_DUP_08162023': 'R4_variant_7334_DUP', - 'prefix_112949_DEL': 'suffix_210553_DEL', - 'prefix_131670_DUP': 'suffix_251025_DUP', - 'suffix_19443_DEL_2': 'suffix_20030_DEL', - 'prefix_184342_DEL': 'suffix_194439_DEL', - 'prefix_255018_DEL': 'suffix_155939_DEL', - 'prefix_188042_DUP': 'suffix_107531_DUP', - 'suffix_104367_DUP_2': 'suffix_107531_DUP', - 'prefix_59865_DEL': 'suffix_124465_DEL', - 'suffix_120814_DEL_2': 'suffix_124465_DEL', - 'prefix_152595_DEL': 'suffix_286580_DEL', - 'suffix_277504_DEL_2': 'suffix_286581_DEL', - 'prefix_185630_DEL': 'suffix_217490_DEL', - 'suffix_210888_DEL_2': 'suffix_217490_DEL', - 'prefix_126071_DUP': 'suffix_48694_DUP', - 'suffix_47226_DUP_2': 'suffix_48694_DUP', - 'prefix_177696_DUP': 'suffix_23_DUP', - 'prefix_33049_DEL': 'suffix_123373_DEL', - 'suffix_119753_DEL_2': 'suffix_123373_DEL', - 'prefix_6968_DUP': 'suffix_8158_DUP', - 'prefix_111128_DEL': 'suffix_124250_DEL', - 'prefix_265455_DEL': 'suffix_124250_DEL', - 'prefix_133233_DEL': 'suffix_149757_DEL', - 'suffix_145373_DEL_2': 'suffix_149757_DEL', - 'prefix_104962_DEL': 'suffix_262703_DEL', - 'prefix_230669_DUP': 'suffix_37212_DUP', - 'prefix_195634_DUP': 'suffix_337680_DUP', - 'suffix_326904_DUP_2': 'suffix_337680_DUP', - 'prefix_252896_DUP': 'suffix_154760_DUP', - 'prefix_252895_DUP': 'suffix_150484_DUP', - 'prefix_192808_DUP': 'suffix_182662_DUP', - 'prefix_31312_DUP': 'suffix_343278_DUP', - 'suffix_332369_DUP_2': 'suffix_343278_DUP', - 'prefix_201559_DEL': 'suffix_228780_DEL', - 'prefix_173086_DEL': 'suffix_195404_DEL', - 'prefix_230567_DEL': 'suffix_261336_DEL', - 'prefix_25206_DEL': 'suffix_27995_DEL', - 'prefix_72517_DEL': 'suffix_27995_DEL', - 'prefix_194936_DUP': 'suffix_220746_DUP', - 'prefix_117065_DEL': 'suffix_131048_DEL', - 'suffix_127235_DEL_2': 'suffix_131048_DEL', - 'prefix_168621_DEL': 'suffix_336993_DEL', - 'prefix_297236_DEL': 'suffix_336993_DEL', - 'prefix_198998_DUP': 'suffix_225862_DUP', - 'prefix_239300_DUP': 'suffix_271302_DUP', - 'prefix_236836_DEL': 'suffix_268435_DEL', - 'prefix_31131_DUP': 'suffix_34872_DUP', - 'prefix_238314_DEL': 'suffix_336979_DEL', - 'prefix_75798_DUP': 'suffix_84697_DUP', - 'prefix_261715_DEL': 'suffix_337689_DEL', - 'prefix_183989_DEL': 'suffix_4946_DEL', - 'prefix_26448_DEL': 'suffix_43066_DEL', - 'suffix_41786_DEL_2': 'suffix_43066_DEL', - 'prefix_103176_DEL': 'suffix_114811_DEL', - 'suffix_111479_DEL_2': 'suffix_114811_DEL', - 'prefix_120528_DEL': 'suffix_135375_DEL', - 'prefix_296302_DEL': 'suffix_335637_DEL', - 'prefix_108274_DEL': 'suffix_121089_DEL', - 'prefix_46615_DEL': 'suffix_118481_DEL', - 'prefix_185022_DUP': 'suffix_12708_DUP', - 'suffix_12284_DUP_2': 'suffix_12708_DUP', - 'prefix_252585_DUP': 'suffix_309117_DUP', - 'suffix_299264_DUP_2': 'suffix_309117_DUP', - 'prefix_80031_DUP': 'suffix_241944_DUP', - 'suffix_234357_DUP_2': 'suffix_241944_DUP', - 'prefix_117179_DEL': 'suffix_131305_DEL', - 'prefix_71499_DUP': 'suffix_330986_DUP', - 'suffix_320433_DUP_2': 'suffix_330986_DUP', - 'prefix_24788_DUP': 'suffix_336670_DUP', - 'suffix_325911_DUP_2': 'suffix_336670_DUP', - 'prefix_192365_DUP': 'suffix_139090_DUP', - 'prefix_255360_DUP': 'suffix_337082_DUP', - 'prefix_237614_DEL': 'suffix_269379_DEL', - 'prefix_284003_DEL': 'suffix_321174_DEL', - 'suffix_12426_DEL_2': 'suffix_12852_DEL', - 'prefix_9852_DEL': 'suffix_12852_DEL', - 'suffix_50042_DEL_2': 'suffix_51609_DEL', - 'prefix_223675_DEL': 'suffix_51609_DEL', - 'prefix_175156_DEL': 'suffix_54122_DEL', - 'suffix_52499_DEL_2': 'suffix_54122_DEL', - 'prefix_3138_DEL': 'suffix_109381_DEL', - 'suffix_106163_DEL_2': 'suffix_109381_DEL', - 'prefix_255276_DEL': 'suffix_93464_DEL', - 'prefix_152594_DEL': 'suffix_286542_DEL', - 'prefix_77396_DEL': 'suffix_287070_DEL', - 'prefix_152599_DEL': 'suffix_287163_DEL', - 'prefix_152600_DEL': 'suffix_287381_DEL', - 'prefix_108317_DEL': 'suffix_161791_DEL', - 'prefix_201562_DEL': 'suffix_168210_DEL', - 'suffix_163193_DEL_2': 'suffix_168210_DEL', - 'prefix_246635_DEL': 'suffix_92687_DEL', - 'suffix_89974_DEL_2': 'suffix_92687_DEL', - 'prefix_246642_DEL': 'suffix_93299_DEL', - 'suffix_90573_DEL_2': 'suffix_93299_DEL', - 'prefix_246674_DUP': 'suffix_126676_DUP', - 'suffix_122983_DUP_2': 'suffix_126676_DUP', - 'prefix_199389_DEL': 'suffix_94004_DEL', - 'prefix_3593_DEL': 'suffix_27094_DEL', - 'prefix_100725_DEL': 'suffix_336955_DEL', - 'prefix_297227_DEL': 'suffix_336980_DEL', - 'prefix_179648_DEL': 'suffix_202719_DEL', - 'prefix_143182_DEL': 'suffix_100756_DEL', - 'prefix_34890_DEL': 'suffix_38919_DEL', - 'prefix_36703_DEL': 'suffix_39815_DEL', - 'prefix_164553_DEL': 'suffix_185929_DEL', - 'prefix_132206_DEL': 'suffix_282692_DEL', - 'suffix_273714_DEL_2': 'suffix_282692_DEL', - 'prefix_144442_DEL': 'suffix_282716_DEL', - 'suffix_273738_DEL_2': 'suffix_282716_DEL', - 'suffix_273732_DEL_2': 'suffix_282710_DEL', - 'prefix_6326_DEL': 'suffix_282710_DEL', - 'prefix_178200_DEL': 'suffix_118870_DEL', - 'suffix_115409_DEL_2': 'suffix_118870_DEL', - 'prefix_254507_DUP': 'suffix_283948_DUP', - 'prefix_254510_DUP': 'suffix_39298_DUP', - 'prefix_13260_DEL': 'suffix_38202_DEL', - 'suffix_37073_DEL_2': 'suffix_38202_DEL', - 'prefix_278497_DEL': 'suffix_338533_DEL', - 'suffix_327734_DEL_2': 'suffix_338533_DEL', - 'prefix_242131_DEL': 'suffix_176483_DEL', - 'suffix_171236_DEL_2': 'suffix_176483_DEL', - 'suffix_32210_DEL_2': 'suffix_33192_DEL', - 'prefix_37286_DEL': 'suffix_33192_DEL', - 'prefix_113302_DUP': 'suffix_236396_DUP', - 'suffix_228996_DUP_2': 'suffix_236396_DUP', - 'prefix_154396_DEL': 'suffix_174270_DEL', - 'prefix_71769_DEL': 'suffix_227830_DEL', - 'suffix_220764_DEL_2': 'suffix_227830_DEL', - 'prefix_44455_DEL': 'suffix_50096_DEL', - 'prefix_135473_DEL': 'suffix_152259_DEL', - 'prefix_83881_DUP': 'suffix_93771_DUP', - 'prefix_185180_DEL': 'suffix_190807_DEL', - 'prefix_230340_DEL': 'suffix_217490_DEL', - 'prefix_98781_DUP': 'suffix_118080_DUP', - 'suffix_333515_DEL_2': 'suffix_344457_DEL', - 'prefix_50567_DEL': 'suffix_344457_DEL', - 'suffix_243997_DEL_2': 'suffix_251931_DEL', - 'prefix_185688_DEL': 'suffix_251931_DEL', - 'prefix_185697_DEL': 'suffix_252985_DEL', - 'suffix_245019_DEL_2': 'suffix_252985_DEL', - 'prefix_185700_DEL': 'suffix_253165_DEL', - 'suffix_245194_DEL_2': 'suffix_253165_DEL', - 'prefix_164600_DUP': 'suffix_33272_DUP', - 'suffix_32287_DUP_2': 'suffix_33272_DUP', - 'prefix_239826_DUP': 'suffix_339672_DUP', - 'suffix_328837_DUP_2': 'suffix_339672_DUP', - 'prefix_239847_DUP': 'suffix_341812_DUP', - 'suffix_330933_DUP_2': 'suffix_341812_DUP', - 'suffix_217083_DUP_2': 'suffix_223945_DUP', - 'prefix_78332_DUP': 'suffix_223945_DUP', - 'prefix_81687_DUP': 'suffix_132894_DUP', - 'suffix_129007_DUP_2': 'suffix_132894_DUP', - 'suffix_150605_DEL_2': 'suffix_155170_DEL', - 'prefix_47681_DEL': 'suffix_155170_DEL', - 'suffix_119752_DUP_2': 'suffix_123374_DUP', - 'prefix_75741_DUP': 'suffix_123374_DUP', - 'prefix_17370_DUP': 'suffix_19297_DUP', - 'prefix_54006_DUP': 'suffix_19297_DUP', - 'prefix_137115_DUP': 'suffix_67761_DUP', - 'suffix_65769_DUP_2': 'suffix_67761_DUP', - 'prefix_127018_DUP': 'suffix_183995_DUP', - 'prefix_179697_DUP': 'suffix_183995_DUP', - 'prefix_65335_DEL': 'suffix_334692_DEL', - 'prefix_26523_DEL': 'suffix_92931_DEL', - 'suffix_90209_DEL_2': 'suffix_92931_DEL', - 'prefix_121355_DEL': 'suffix_310958_DEL', - 'prefix_3970_DEL': 'suffix_311681_DEL', - 'prefix_16643_DEL': 'suffix_337172_DEL', - 'suffix_326410_DEL_2': 'suffix_337172_DEL', - 'prefix_98251_DEL': 'suffix_179586_DEL', - 'suffix_174257_DEL_2': 'suffix_179586_DEL', - 'prefix_302982_DUP': 'suffix_344928_DUP', - 'prefix_78630_DUP': 'suffix_344928_DUP', - 'prefix_248115_DEL': 'suffix_344177_DEL', - 'prefix_60051_DUP': 'suffix_338928_DUP', - 'suffix_328124_DUP_2': 'suffix_338928_DUP', - 'prefix_202120_DUP': 'suffix_10024_DUP', - 'prefix_137202_DEL': 'suffix_105069_DEL', - 'suffix_101981_DEL_2': 'suffix_105069_DEL', - 'suffix_312453_DEL_2': 'suffix_322732_DEL', - 'prefix_101760_DEL': 'suffix_322732_DEL', - 'prefix_164827_DUP': 'suffix_67293_DUP', - 'prefix_164831_DEL': 'suffix_223792_DEL', - 'suffix_216934_DEL_2': 'suffix_223792_DEL', - 'prefix_47704_DEL': 'suffix_24051_DEL', - 'prefix_108297_DEL': 'suffix_155176_DEL', - 'prefix_284632_DEL': 'suffix_155176_DEL', - 'prefix_37083_DUP': 'suffix_246174_DUP', - 'prefix_37090_DUP': 'suffix_246928_DUP', - 'prefix_37105_DUP': 'suffix_247737_DUP', - 'prefix_164627_DEL': 'suffix_308855_DEL', - 'prefix_164630_DUP': 'suffix_309488_DUP', - 'prefix_247841_DUP': 'suffix_310746_DUP', - 'prefix_164644_DUP': 'suffix_311682_DUP', - 'suffix_301772_DUP_2': 'suffix_311682_DUP', - 'prefix_137780_DEL': 'suffix_223761_DEL', - 'prefix_12281_DEL': 'suffix_30168_DEL', - 'suffix_29274_DEL_2': 'suffix_30168_DEL', - 'prefix_202150_DEL': 'suffix_5892_DEL', - 'prefix_249235_DUP': 'suffix_178512_DUP', - 'suffix_173214_DUP_2': 'suffix_178512_DUP', - 'prefix_286336_DUP': 'suffix_137695_DUP', - 'suffix_133674_DUP_2': 'suffix_137695_DUP', - 'prefix_202153_DUP': 'suffix_217142_DUP', - 'suffix_210547_DUP_2': 'suffix_217142_DUP', - 'suffix_101293_DUP_2': 'suffix_104364_DUP', - 'prefix_240446_DUP': 'suffix_104364_DUP', - 'suffix_326211_DEL_2': 'suffix_336972_DEL', - 'suffix_214414_DUP_2': 'suffix_221162_DUP', - 'suffix_264403_DUP_2': 'suffix_273062_DUP', - 'suffix_92903_DEL_2': 'suffix_95683_DEL', - 'suffix_326915_DEL_2': 'suffix_337695_DEL', - 'suffix_432_DEL_2': 'suffix_442_DEL', - 'suffix_96995_DEL_2': 'suffix_99914_DEL', - 'suffix_16343_DEL_2': 'suffix_16857_DEL', - 'suffix_239134_DUP_2': 'suffix_246928_DUP', - 'suffix_239928_DUP_2': 'suffix_247737_DUP', - 'suffix_299627_DUP_2': 'suffix_309488_DUP', - 'suffix_300851_DUP_2': 'suffix_310746_DUP', - 'prefix_100003_DEL': 'suffix_183787_DEL', - 'prefix_101443_DEL': 'suffix_308476_DEL', - 'prefix_10035_DEL': 'suffix_191074_DEL', - 'prefix_102085_DUP': 'suffix_258879_DUP', - 'prefix_102166_DUP': 'suffix_308496_DUP', - 'prefix_102435_DEL': 'suffix_4265_DEL', - 'prefix_102446_DEL': 'suffix_8242_DEL', - 'prefix_102604_DEL': 'suffix_114190_DEL', - 'prefix_10313_DEL': 'suffix_282612_DEL', - 'prefix_10397_DEL': 'suffix_299122_DEL', - 'prefix_104124_DEL': 'suffix_115920_DEL', - 'prefix_104792_DEL': 'suffix_116623_DEL', - 'prefix_105204_DEL': 'suffix_321184_DEL', - 'prefix_10520_DEL': 'suffix_12114_DEL', - 'prefix_105408_DEL': 'suffix_124299_DEL', - 'prefix_105567_DEL': 'suffix_137041_DEL', - 'prefix_105575_DEL': 'suffix_132895_DEL', - 'prefix_105957_DEL': 'suffix_117939_DEL', - 'prefix_106328_DUP': 'suffix_118372_DUP', - 'prefix_10674_DEL': 'suffix_62051_DEL', - 'prefix_106904_DUP': 'suffix_119175_DUP', - 'prefix_10706_DUP': 'suffix_12310_DUP', - 'prefix_107471_DEL': 'suffix_119959_DEL', - 'prefix_107474_DEL': 'suffix_107226_DEL', - 'prefix_107640_DEL': 'suffix_42123_DEL', - 'prefix_107682_DEL': 'suffix_328436_DEL', - 'prefix_107713_DEL': 'suffix_295713_DEL', - 'prefix_107739_DEL': 'suffix_120359_DEL', - 'prefix_108245_DUP': 'suffix_345422_DUP', - 'prefix_108246_DUP': 'suffix_345434_DUP', - 'prefix_108445_DEL': 'suffix_193406_DEL', - 'prefix_108449_DUP': 'suffix_306110_DUP', - 'prefix_108682_DEL': 'suffix_121584_DEL', - 'prefix_108787_DUP': 'suffix_121697_DUP', - 'prefix_109461_DUP': 'suffix_69248_DUP', - 'prefix_109954_DEL': 'suffix_136917_DEL', - 'prefix_11087_DEL': 'suffix_12711_DEL', - 'prefix_111170_DUP': 'suffix_124298_DUP', - 'prefix_111287_DUP': 'suffix_124448_DUP', - 'prefix_111457_DEL': 'suffix_325622_DEL', - 'prefix_112147_DEL': 'suffix_176498_DEL', - 'prefix_112170_DEL': 'suffix_189247_DEL', - 'prefix_112473_DEL': 'suffix_189253_DEL', - 'prefix_112579_DUP': 'suffix_218784_DUP', - 'prefix_112737_DEL': 'suffix_322979_DEL', - 'prefix_113007_DEL': 'suffix_189763_DEL', - 'prefix_113045_DEL': 'suffix_182363_DEL', - 'prefix_113061_DUP': 'suffix_258764_DUP', - 'prefix_113221_DUP': 'suffix_345607_DUP', - 'prefix_113248_DEL': 'suffix_220751_DEL', - 'prefix_113388_DEL': 'suffix_49218_DEL', - 'prefix_114137_DUP': 'suffix_181479_DUP', - 'prefix_114155_DEL': 'suffix_203405_DEL', - 'prefix_114307_DUP': 'suffix_127847_DUP', - 'prefix_114524_DEL': 'suffix_236802_DEL', - 'prefix_115161_DEL': 'suffix_337104_DEL', - 'prefix_115528_DEL': 'suffix_345416_DEL', - 'prefix_115594_DEL': 'suffix_185881_DEL', - 'prefix_1164_DEL': 'suffix_1403_DEL', - 'prefix_116701_DEL': 'suffix_130655_DEL', - 'prefix_11784_DEL': 'suffix_193515_DEL', - 'prefix_11793_DEL': 'suffix_261170_DEL', - 'prefix_118448_DUP': 'suffix_132890_DUP', - 'prefix_118567_DUP': 'suffix_257299_DUP', - 'prefix_118926_DEL': 'suffix_272807_DEL', - 'prefix_119331_DEL': 'suffix_133934_DEL', - 'prefix_119619_DUP': 'suffix_251043_DUP', - 'prefix_12014_DEL': 'suffix_171415_DEL', - 'prefix_120661_DUP': 'suffix_253757_DUP', - 'prefix_120683_DUP': 'suffix_299274_DUP', - 'prefix_121292_DUP': 'suffix_136324_DUP', - 'prefix_122017_DUP': 'suffix_169951_DUP', - 'prefix_122819_DEL': 'suffix_137963_DEL', - 'prefix_122974_DUP': 'suffix_138130_DUP', - 'prefix_123007_DUP': 'suffix_138159_DUP', - 'prefix_123041_DEL': 'suffix_335303_DEL', - 'prefix_123977_DEL': 'suffix_139305_DEL', - 'prefix_12470_DEL': 'suffix_226823_DEL', - 'prefix_124928_DEL': 'suffix_140340_DEL', - 'prefix_12605_DEL': 'suffix_241864_DEL', - 'prefix_127020_DUP': 'suffix_202584_DUP', - 'prefix_127505_DUP': 'suffix_252433_DUP', - 'prefix_12830_DEL': 'suffix_282628_DEL', - 'prefix_13020_DEL': 'suffix_14730_DEL', - 'prefix_13052_DEL': 'suffix_311669_DEL', - 'prefix_131669_DEL': 'suffix_33717_DEL', - 'prefix_131746_DEL': 'suffix_147895_DEL', - 'prefix_131753_DEL': 'suffix_111130_DEL', - 'prefix_13269_DUP': 'suffix_14995_DUP', - 'prefix_133114_DUP': 'suffix_217164_DUP', - 'prefix_133204_DUP': 'suffix_49743_DUP', - 'prefix_133405_DEL': 'suffix_342958_DEL', - 'prefix_13359_DEL': 'suffix_15085_DEL', - 'prefix_13369_DEL': 'suffix_52450_DEL', - 'prefix_133839_DUP': 'suffix_82443_DUP', - 'prefix_134071_DEL': 'suffix_307896_DEL', - 'prefix_134074_DEL': 'suffix_308281_DEL', - 'prefix_134075_DEL': 'suffix_308321_DEL', - 'prefix_134411_DEL': 'suffix_196304_DEL', - 'prefix_134571_DEL': 'suffix_150896_DEL', - 'prefix_134858_DEL': 'suffix_151606_DEL', - 'prefix_135555_DEL': 'suffix_152343_DEL', - 'prefix_136351_DUP': 'suffix_153160_DUP', - 'prefix_137771_DUP': 'suffix_290375_DUP', - 'prefix_138469_DUP': 'suffix_211854_DUP', - 'prefix_138595_DUP': 'suffix_97099_DUP', - 'prefix_138886_DUP': 'suffix_118224_DUP', - 'prefix_139433_DEL': 'suffix_21321_DEL', - 'prefix_139627_DEL': 'suffix_228579_DEL', - 'prefix_141117_DEL': 'suffix_75812_DEL', - 'prefix_141394_DEL': 'suffix_212333_DEL', - 'prefix_141485_DEL': 'suffix_158734_DEL', - 'prefix_141766_DUP': 'suffix_250308_DUP', - 'prefix_142256_DEL': 'suffix_151736_DEL', - 'prefix_142808_DEL': 'suffix_50260_DEL', - 'prefix_143266_DEL': 'suffix_283626_DEL', - 'prefix_143618_DEL': 'suffix_336978_DEL', - 'prefix_144319_DEL': 'suffix_12781_DEL', - 'prefix_144571_DEL': 'suffix_225727_DEL', - 'prefix_144878_DEL': 'suffix_162955_DEL', - 'prefix_144935_DEL': 'suffix_88144_DEL', - 'prefix_14542_DEL': 'suffix_29900_DEL', - 'prefix_145614_DEL': 'suffix_64370_DEL', - 'prefix_145631_DUP': 'suffix_213097_DUP', - 'prefix_145739_DUP': 'suffix_29512_DUP', - 'prefix_145751_DUP': 'suffix_142338_DUP', - 'prefix_145859_DEL': 'suffix_236767_DEL', - 'prefix_146138_DEL': 'suffix_341905_DEL', - 'prefix_146698_DUP': 'suffix_75558_DUP', - 'prefix_146725_DEL': 'suffix_92679_DEL', - 'prefix_146819_DEL': 'suffix_204036_DEL', - 'prefix_146982_DUP': 'suffix_80598_DUP', - 'prefix_147832_DUP': 'suffix_36481_DUP', - 'prefix_147905_DEL': 'suffix_116249_DEL', - 'prefix_14586_DEL': 'suffix_189167_DEL', - 'prefix_150663_DUP': 'suffix_169723_DUP', - 'prefix_150988_DEL': 'suffix_170148_DEL', - 'prefix_151487_DEL': 'suffix_87868_DEL', - 'prefix_151896_DUP': 'suffix_298612_DUP', - 'prefix_151897_DUP': 'suffix_67741_DUP', - 'prefix_151958_DEL': 'suffix_72228_DEL', - 'prefix_152188_DEL': 'suffix_258374_DEL', - 'prefix_152243_DEL': 'suffix_171706_DEL', - 'prefix_152598_DEL': 'suffix_287152_DEL', - 'prefix_153441_DUP': 'suffix_181703_DUP', - 'prefix_15373_DEL': 'suffix_17189_DEL', - 'prefix_155426_DEL': 'suffix_175579_DEL', - 'prefix_156182_DEL': 'suffix_12082_DEL', - 'prefix_157522_DEL': 'suffix_222475_DEL', - 'prefix_157813_DUP': 'suffix_178342_DUP', - 'prefix_158523_DUP': 'suffix_313465_DUP', - 'prefix_158661_DEL': 'suffix_88654_DEL', - 'prefix_160612_DEL': 'suffix_248116_DEL', - 'prefix_163949_DUP': 'suffix_344732_DUP', - 'prefix_164522_DEL': 'suffix_185883_DEL', - 'prefix_164622_DEL': 'suffix_186016_DEL', - 'prefix_164629_DUP': 'suffix_309441_DUP', - 'prefix_164642_DUP': 'suffix_311548_DUP', - 'prefix_164777_DEL': 'suffix_186196_DEL', - 'prefix_165076_DEL': 'suffix_328506_DEL', - 'prefix_165078_DEL': 'suffix_328786_DEL', - 'prefix_1657_DEL': 'suffix_2017_DEL', - 'prefix_167202_DEL': 'suffix_336972_DEL', - 'prefix_168614_DUP': 'suffix_190522_DUP', - 'prefix_168800_DUP': 'suffix_122588_DUP', - 'prefix_169366_DEL': 'suffix_191327_DEL', - 'prefix_170220_DUP': 'suffix_122478_DUP', - 'prefix_170275_DEL': 'suffix_329992_DEL', - 'prefix_170760_DEL': 'suffix_192854_DEL', - 'prefix_171486_DEL': 'suffix_193608_DEL', - 'prefix_17150_DEL': 'suffix_98009_DEL', - 'prefix_17230_DEL': 'suffix_336965_DEL', - 'prefix_17231_DEL': 'suffix_337022_DEL', - 'prefix_172982_DUP': 'suffix_195278_DUP', - 'prefix_17317_DEL': 'suffix_275059_DEL', - 'prefix_173203_DEL': 'suffix_195570_DEL', - 'prefix_173318_DEL': 'suffix_173502_DEL', - 'prefix_173699_DUP': 'suffix_127421_DUP', - 'prefix_173819_DEL': 'suffix_344967_DEL', - 'prefix_174099_DEL': 'suffix_151801_DEL', - 'prefix_174146_DEL': 'suffix_193980_DEL', - 'prefix_174221_DUP': 'suffix_125584_DUP', - 'prefix_174279_DEL': 'suffix_157330_DEL', - 'prefix_174372_DEL': 'suffix_262729_DEL', - 'prefix_174562_DEL': 'suffix_174111_DEL', - 'prefix_174564_DEL': 'suffix_1324_DEL', - 'prefix_175140_DEL': 'suffix_272786_DEL', - 'prefix_175243_DUP': 'suffix_322125_DUP', - 'prefix_175281_DUP': 'suffix_323384_DUP', - 'prefix_175291_DUP': 'suffix_323971_DUP', - 'prefix_176490_DUP': 'suffix_337504_DUP', - 'prefix_176785_DEL': 'suffix_256937_DEL', - 'prefix_177587_DUP': 'suffix_56526_DUP', - 'prefix_177834_DUP': 'suffix_276088_DUP', - 'prefix_178234_DEL': 'suffix_170832_DEL', - 'prefix_178374_DUP': 'suffix_173115_DUP', - 'prefix_178921_DEL': 'suffix_201891_DEL', - 'prefix_179719_DEL': 'suffix_326378_DEL', - 'prefix_179787_DEL': 'suffix_337951_DEL', - 'prefix_179804_DEL': 'suffix_289797_DEL', - 'prefix_180023_DEL': 'suffix_298661_DEL', - 'prefix_180149_DUP': 'suffix_203322_DUP', - 'prefix_180338_DEL': 'suffix_45778_DEL', - 'prefix_180351_DEL': 'suffix_107299_DEL', - 'prefix_180464_DEL': 'suffix_342917_DEL', - 'prefix_180500_DEL': 'suffix_183250_DEL', - 'prefix_180680_DUP': 'suffix_82979_DUP', - 'prefix_180721_DEL': 'suffix_4948_DEL', - 'prefix_181685_DEL': 'suffix_42286_DEL', - 'prefix_181724_DUP': 'suffix_343954_DUP', - 'prefix_18172_DUP': 'suffix_94217_DUP', - 'prefix_182085_DUP': 'suffix_248675_DUP', - 'prefix_182185_DUP': 'suffix_185453_DUP', - 'prefix_182381_DEL': 'suffix_65928_DEL', - 'prefix_182507_DEL': 'suffix_162956_DEL', - 'prefix_182_DEL': 'suffix_106425_DEL', - 'prefix_183723_DUP': 'suffix_207297_DUP', - 'prefix_184282_DUP': 'suffix_301766_DUP', - 'prefix_184370_DEL': 'suffix_156979_DEL', - 'prefix_184996_DUP': 'suffix_328901_DUP', - 'prefix_185023_DEL': 'suffix_15090_DEL', - 'prefix_185054_DEL': 'suffix_280761_DEL', - 'prefix_185055_DEL': 'suffix_132024_DEL', - 'prefix_185138_DEL': 'suffix_127756_DEL', - 'prefix_185141_DEL': 'suffix_134624_DEL', - 'prefix_185531_DEL': 'suffix_132027_DEL', - 'prefix_185615_DEL': 'suffix_217486_DEL', - 'prefix_185627_DEL': 'suffix_224393_DEL', - 'prefix_185677_DEL': 'suffix_173146_DEL', - 'prefix_185750_DUP': 'suffix_102606_DUP', - 'prefix_185860_DEL': 'suffix_241831_DEL', - 'prefix_185877_DEL': 'suffix_302969_DEL', - 'prefix_185935_DEL': 'suffix_173921_DEL', - 'prefix_186018_DEL': 'suffix_217498_DEL', - 'prefix_186031_DEL': 'suffix_44013_DEL', - 'prefix_156507_DEL': 'suffix_176830_DEL', - 'prefix_186032_DUP': 'suffix_2023_DUP', - 'prefix_186063_DEL': 'suffix_209934_DEL', - 'prefix_186624_DUP': 'suffix_210575_DUP', - 'prefix_186854_DUP': 'suffix_210807_DUP', - 'prefix_187881_DEL': 'suffix_92683_DEL', - 'prefix_190360_DEL': 'suffix_214876_DEL', - 'prefix_191004_DUP': 'suffix_215642_DUP', - 'prefix_191257_DEL': 'suffix_228609_DEL', - 'prefix_191292_DUP': 'suffix_234879_DUP', - 'prefix_191354_DUP': 'suffix_187230_DUP', - 'prefix_191471_DEL': 'suffix_208014_DEL', - 'prefix_191474_DEL': 'suffix_280450_DEL', - 'prefix_191494_DUP': 'suffix_202343_DUP', - 'prefix_191709_DEL': 'suffix_171576_DEL', - 'prefix_191829_DUP': 'suffix_217491_DUP', - 'prefix_191939_DEL': 'suffix_62014_DEL', - 'prefix_192002_DUP': 'suffix_271568_DUP', - 'prefix_192090_DEL': 'suffix_132549_DEL', - 'prefix_192468_DUP': 'suffix_217487_DUP', - 'prefix_192692_DEL': 'suffix_217799_DEL', - 'prefix_192750_DEL': 'suffix_217878_DEL', - 'prefix_192794_DEL': 'suffix_103053_DEL', - 'prefix_192809_DEL': 'suffix_242803_DEL', - 'prefix_192811_DUP': 'suffix_288571_DUP', - 'prefix_192812_DUP': 'suffix_288581_DUP', - 'prefix_192881_DEL': 'suffix_203645_DEL', - 'prefix_193000_DEL': 'suffix_327418_DEL', - 'prefix_193117_DEL': 'suffix_105935_DEL', - 'prefix_193570_DEL': 'suffix_218935_DEL', - 'prefix_193750_DEL': 'suffix_171289_DEL', - 'prefix_193822_DEL': 'suffix_219267_DEL', - 'prefix_193980_DEL': 'suffix_173513_DEL', - 'prefix_193998_DEL': 'suffix_342920_DEL', - 'prefix_193_DEL': 'suffix_175579_DEL', - 'prefix_195154_DUP': 'suffix_262756_DUP', - 'prefix_195332_DUP': 'suffix_301004_DUP', - 'prefix_195356_DEL': 'suffix_67397_DEL', - 'prefix_195394_DUP': 'suffix_32943_DUP', - 'prefix_195409_DEL': 'suffix_345343_DEL', - 'prefix_195446_DUP': 'suffix_218272_DUP', - 'prefix_195686_DEL': 'suffix_36573_DEL', - 'prefix_195893_DEL': 'suffix_216802_DEL', - 'prefix_196343_DEL': 'suffix_187682_DEL', - 'prefix_196653_DEL': 'suffix_287837_DEL', - 'prefix_197018_DEL': 'suffix_176475_DEL', - 'prefix_197343_DEL': 'suffix_329991_DEL', - 'prefix_197345_DUP': 'suffix_217953_DUP', - 'prefix_197514_DEL': 'suffix_224017_DEL', - 'prefix_197601_DUP': 'suffix_92304_DUP', - 'prefix_197689_DEL': 'suffix_4035_DEL', - 'prefix_19776_DUP': 'suffix_22049_DUP', - 'prefix_197839_DEL': 'suffix_37637_DEL', - 'prefix_197846_DEL': 'suffix_201717_DEL', - 'prefix_197848_DUP': 'suffix_285533_DUP', - 'prefix_197857_DEL': 'suffix_33261_DEL', - 'prefix_197858_DEL': 'suffix_33273_DEL', - 'prefix_1981_DEL': 'suffix_2420_DEL', - 'prefix_198681_DEL': 'suffix_225482_DEL', - 'prefix_198960_DEL': 'suffix_214208_DEL', - 'prefix_198962_DUP': 'suffix_298013_DUP', - 'prefix_199230_DUP': 'suffix_219340_DUP', - 'prefix_199526_DEL': 'suffix_244411_DEL', - 'prefix_199572_DEL': 'suffix_273734_DEL', - 'prefix_199578_DEL': 'suffix_132651_DEL', - 'prefix_201088_DEL': 'suffix_343155_DEL', - 'prefix_201199_DUP': 'suffix_136419_DUP', - 'prefix_201221_DEL': 'suffix_79200_DEL', - 'prefix_201288_DEL': 'suffix_228487_DEL', - 'prefix_201337_DEL': 'suffix_228541_DEL', - 'prefix_201565_DEL': 'suffix_168578_DEL', - 'prefix_202110_DEL': 'suffix_177304_DEL', - 'prefix_202154_DUP': 'suffix_217485_DUP', - 'prefix_202256_DEL': 'suffix_331004_DEL', - 'prefix_202259_DUP': 'suffix_163215_DUP', - 'prefix_203581_DEL': 'suffix_231189_DEL', - 'prefix_20401_DEL': 'suffix_22739_DEL', - 'prefix_204188_DUP': 'suffix_1547_DUP', - 'prefix_204826_DUP': 'suffix_117758_DUP', - 'prefix_20504_DUP': 'suffix_61796_DUP', - 'prefix_205313_DEL': 'suffix_233124_DEL', - 'prefix_205330_DEL': 'suffix_233146_DEL', - 'prefix_20552_DEL': 'suffix_162107_DEL', - 'prefix_20587_DEL': 'suffix_177607_DEL', - 'prefix_206642_DUP': 'suffix_191476_DUP', - 'prefix_20842_DUP': 'suffix_247208_DUP', - 'prefix_208997_DUP': 'suffix_68391_DUP', - 'prefix_209842_DEL': 'suffix_238144_DEL', - 'prefix_210581_DEL': 'suffix_282703_DEL', - 'prefix_210630_DEL': 'suffix_126646_DEL', - 'prefix_210904_DUP': 'suffix_239309_DUP', - 'prefix_211888_DEL': 'suffix_337006_DEL', - 'prefix_212443_DEL': 'suffix_19298_DEL', - 'prefix_215700_DUP': 'suffix_244607_DUP', - 'prefix_216096_DEL': 'suffix_271336_DEL', - 'prefix_21663_DUP': 'suffix_30036_DUP', - 'prefix_21679_DUP': 'suffix_343012_DUP', - 'prefix_21680_DEL': 'suffix_344961_DEL', - 'prefix_218702_DUP': 'suffix_248113_DUP', - 'prefix_219145_DEL': 'suffix_198229_DEL', - 'prefix_221051_DEL': 'suffix_32473_DEL', - 'prefix_22118_DEL': 'suffix_291073_DEL', - 'prefix_222234_DEL': 'suffix_176476_DEL', - 'prefix_22347_DUP': 'suffix_336271_DUP', - 'prefix_223818_DEL': 'suffix_253681_DEL', - 'prefix_223972_DEL': 'suffix_253871_DEL', - 'prefix_224087_DEL': 'suffix_254000_DEL', - 'prefix_22411_DEL': 'suffix_329985_DEL', - 'prefix_22607_DEL': 'suffix_107271_DEL', - 'prefix_22608_DEL': 'suffix_107295_DEL', - 'prefix_22727_DEL': 'suffix_93545_DEL', - 'prefix_227942_DUP': 'suffix_258345_DUP', - 'prefix_228561_DEL': 'suffix_247820_DEL', - 'prefix_228798_DEL': 'suffix_224733_DEL', - 'prefix_229038_DUP': 'suffix_40155_DUP', - 'prefix_229254_DUP': 'suffix_345612_DUP', - 'prefix_229345_DUP': 'suffix_178578_DUP', - 'prefix_229543_DUP': 'suffix_260172_DUP', - 'prefix_229784_DEL': 'suffix_4824_DEL', - 'prefix_229791_DUP': 'suffix_188332_DUP', - 'prefix_229805_DUP': 'suffix_301762_DUP', - 'prefix_229839_DEL': 'suffix_39713_DEL', - 'prefix_229851_DEL': 'suffix_134949_DEL', - 'prefix_229879_DUP': 'suffix_12928_DUP', - 'prefix_230003_DEL': 'suffix_107348_DEL', - 'prefix_230020_DUP': 'suffix_118182_DUP', - 'prefix_230025_DUP': 'suffix_223544_DUP', - 'prefix_230090_DUP': 'suffix_332572_DUP', - 'prefix_230178_DUP': 'suffix_302641_DUP', - 'prefix_230183_DUP': 'suffix_74958_DUP', - 'prefix_230251_DEL': 'suffix_133150_DEL', - 'prefix_230271_DEL': 'suffix_161946_DEL', - 'prefix_230308_DEL': 'suffix_244610_DEL', - 'prefix_230350_DEL': 'suffix_252059_DEL', - 'prefix_230445_DEL': 'suffix_19341_DEL', - 'prefix_230454_DUP': 'suffix_313052_DUP', - 'prefix_230603_DEL': 'suffix_77668_DEL', - 'prefix_230616_DEL': 'suffix_316424_DEL', - 'prefix_230738_DEL': 'suffix_261522_DEL', - 'prefix_231752_DEL': 'suffix_262711_DEL', - 'prefix_231770_DEL': 'suffix_262731_DEL', - 'prefix_231952_DUP': 'suffix_322130_DUP', - 'prefix_232113_DUP': 'suffix_85765_DUP', - 'prefix_232394_DEL': 'suffix_308668_DEL', - 'prefix_232438_DEL': 'suffix_263531_DEL', - 'prefix_232444_DEL': 'suffix_263537_DEL', - 'prefix_232452_DUP': 'suffix_80443_DUP', - 'prefix_232540_DUP': 'suffix_134297_DUP', - 'prefix_232595_DEL': 'suffix_189246_DEL', - 'prefix_232637_DEL': 'suffix_182330_DEL', - 'prefix_232677_DEL': 'suffix_55000_DEL', - 'prefix_232776_DUP': 'suffix_262197_DUP', - 'prefix_232899_DUP': 'suffix_329033_DUP', - 'prefix_232900_DUP': 'suffix_329084_DUP', - 'prefix_233045_DEL': 'suffix_150737_DEL', - 'prefix_233119_DEL': 'suffix_264248_DEL', - 'prefix_233311_DEL': 'suffix_241927_DEL', - 'prefix_233321_DEL': 'suffix_344725_DEL', - 'prefix_233717_DUP': 'suffix_201544_DUP', - 'prefix_234863_DEL': 'suffix_185898_DEL', - 'prefix_23489_DEL': 'suffix_331839_DEL', - 'prefix_234953_DEL': 'suffix_109390_DEL', - 'prefix_234990_DEL': 'suffix_200659_DEL', - 'prefix_236431_DUP': 'suffix_267998_DUP', - 'prefix_237073_DUP': 'suffix_319276_DUP', - 'prefix_237178_DEL': 'suffix_93427_DEL', - 'prefix_237197_DEL': 'suffix_65316_DEL', - 'prefix_237515_DUP': 'suffix_269265_DUP', - 'prefix_237761_DEL': 'suffix_218136_DEL', - 'prefix_237780_DUP': 'suffix_237634_DUP', - 'prefix_237791_DEL': 'suffix_37586_DEL', - 'prefix_237797_DUP': 'suffix_279247_DUP', - 'prefix_237833_DUP': 'suffix_118074_DUP', - 'prefix_237857_DEL': 'suffix_217856_DEL', - 'prefix_237872_DUP': 'suffix_54996_DUP', - 'prefix_237874_DEL': 'suffix_172470_DEL', - 'prefix_237991_DUP': 'suffix_138522_DUP', - 'prefix_238021_DUP': 'suffix_136966_DUP', - 'prefix_238147_DUP': 'suffix_270013_DUP', - 'prefix_238938_DEL': 'suffix_270899_DEL', - 'prefix_238945_DUP': 'suffix_308856_DUP', - 'prefix_238961_DEL': 'suffix_310956_DEL', - 'prefix_239339_DEL': 'suffix_255507_DEL', - 'prefix_240008_DEL': 'suffix_272099_DEL', - 'prefix_240186_DEL': 'suffix_264944_DEL', - 'prefix_240251_DUP': 'suffix_151050_DUP', - 'prefix_240270_DEL': 'suffix_39288_DEL', - 'prefix_240483_DEL': 'suffix_39678_DEL', - 'prefix_241524_DEL': 'suffix_121573_DEL', - 'prefix_241665_DUP': 'suffix_274132_DUP', - 'prefix_241760_DEL': 'suffix_274237_DEL', - 'prefix_242143_DEL': 'suffix_178407_DEL', - 'prefix_242192_DUP': 'suffix_317053_DUP', - 'prefix_242247_DUP': 'suffix_345761_DUP', - 'prefix_242331_DEL': 'suffix_169106_DEL', - 'prefix_24255_DEL': 'suffix_212769_DEL', - 'prefix_242646_DEL': 'suffix_212305_DEL', - 'prefix_242678_DEL': 'suffix_259663_DEL', - 'prefix_242679_DEL': 'suffix_259723_DEL', - 'prefix_242681_DEL': 'suffix_259864_DEL', - 'prefix_242685_DEL': 'suffix_260161_DEL', - 'prefix_244399_DEL': 'suffix_277206_DEL', - 'prefix_244573_DUP': 'suffix_277418_DUP', - 'prefix_24479_DUP': 'suffix_119594_DUP', - 'prefix_245216_DEL': 'suffix_270267_DEL', - 'prefix_245217_DEL': 'suffix_89519_DEL', - 'prefix_245531_DEL': 'suffix_21740_DEL', - 'prefix_245561_DEL': 'suffix_56540_DEL', - 'prefix_245634_DUP': 'suffix_70271_DUP', - 'prefix_245655_DEL': 'suffix_38769_DEL', - 'prefix_245659_DUP': 'suffix_175204_DUP', - 'prefix_245721_DUP': 'suffix_199804_DUP', - 'prefix_245813_DUP': 'suffix_40862_DUP', - 'prefix_24581_DUP': 'suffix_141949_DUP', - 'prefix_24583_DEL': 'suffix_217839_DEL', - 'prefix_245903_DUP': 'suffix_127338_DUP', - 'prefix_24592_DEL': 'suffix_46695_DEL', - 'prefix_245933_DUP': 'suffix_279006_DUP', - 'prefix_24593_DEL': 'suffix_46762_DEL', - 'prefix_245959_DUP': 'suffix_121405_DUP', - 'prefix_24595_DEL': 'suffix_46839_DEL', - 'prefix_245986_DUP': 'suffix_239962_DUP', - 'prefix_246310_DEL': 'suffix_139843_DEL', - 'prefix_24637_DEL': 'suffix_185437_DEL', - 'prefix_24662_DEL': 'suffix_302153_DEL', - 'prefix_247373_DEL': 'suffix_280718_DEL', - 'prefix_247836_DUP': 'suffix_309310_DUP', - 'prefix_247847_DUP': 'suffix_311619_DUP', - 'prefix_247848_DUP': 'suffix_311683_DUP', - 'prefix_248168_DEL': 'suffix_132850_DEL', - 'prefix_24841_DUP': 'suffix_2568_DUP', - 'prefix_248446_DUP': 'suffix_90044_DUP', - 'prefix_24870_DEL': 'suffix_337649_DEL', - 'prefix_248998_DUP': 'suffix_282433_DUP', - 'prefix_24904_DEL': 'suffix_298012_DEL', - 'prefix_249320_DUP': 'suffix_227500_DUP', - 'prefix_249361_DEL': 'suffix_220755_DEL', - 'prefix_24943_DEL': 'suffix_157892_DEL', - 'prefix_249451_DUP': 'suffix_6339_DUP', - 'prefix_250214_DEL': 'suffix_293400_DEL', - 'prefix_250459_DEL': 'suffix_284644_DEL', - 'prefix_250460_DEL': 'suffix_69510_DEL', - 'prefix_250461_DEL': 'suffix_134254_DEL', - 'prefix_250483_DUP': 'suffix_277241_DUP', - 'prefix_250509_DEL': 'suffix_218771_DEL', - 'prefix_250600_DEL': 'suffix_52226_DEL', - 'prefix_250616_DEL': 'suffix_322013_DEL', - 'prefix_250674_DUP': 'suffix_304173_DUP', - 'prefix_250694_DUP': 'suffix_313136_DUP', - 'prefix_250818_DEL': 'suffix_27205_DEL', - 'prefix_250819_DEL': 'suffix_27210_DEL', - 'prefix_250889_DUP': 'suffix_111556_DUP', - 'prefix_250932_DUP': 'suffix_133045_DUP', - 'prefix_250981_DEL': 'suffix_91074_DEL', - 'prefix_251102_DEL': 'suffix_146078_DEL', - 'prefix_251291_DUP': 'suffix_284913_DUP', - 'prefix_251847_DEL': 'suffix_224352_DEL', - 'prefix_251893_DEL': 'suffix_212162_DEL', - 'prefix_252103_DEL': 'suffix_306862_DEL', - 'prefix_252152_DEL': 'suffix_252976_DEL', - 'prefix_252262_DEL': 'suffix_171030_DEL', - 'prefix_252266_DUP': 'suffix_253194_DUP', - 'prefix_252268_DEL': 'suffix_127157_DEL', - 'prefix_25237_DEL': 'suffix_344078_DEL', - 'prefix_252465_DEL': 'suffix_255499_DEL', - 'prefix_252468_DUP': 'suffix_61453_DUP', - 'prefix_252588_DEL': 'suffix_179491_DEL', - 'prefix_252589_DEL': 'suffix_158777_DEL', - 'prefix_252657_DEL': 'suffix_332312_DEL', - 'prefix_252672_DUP': 'suffix_8208_DUP', - 'prefix_252677_DEL': 'suffix_297350_DEL', - 'prefix_252719_DEL': 'suffix_118143_DEL', - 'prefix_253047_DUP': 'suffix_211078_DUP', - 'prefix_254257_DEL': 'suffix_258839_DEL', - 'prefix_254295_DEL': 'suffix_37577_DEL', - 'prefix_254321_DUP': 'suffix_102639_DUP', - 'prefix_254471_DUP': 'suffix_210560_DUP', - 'prefix_254486_DEL': 'suffix_61825_DEL', - 'prefix_254504_DEL': 'suffix_89660_DEL', - 'prefix_254514_DEL': 'suffix_100226_DEL', - 'prefix_254902_DUP': 'suffix_165508_DUP', - 'prefix_254973_DEL': 'suffix_166161_DEL', - 'prefix_255104_DEL': 'suffix_4167_DEL', - 'prefix_255420_DUP': 'suffix_72092_DUP', - 'prefix_255620_DEL': 'suffix_258860_DEL', - 'prefix_255624_DEL': 'suffix_260164_DEL', - 'prefix_256004_DEL': 'suffix_211645_DEL', - 'prefix_257014_DUP': 'suffix_170805_DUP', - 'prefix_257015_DUP': 'suffix_170872_DUP', - 'prefix_257016_DUP': 'suffix_170906_DUP', - 'prefix_257671_DUP': 'suffix_291855_DUP', - 'prefix_258103_DEL': 'suffix_292360_DEL', - 'prefix_25895_DEL': 'suffix_105875_DEL', - 'prefix_259285_DEL': 'suffix_176497_DEL', - 'prefix_260110_DEL': 'suffix_8146_DEL', - 'prefix_260118_DEL': 'suffix_106667_DEL', - 'prefix_260165_DUP': 'suffix_241615_DUP', - 'prefix_260287_DUP': 'suffix_130663_DUP', - 'prefix_260388_DUP': 'suffix_68171_DUP', - 'prefix_260444_DUP': 'suffix_295009_DUP', - 'prefix_260683_DEL': 'suffix_169109_DEL', - 'prefix_260715_DEL': 'suffix_100707_DEL', - 'prefix_260762_DEL': 'suffix_117930_DEL', - 'prefix_260790_DUP': 'suffix_172476_DUP', - 'prefix_260802_DEL': 'suffix_323217_DEL', - 'prefix_260879_DEL': 'suffix_27705_DEL', - 'prefix_260902_DEL': 'suffix_67106_DEL', - 'prefix_260908_DEL': 'suffix_129735_DEL', - 'prefix_260_DEL': 'suffix_55007_DEL', - 'prefix_261548_DEL': 'suffix_16881_DEL', - 'prefix_26254_DEL': 'suffix_29316_DEL', - 'prefix_262735_DEL': 'suffix_297605_DEL', - 'prefix_263583_DUP': 'suffix_318930_DUP', - 'prefix_263614_DEL': 'suffix_309554_DEL', - 'prefix_263688_DUP': 'suffix_287109_DUP', - 'prefix_263713_DEL': 'suffix_315234_DEL', - 'prefix_263717_DEL': 'suffix_206974_DEL', - 'prefix_263844_DEL': 'suffix_93604_DEL', - 'prefix_264229_DEL': 'suffix_299315_DEL', - 'prefix_264544_DEL': 'suffix_159878_DEL', - 'prefix_26468_DEL': 'suffix_29565_DEL', - 'prefix_264705_DUP': 'suffix_122589_DUP', - 'prefix_264818_DEL': 'suffix_165079_DEL', - 'prefix_264891_DUP': 'suffix_113524_DUP', - 'prefix_265301_DEL': 'suffix_300475_DEL', - 'prefix_265379_DEL': 'suffix_246254_DEL', - 'prefix_266533_DEL': 'suffix_301845_DEL', - 'prefix_266544_DEL': 'suffix_301853_DEL', - 'prefix_266545_DEL': 'suffix_301855_DEL', - 'prefix_267_DEL': 'suffix_122539_DEL', - 'prefix_269805_DEL': 'suffix_305404_DEL', - 'prefix_2705_DEL': 'suffix_139305_DEL', - 'prefix_270760_DEL': 'suffix_306509_DEL', - 'prefix_27114_DEL': 'suffix_150454_DEL', - 'prefix_271645_DEL': 'suffix_30695_DEL', - 'prefix_272591_DEL': 'suffix_308628_DEL', - 'prefix_272819_DEL': 'suffix_308867_DEL', - 'prefix_276988_DEL': 'suffix_174302_DEL', - 'prefix_277163_DUP': 'suffix_29468_DUP', - 'prefix_277765_DEL': 'suffix_190995_DEL', - 'prefix_277834_DEL': 'suffix_237788_DEL', - 'prefix_277849_DUP': 'suffix_187676_DUP', - 'prefix_278532_DEL': 'suffix_224674_DEL', - 'prefix_278576_DEL': 'suffix_225982_DEL', - 'prefix_27857_DEL': 'suffix_31144_DEL', - 'prefix_278620_DEL': 'suffix_61479_DEL', - 'prefix_278624_DEL': 'suffix_62018_DEL', - 'prefix_278768_DEL': 'suffix_260377_DEL', - 'prefix_280607_DUP': 'suffix_49717_DUP', - 'prefix_280679_DEL': 'suffix_88375_DEL', - 'prefix_280785_DUP': 'suffix_317588_DUP', - 'prefix_282546_DUP': 'suffix_188434_DUP', - 'prefix_282581_DUP': 'suffix_232495_DUP', - 'prefix_283354_DEL': 'suffix_183799_DEL', - 'prefix_283359_DEL': 'suffix_139093_DEL', - 'prefix_283381_DUP': 'suffix_271584_DUP', - 'prefix_283498_DUP': 'suffix_184079_DUP', - 'prefix_284364_DEL': 'suffix_16928_DEL', - 'prefix_2843_DUP': 'suffix_3431_DUP', - 'prefix_284546_DUP': 'suffix_234318_DUP', - 'prefix_284653_DEL': 'suffix_29356_DEL', - 'prefix_284718_DEL': 'suffix_233901_DEL', - 'prefix_284720_DEL': 'suffix_233993_DEL', - 'prefix_285131_DEL': 'suffix_322576_DEL', - 'prefix_285518_DUP': 'suffix_219653_DUP', - 'prefix_285910_DUP': 'suffix_131091_DUP', - 'prefix_285938_DEL': 'suffix_114389_DEL', - 'prefix_285942_DUP': 'suffix_261140_DUP', - 'prefix_285998_DUP': 'suffix_191729_DUP', - 'prefix_286350_DUP': 'suffix_151353_DUP', - 'prefix_286450_DUP': 'suffix_77791_DUP', - 'prefix_287805_DEL': 'suffix_325622_DEL', - 'prefix_295024_DUP': 'suffix_333861_DUP', - 'prefix_295802_DEL': 'suffix_334964_DEL', - 'prefix_296006_DUP': 'suffix_335221_DUP', - 'prefix_296764_DUP': 'suffix_336317_DUP', - 'prefix_297208_DEL': 'suffix_336953_DEL', - 'prefix_297213_DEL': 'suffix_336958_DEL', - 'prefix_297223_DEL': 'suffix_336975_DEL', - 'prefix_191716_DEL': 'suffix_107536_DEL', - 'prefix_297224_DEL': 'suffix_336978_DEL', - 'prefix_297279_DEL': 'suffix_337045_DEL', - 'prefix_297283_DEL': 'suffix_337053_DEL', - 'prefix_297309_DUP': 'suffix_337088_DUP', - 'prefix_29997_DEL': 'suffix_223825_DEL', - 'prefix_301818_DEL': 'suffix_343282_DEL', - 'prefix_30205_DEL': 'suffix_253777_DEL', - 'prefix_30217_DEL': 'suffix_264896_DEL', - 'prefix_302357_DEL': 'suffix_344078_DEL', - 'prefix_303466_DUP': 'suffix_345668_DUP', - 'prefix_32087_DEL': 'suffix_9607_DEL', - 'prefix_32332_DUP': 'suffix_88460_DUP', - 'prefix_32367_DEL': 'suffix_119716_DEL', - 'prefix_32706_DEL': 'suffix_36588_DEL', - 'prefix_33082_DUP': 'suffix_248523_DUP', - 'prefix_3322_DEL': 'suffix_336956_DEL', - 'prefix_33597_DUP': 'suffix_37540_DUP', - 'prefix_33614_DUP': 'suffix_37570_DUP', - 'prefix_34068_DEL': 'suffix_38077_DEL', - 'prefix_34213_DUP': 'suffix_92153_DUP', - 'prefix_34545_DEL': 'suffix_38571_DEL', - 'prefix_34597_DEL': 'suffix_109532_DEL', - 'prefix_34869_DEL': 'suffix_87992_DEL', - 'prefix_35040_DEL': 'suffix_27009_DEL', - 'prefix_35106_DEL': 'suffix_237735_DEL', - 'prefix_35180_DUP': 'suffix_190057_DUP', - 'prefix_35775_DEL': 'suffix_39951_DEL', - 'prefix_36042_DUP': 'suffix_42_DUP', - 'prefix_36108_DUP': 'suffix_339711_DUP', - 'prefix_36111_DEL': 'suffix_297767_DEL', - 'prefix_3617_DEL': 'suffix_183767_DEL', - 'prefix_36425_DEL': 'suffix_109536_DEL', - 'prefix_36528_DEL': 'suffix_338271_DEL', - 'prefix_36589_DUP': 'suffix_225864_DUP', - 'prefix_36614_DEL': 'suffix_36479_DEL', - 'prefix_36746_DUP': 'suffix_72227_DUP', - 'prefix_36846_DUP': 'suffix_107654_DUP', - 'prefix_37079_DUP': 'suffix_245784_DUP', - 'prefix_37080_DUP': 'suffix_245791_DUP', - 'prefix_37081_DUP': 'suffix_245798_DUP', - 'prefix_37082_DUP': 'suffix_246118_DUP', - 'prefix_37094_DUP': 'suffix_247108_DUP', - 'prefix_37095_DUP': 'suffix_247270_DUP', - 'prefix_37096_DUP': 'suffix_247352_DUP', - 'prefix_37098_DUP': 'suffix_247400_DUP', - 'prefix_37100_DUP': 'suffix_247574_DUP', - 'prefix_37108_DUP': 'suffix_248111_DUP', - 'prefix_37112_DUP': 'suffix_248141_DUP', - 'prefix_37282_DEL': 'suffix_32945_DEL', - 'prefix_37834_DEL': 'suffix_42257_DEL', - 'prefix_3816_DEL': 'suffix_265546_DEL', - 'prefix_3852_DEL': 'suffix_282714_DEL', - 'prefix_39151_DEL': 'suffix_336958_DEL', - 'prefix_39601_DEL': 'suffix_337053_DEL', - 'prefix_39656_DEL': 'suffix_221906_DEL', - 'prefix_3991_DEL': 'suffix_317434_DEL', - 'prefix_40087_DUP': 'suffix_198027_DUP', - 'prefix_41276_DUP': 'suffix_46123_DUP', - 'prefix_41357_DEL': 'suffix_121015_DEL', - 'prefix_42493_DUP': 'suffix_118615_DUP', - 'prefix_42547_DUP': 'suffix_47670_DUP', - 'prefix_42549_DEL': 'suffix_47672_DEL', - 'prefix_43054_DEL': 'suffix_253999_DEL', - 'prefix_43130_DEL': 'suffix_48422_DEL', - 'prefix_43183_DEL': 'suffix_48494_DEL', - 'prefix_43482_DEL': 'suffix_29927_DEL', - 'prefix_43657_DEL': 'suffix_49127_DEL', - 'prefix_44112_DEL': 'suffix_12769_DEL', - 'prefix_44349_DEL': 'suffix_234536_DEL', - 'prefix_44363_DEL': 'suffix_235902_DEL', - 'prefix_45800_DEL': 'suffix_182593_DEL', - 'prefix_45862_DEL': 'suffix_51654_DEL', - 'prefix_45965_DUP': 'suffix_140827_DUP', - 'prefix_46144_DEL': 'suffix_299340_DEL', - 'prefix_46154_DUP': 'suffix_308320_DUP', - 'prefix_46234_DEL': 'suffix_329984_DEL', - 'prefix_46275_DEL': 'suffix_292777_DEL', - 'prefix_46406_DUP': 'suffix_52251_DUP', - 'prefix_46447_DEL': 'suffix_22508_DEL', - 'prefix_4670_DEL': 'suffix_52153_DEL', - 'prefix_46723_DEL': 'suffix_297314_DEL', - 'prefix_4687_DEL': 'suffix_337096_DEL', - 'prefix_46883_DEL': 'suffix_322347_DEL', - 'prefix_46902_DEL': 'suffix_16522_DEL', - 'prefix_46949_DEL': 'suffix_221643_DEL', - 'prefix_47442_DEL': 'suffix_32709_DEL', - 'prefix_47443_DEL': 'suffix_32942_DEL', - 'prefix_47446_DEL': 'suffix_33262_DEL', - 'prefix_47486_DUP': 'suffix_341592_DUP', - 'prefix_47629_DUP': 'suffix_309398_DUP', - 'prefix_47726_DEL': 'suffix_290347_DEL', - 'prefix_47796_DEL': 'suffix_1449_DEL', - 'prefix_477_DUP': 'suffix_59131_DUP', - 'prefix_4783_DUP': 'suffix_335171_DUP', - 'prefix_47920_DUP': 'suffix_69500_DUP', - 'prefix_48061_DUP': 'suffix_297897_DUP', - 'prefix_48220_DEL': 'suffix_234836_DEL', - 'prefix_48353_DUP': 'suffix_84392_DUP', - 'prefix_49235_DEL': 'suffix_39714_DEL', - 'prefix_4948_DEL': 'suffix_5834_DEL', - 'prefix_49781_DEL': 'suffix_56043_DEL', - 'prefix_49980_DUP': 'suffix_56297_DUP', - 'prefix_50167_DEL': 'suffix_56527_DEL', - 'prefix_51403_DEL': 'suffix_13373_DEL', - 'prefix_51897_DEL': 'suffix_336964_DEL', - 'prefix_52447_DEL': 'suffix_179521_DEL', - 'prefix_53019_DEL': 'suffix_249573_DEL', - 'prefix_53360_DEL': 'suffix_28019_DEL', - 'prefix_53373_DEL': 'suffix_169978_DEL', - 'prefix_54115_DEL': 'suffix_124254_DEL', - 'prefix_5415_DEL': 'suffix_274987_DEL', - 'prefix_55334_DEL': 'suffix_182678_DEL', - 'prefix_55463_DUP': 'suffix_62593_DUP', - 'prefix_5580_DEL': 'suffix_106522_DEL', - 'prefix_56296_DUP': 'suffix_20205_DUP', - 'prefix_56375_DEL': 'suffix_122348_DEL', - 'prefix_56758_DEL': 'suffix_133934_DEL', - 'prefix_5719_DUP': 'suffix_46123_DUP', - 'prefix_57494_DEL': 'suffix_64715_DEL', - 'prefix_5756_DEL': 'suffix_195683_DEL', - 'prefix_58444_DUP': 'suffix_65759_DUP', - 'prefix_59270_DEL': 'suffix_88382_DEL', - 'prefix_598_DEL': 'suffix_729_DEL', - 'prefix_59944_DEL': 'suffix_105066_DEL', - 'prefix_60675_DEL': 'suffix_215556_DEL', - 'prefix_61714_DUP': 'suffix_69571_DUP', - 'prefix_62048_DEL': 'suffix_220763_DEL', - 'prefix_63074_DEL': 'suffix_262731_DEL', - 'prefix_63258_DEL': 'suffix_200662_DEL', - 'prefix_6325_DEL': 'suffix_282673_DEL', - 'prefix_63634_DEL': 'suffix_192720_DEL', - 'prefix_6643_DEL': 'suffix_131071_DEL', - 'prefix_67724_DEL': 'suffix_75807_DEL', - 'prefix_67761_DEL': 'suffix_300474_DEL', - 'prefix_68258_DEL': 'suffix_183179_DEL', - 'prefix_68433_DEL': 'suffix_241887_DEL', - 'prefix_69473_DEL': 'suffix_213975_DEL', - 'prefix_70643_DEL': 'suffix_173247_DEL', - 'prefix_70766_DEL': 'suffix_255510_DEL', - 'prefix_70923_DUP': 'suffix_64371_DUP', - 'prefix_71010_DEL': 'suffix_298994_DEL', - 'prefix_71023_DEL': 'suffix_102982_DEL', - 'prefix_71045_DEL': 'suffix_262680_DEL', - 'prefix_71163_DEL': 'suffix_79602_DEL', - 'prefix_7128_DEL': 'suffix_340564_DEL', - 'prefix_71349_DUP': 'suffix_310205_DUP', - 'prefix_71400_DUP': 'suffix_88019_DUP', - 'prefix_71606_DUP': 'suffix_220541_DUP', - 'prefix_71676_DUP': 'suffix_327952_DUP', - 'prefix_71691_DEL': 'suffix_131549_DEL', - 'prefix_718_DEL': 'suffix_263531_DEL', - 'prefix_71936_DUP': 'suffix_116525_DUP', - 'prefix_71961_DEL': 'suffix_140125_DEL', - 'prefix_71969_DUP': 'suffix_157089_DUP', - 'prefix_72062_DUP': 'suffix_253345_DUP', - 'prefix_72143_DEL': 'suffix_172606_DEL', - 'prefix_72208_DEL': 'suffix_197482_DEL', - 'prefix_72369_DEL': 'suffix_115813_DEL', - 'prefix_73088_DEL': 'suffix_81641_DEL', - 'prefix_73094_DUP': 'suffix_185884_DUP', - 'prefix_73451_DEL': 'suffix_307441_DEL', - 'prefix_7358_DEL': 'suffix_337112_DEL', - 'prefix_74031_DUP': 'suffix_134917_DUP', - 'prefix_74343_DUP': 'suffix_83006_DUP', - 'prefix_74628_DEL': 'suffix_208625_DEL', - 'prefix_74629_DUP': 'suffix_5338_DUP', - 'prefix_74896_DEL': 'suffix_226397_DEL', - 'prefix_75369_DEL': 'suffix_136965_DEL', - 'prefix_75422_DEL': 'suffix_84245_DEL', - 'prefix_75547_DEL': 'suffix_126315_DEL', - 'prefix_75551_DUP': 'suffix_155780_DUP', - 'prefix_75625_DUP': 'suffix_310853_DUP', - 'prefix_7613_DEL': 'suffix_247991_DEL', - 'prefix_7837_DUP': 'suffix_9141_DUP', - 'prefix_78409_DUP': 'suffix_159950_DUP', - 'prefix_78419_DEL': 'suffix_87809_DEL', - 'prefix_78493_DUP': 'suffix_87885_DUP', - 'prefix_78575_DEL': 'suffix_310958_DEL', - 'prefix_78580_DEL': 'suffix_312313_DEL', - 'prefix_78692_DEL': 'suffix_332978_DEL', - 'prefix_78775_DUP': 'suffix_178999_DUP', - 'prefix_78808_DUP': 'suffix_88248_DUP', - 'prefix_78890_DEL': 'suffix_203497_DEL', - 'prefix_78923_DUP': 'suffix_305735_DUP', - 'prefix_78942_DUP': 'suffix_307917_DUP', - 'prefix_78949_DEL': 'suffix_308466_DEL', - 'prefix_78964_DUP': 'suffix_244587_DUP', - 'prefix_79085_DUP': 'suffix_319372_DUP', - 'prefix_79632_DUP': 'suffix_260100_DUP', - 'prefix_80034_DUP': 'suffix_298803_DUP', - 'prefix_8058_DUP': 'suffix_177412_DUP', - 'prefix_80933_DEL': 'suffix_90592_DEL', - 'prefix_8121_DUP': 'suffix_9447_DUP', - 'prefix_816_DUP': 'suffix_986_DUP', - 'prefix_81889_DEL': 'suffix_91604_DEL', - 'prefix_8206_DUP': 'suffix_170741_DUP', - 'prefix_7357_DEL': 'suffix_337078_DEL', - 'prefix_82314_DUP': 'suffix_89153_DUP', - 'prefix_82745_DEL': 'suffix_95957_DEL', - 'prefix_83209_DUP': 'suffix_256121_DUP', - 'prefix_83230_DUP': 'suffix_57589_DUP', - 'prefix_83245_DUP': 'suffix_49612_DUP', - 'prefix_83313_DEL': 'suffix_31491_DEL', - 'prefix_83315_DEL': 'suffix_93129_DEL', - 'prefix_83649_DEL': 'suffix_105765_DEL', - 'prefix_83650_DEL': 'suffix_105792_DEL', - 'prefix_83651_DEL': 'suffix_105804_DEL', - 'prefix_83935_DEL': 'suffix_336309_DEL', - 'prefix_83975_DUP': 'suffix_134029_DUP', - 'prefix_83980_DEL': 'suffix_242921_DEL', - 'prefix_83984_DUP': 'suffix_32384_DUP', - 'prefix_84015_DUP': 'suffix_271757_DUP', - 'prefix_84044_DEL': 'suffix_263835_DEL', - 'prefix_8420_DEL': 'suffix_75588_DEL', - 'prefix_84288_DUP': 'suffix_338248_DUP', - 'prefix_84403_DUP': 'suffix_345428_DUP', - 'prefix_84717_DEL': 'suffix_94529_DEL', - 'prefix_85642_DEL': 'suffix_236439_DEL', - 'prefix_86089_DEL': 'suffix_109334_DEL', - 'prefix_86353_DEL': 'suffix_12713_DEL', - 'prefix_86387_DEL': 'suffix_24837_DEL', - 'prefix_87733_DEL': 'suffix_147400_DEL', - 'prefix_87760_DEL': 'suffix_337119_DEL', - 'prefix_88833_DUP': 'suffix_37585_DUP', - 'prefix_88886_DEL': 'suffix_98841_DEL', - 'prefix_889_DUP': 'suffix_1064_DUP', - 'prefix_8950_DEL': 'suffix_314461_DEL', - 'prefix_9017_DEL': 'suffix_344075_DEL', - 'prefix_90242_DUP': 'suffix_340596_DUP', - 'prefix_9027_DEL': 'suffix_10504_DEL', - 'prefix_90299_DEL': 'suffix_170093_DEL', - 'prefix_92044_DUP': 'suffix_229729_DUP', - 'prefix_92416_DEL': 'suffix_41456_DEL', - 'prefix_92442_DUP': 'suffix_140124_DUP', - 'prefix_92506_DUP': 'suffix_244044_DUP', - 'prefix_92527_DUP': 'suffix_308955_DUP', - 'prefix_92550_DEL': 'suffix_336957_DEL', - 'prefix_92558_DUP': 'suffix_309329_DUP', - 'prefix_92559_DUP': 'suffix_309345_DUP', - 'prefix_92627_DUP': 'suffix_342164_DUP', - 'prefix_92640_DEL': 'suffix_256704_DEL', - 'prefix_92708_DUP': 'suffix_259795_DUP', - 'prefix_92724_DEL': 'suffix_252290_DEL', - 'prefix_99990_DUP': 'suffix_306887_DUP', - 'prefix_99998_DUP': 'suffix_308100_DUP', - 'suffix_102137_DEL_2': 'suffix_105232_DEL', - 'suffix_102188_DEL_2': 'suffix_105284_DEL', - 'suffix_102279_DEL_2': 'suffix_105380_DEL', - 'suffix_10229_DUP_2': 'suffix_10602_DUP', - 'suffix_102741_DUP_2': 'suffix_105859_DUP', - 'suffix_103521_DUP_2': 'suffix_106670_DUP', - 'suffix_103609_DEL_2': 'suffix_106761_DEL', - 'suffix_106849_DUP_2': 'suffix_110077_DUP', - 'suffix_107254_DUP_2': 'suffix_110487_DUP', - 'suffix_107284_DEL_2': 'suffix_110517_DEL', - 'suffix_108892_DUP_2': 'suffix_112158_DUP', - 'suffix_112876_DUP_2': 'suffix_116262_DUP', - 'suffix_115899_DUP_2': 'suffix_119375_DUP', - 'suffix_116678_DUP_2': 'suffix_120179_DUP', - 'suffix_116680_DEL_2': 'suffix_120181_DEL', - 'suffix_1179_DEL_2': 'suffix_1216_DEL', - 'suffix_118089_DEL_2': 'suffix_121646_DEL', - 'suffix_118753_DUP_2': 'suffix_122349_DUP', - 'suffix_118957_DUP_2': 'suffix_122563_DUP', - 'suffix_121705_DUP_2': 'suffix_125377_DUP', - 'suffix_121728_DUP_2': 'suffix_125401_DUP', - 'suffix_123193_DEL_2': 'suffix_126890_DEL', - 'suffix_12347_DEL_2': 'suffix_12771_DEL', - 'suffix_123507_DEL_2': 'suffix_127211_DEL', - 'suffix_123859_DUP_2': 'suffix_127567_DUP', - 'suffix_126255_DEL_2': 'suffix_130031_DEL', - 'suffix_126378_DEL_2': 'suffix_130158_DEL', - 'suffix_126871_DEL_2': 'suffix_130662_DEL', - 'suffix_127257_DEL_2': 'suffix_131070_DEL', - 'suffix_127603_DUP_2': 'suffix_131424_DUP', - 'suffix_127635_DUP_2': 'suffix_131456_DUP', - 'suffix_128475_DEL_2': 'suffix_132338_DEL', - 'suffix_130874_DUP_2': 'suffix_134810_DUP', - 'suffix_133323_DUP_2': 'suffix_137335_DUP', - 'suffix_137071_DUP_2': 'suffix_141198_DUP', - 'suffix_1371_DUP_2': 'suffix_1425_DUP', - 'suffix_137756_DUP_2': 'suffix_141889_DUP', - 'suffix_138407_DUP_2': 'suffix_142558_DUP', - 'suffix_139177_DUP_2': 'suffix_143347_DUP', - 'suffix_139287_DEL_2': 'suffix_143460_DEL', - 'suffix_140067_DUP_2': 'suffix_144262_DUP', - 'suffix_140211_DEL_2': 'suffix_144411_DEL', - 'suffix_140216_DUP_2': 'suffix_144417_DUP', - 'suffix_140998_DUP_2': 'suffix_145233_DUP', - 'suffix_143072_DUP_2': 'suffix_147367_DUP', - 'suffix_144703_DUP_2': 'suffix_149060_DUP', - 'suffix_146079_DEL_2': 'suffix_150480_DEL', - 'suffix_146082_DEL_2': 'suffix_150483_DEL', - 'suffix_146309_DEL_2': 'suffix_150716_DEL', - 'suffix_146679_DEL_2': 'suffix_151094_DEL', - 'suffix_147023_DUP_2': 'suffix_151455_DUP', - 'suffix_147062_DUP_2': 'suffix_151496_DUP', - 'suffix_147065_DEL_2': 'suffix_151498_DEL', - 'suffix_147755_DUP_2': 'suffix_152214_DUP', - 'suffix_148894_DUP_2': 'suffix_153392_DUP', - 'suffix_150694_DUP_2': 'suffix_155261_DUP', - 'suffix_152328_DUP_2': 'suffix_156958_DUP', - 'suffix_152423_DUP_2': 'suffix_157059_DUP', - 'suffix_152437_DUP_2': 'suffix_157073_DUP', - 'suffix_152439_DEL_2': 'suffix_157075_DEL', - 'suffix_153155_DUP_2': 'suffix_157821_DUP', - 'suffix_153923_DEL_2': 'suffix_158633_DEL', - 'suffix_15410_DEL_2': 'suffix_15899_DEL', - 'suffix_15414_DEL_2': 'suffix_15903_DEL', - 'suffix_154873_DEL_2': 'suffix_159630_DEL', - 'suffix_156350_DUP_2': 'suffix_161149_DUP', - 'suffix_156426_DUP_2': 'suffix_161225_DUP', - 'suffix_157651_DUP_2': 'suffix_162486_DUP', - 'suffix_161505_DUP_2': 'suffix_166461_DUP', - 'suffix_162350_DUP_2': 'suffix_167337_DUP', - 'suffix_16449_DEL_2': 'suffix_16965_DEL', - 'suffix_16476_DEL_2': 'suffix_16992_DEL', - 'suffix_164891_DUP_2': 'suffix_169964_DUP', - 'suffix_165535_DEL_2': 'suffix_170615_DEL', - 'suffix_167366_DUP_2': 'suffix_172492_DUP', - 'suffix_167382_DUP_2': 'suffix_172508_DUP', - 'suffix_169697_DUP_2': 'suffix_174892_DUP', - 'suffix_170359_DEL_2': 'suffix_175579_DEL', - 'suffix_171232_DEL_2': 'suffix_176479_DEL', - 'suffix_171251_DEL_2': 'suffix_176499_DEL', - 'suffix_171281_DEL_2': 'suffix_176529_DEL', - 'suffix_172345_DUP_2': 'suffix_177610_DUP', - 'suffix_173330_DUP_2': 'suffix_178634_DUP', - 'suffix_173374_DUP_2': 'suffix_178679_DUP', - 'suffix_173898_DEL_2': 'suffix_179214_DEL', - 'suffix_176174_DUP_2': 'suffix_181566_DUP', - 'suffix_176223_DUP_2': 'suffix_181617_DUP', - 'suffix_177584_DEL_2': 'suffix_183019_DEL', - 'suffix_177830_DUP_2': 'suffix_183274_DUP', - 'suffix_178098_DUP_2': 'suffix_183558_DUP', - 'suffix_178276_DEL_2': 'suffix_183739_DEL', - 'suffix_178301_DEL_2': 'suffix_183764_DEL', - 'suffix_179460_DEL_2': 'suffix_184960_DEL', - 'suffix_180373_DEL_2': 'suffix_185895_DEL', - 'suffix_180638_DEL_2': 'suffix_186170_DEL', - 'suffix_180645_DUP_2': 'suffix_186177_DUP', - 'suffix_180670_DUP_2': 'suffix_186202_DUP', - 'suffix_182096_DEL_2': 'suffix_187682_DEL', - 'suffix_182350_DUP_2': 'suffix_187944_DUP', - 'suffix_182381_DEL_2': 'suffix_187975_DEL', - 'suffix_183117_DUP_2': 'suffix_188748_DUP', - 'suffix_183487_DUP_2': 'suffix_189139_DUP', - 'suffix_183693_DUP_2': 'suffix_189354_DUP', - 'suffix_184465_DUP_2': 'suffix_190174_DUP', - 'suffix_186206_DUP_2': 'suffix_191986_DUP', - 'suffix_186235_DEL_2': 'suffix_192019_DEL', - 'suffix_186644_DUP_2': 'suffix_192445_DUP', - 'suffix_186658_DEL_2': 'suffix_192460_DEL', - 'suffix_187547_DEL_2': 'suffix_193377_DEL', - 'suffix_188120_DEL_2': 'suffix_193959_DEL', - 'suffix_188699_DEL_2': 'suffix_194556_DEL', - 'suffix_189196_DUP_2': 'suffix_195072_DUP', - 'suffix_189361_DEL_2': 'suffix_195245_DEL', - 'suffix_189787_DEL_2': 'suffix_195683_DEL', - 'suffix_189988_DUP_2': 'suffix_195887_DUP', - 'suffix_192255_DEL_2': 'suffix_198235_DEL', - 'suffix_194657_DUP_2': 'suffix_200737_DUP', - 'suffix_195491_DUP_2': 'suffix_201616_DUP', - 'suffix_196079_DUP_2': 'suffix_202227_DUP', - 'suffix_197227_DUP_2': 'suffix_203421_DUP', - 'suffix_197905_DUP_2': 'suffix_204137_DUP', - 'suffix_199236_DUP_2': 'suffix_205495_DUP', - 'suffix_199351_DUP_2': 'suffix_205610_DUP', - 'suffix_199613_DUP_2': 'suffix_205874_DUP', - 'suffix_199853_DEL_2': 'suffix_206119_DEL', - 'suffix_200568_DEL_2': 'suffix_206849_DEL', - 'suffix_200934_DUP_2': 'suffix_207220_DUP', - 'suffix_2015_DUP_2': 'suffix_2089_DUP', - 'suffix_201955_DUP_2': 'suffix_208285_DUP', - 'suffix_202641_DUP_2': 'suffix_208994_DUP', - 'suffix_205028_DUP_2': 'suffix_211453_DUP', - 'suffix_205123_DEL_2': 'suffix_211548_DEL', - 'suffix_205138_DEL_2': 'suffix_211564_DEL', - 'suffix_206613_DEL_2': 'suffix_213095_DEL', - 'suffix_206780_DEL_2': 'suffix_213264_DEL', - 'suffix_207068_DEL_2': 'suffix_213553_DEL', - 'suffix_207138_DEL_2': 'suffix_213623_DEL', - 'suffix_207672_DEL_2': 'suffix_214167_DEL', - 'suffix_207715_DEL_2': 'suffix_214213_DEL', - 'suffix_208370_DEL_2': 'suffix_214895_DEL', - 'suffix_208573_DEL_2': 'suffix_215104_DEL', - 'suffix_208972_DEL_2': 'suffix_215523_DEL', - 'suffix_209068_DEL_2': 'suffix_215620_DEL', - 'suffix_209199_DUP_2': 'suffix_215756_DUP', - 'suffix_210259_DUP_2': 'suffix_216845_DUP', - 'suffix_210283_DUP_2': 'suffix_216871_DUP', - 'suffix_210613_DEL_2': 'suffix_217209_DEL', - 'suffix_212296_DEL_2': 'suffix_218955_DEL', - 'suffix_212578_DEL_2': 'suffix_219252_DEL', - 'suffix_212637_DEL_2': 'suffix_219317_DEL', - 'suffix_212756_DEL_2': 'suffix_219444_DEL', - 'suffix_213708_DEL_2': 'suffix_220434_DEL', - 'suffix_213875_DEL_2': 'suffix_220605_DEL', - 'suffix_214152_DEL_2': 'suffix_220894_DEL', - 'suffix_214429_DEL_2': 'suffix_221177_DEL', - 'suffix_214857_DEL_2': 'suffix_221627_DEL', - 'suffix_216188_DEL_2': 'suffix_223018_DEL', - 'suffix_216198_DEL_2': 'suffix_223028_DEL', - 'suffix_216958_DUP_2': 'suffix_223818_DUP', - 'suffix_217583_DEL_2': 'suffix_224470_DEL', - 'suffix_217868_DEL_2': 'suffix_224762_DEL', - 'suffix_217871_DEL_2': 'suffix_224765_DEL', - 'suffix_218073_DUP_2': 'suffix_224973_DUP', - 'suffix_218439_DUP_2': 'suffix_225366_DUP', - 'suffix_218904_DUP_2': 'suffix_225864_DUP', - 'suffix_219658_DUP_2': 'suffix_226657_DUP', - 'suffix_220370_DEL_2': 'suffix_227413_DEL', - 'suffix_223567_DUP_2': 'suffix_230759_DUP', - 'suffix_223841_DUP_2': 'suffix_231044_DUP', - 'suffix_22451_DEL_2': 'suffix_23110_DEL', - 'suffix_224608_DEL_2': 'suffix_231845_DEL', - 'suffix_226755_DEL_2': 'suffix_234078_DEL', - 'suffix_227182_DEL_2': 'suffix_234514_DEL', - 'suffix_227203_DEL_2': 'suffix_234535_DEL', - 'suffix_227204_DEL_2': 'suffix_234536_DEL', - 'suffix_228485_DEL_2': 'suffix_235865_DEL', - 'suffix_2306_DEL_2': 'suffix_2387_DEL', - 'suffix_230714_DUP_2': 'suffix_238195_DUP', - 'suffix_230932_DEL_2': 'suffix_238422_DEL', - 'suffix_231136_DEL_2': 'suffix_238630_DEL', - 'suffix_234299_DUP_2': 'suffix_241888_DUP', - 'suffix_236708_DEL_2': 'suffix_244400_DEL', - 'suffix_236887_DEL_2': 'suffix_244582_DEL', - 'suffix_237084_DEL_2': 'suffix_244791_DEL', - 'suffix_237399_DEL_2': 'suffix_245122_DEL', - 'suffix_238815_DEL_2': 'suffix_246599_DEL', - 'suffix_238984_DUP_2': 'suffix_246775_DUP', - 'suffix_239194_DEL_2': 'suffix_246989_DEL', - 'suffix_239311_DUP_2': 'suffix_247109_DUP', - 'suffix_240516_DUP_2': 'suffix_248341_DUP', - 'suffix_24067_DEL_2': 'suffix_24774_DEL', - 'suffix_24112_DEL_2': 'suffix_24819_DEL', - 'suffix_245147_DUP_2': 'suffix_253116_DUP', - 'suffix_245631_DUP_2': 'suffix_253615_DUP', - 'suffix_246007_DEL_2': 'suffix_254014_DEL', - 'suffix_246832_DUP_2': 'suffix_254855_DUP', - 'suffix_249752_DEL_2': 'suffix_257879_DEL', - 'suffix_249760_DUP_2': 'suffix_257887_DUP', - 'suffix_250396_DUP_2': 'suffix_258549_DUP', - 'suffix_251212_DEL_2': 'suffix_259389_DEL', - 'suffix_251719_DUP_2': 'suffix_259912_DUP', - 'suffix_252163_DEL_2': 'suffix_260375_DEL', - 'suffix_254429_DEL_2': 'suffix_262732_DEL', - 'suffix_254575_DEL_2': 'suffix_262881_DEL', - 'suffix_255204_DEL_2': 'suffix_263531_DEL', - 'suffix_25533_DUP_2': 'suffix_26296_DUP', - 'suffix_256655_DEL_2': 'suffix_265034_DEL', - 'suffix_256744_DEL_2': 'suffix_265124_DEL', - 'suffix_256775_DEL_2': 'suffix_265157_DEL', - 'suffix_25949_DEL_2': 'suffix_26718_DEL', - 'suffix_260357_DEL_2': 'suffix_268858_DEL', - 'suffix_260853_DUP_2': 'suffix_269376_DUP', - 'suffix_260855_DEL_2': 'suffix_269379_DEL', - 'suffix_261034_DEL_2': 'suffix_269563_DEL', - 'suffix_262533_DEL_2': 'suffix_271118_DEL', - 'suffix_262539_DEL_2': 'suffix_271125_DEL', - 'suffix_26417_DUP_2': 'suffix_27197_DUP', - 'suffix_265005_DEL_2': 'suffix_273689_DEL', - 'suffix_266230_DEL_2': 'suffix_274984_DEL', - 'suffix_268706_DUP_2': 'suffix_277538_DUP', - 'suffix_269299_DEL_2': 'suffix_278150_DEL', - 'suffix_27000_DUP_2': 'suffix_27795_DUP', - 'suffix_27116_DEL_2': 'suffix_27917_DEL', - 'suffix_272983_DEL_2': 'suffix_281956_DEL', - 'suffix_273392_DEL_2': 'suffix_282368_DEL', - 'suffix_273424_DUP_2': 'suffix_282400_DUP', - 'suffix_273728_DEL_2': 'suffix_282706_DEL', - 'suffix_273736_DEL_2': 'suffix_282714_DEL', - 'suffix_274940_DEL_2': 'suffix_283943_DEL', - 'suffix_276413_DEL_2': 'suffix_285456_DEL', - 'suffix_27644_DEL_2': 'suffix_28486_DEL', - 'suffix_277451_DEL_2': 'suffix_286527_DEL', - 'suffix_278695_DEL_2': 'suffix_287788_DEL', - 'suffix_279450_DEL_2': 'suffix_288554_DEL', - 'suffix_279452_DEL_2': 'suffix_288556_DEL', - 'suffix_280776_DEL_2': 'suffix_289942_DEL', - 'suffix_280836_DUP_2': 'suffix_290002_DUP', - 'suffix_281014_DEL_2': 'suffix_290187_DEL', - 'suffix_281363_DUP_2': 'suffix_290550_DUP', - 'suffix_281846_DEL_2': 'suffix_291053_DEL', - 'suffix_282575_DUP_2': 'suffix_291822_DUP', - 'suffix_283188_DEL_2': 'suffix_292459_DEL', - 'suffix_283756_DEL_2': 'suffix_293051_DEL', - 'suffix_284360_DEL_2': 'suffix_293684_DEL', - 'suffix_284797_DEL_2': 'suffix_294141_DEL', - 'suffix_286080_DUP_2': 'suffix_295482_DUP', - 'suffix_286258_DUP_2': 'suffix_295667_DUP', - 'suffix_286295_DEL_2': 'suffix_295706_DEL', - 'suffix_286314_DUP_2': 'suffix_295727_DUP', - 'suffix_286315_DEL_2': 'suffix_295726_DEL', - 'suffix_288518_DEL_2': 'suffix_298011_DEL', - 'suffix_288856_DUP_2': 'suffix_298358_DUP', - 'suffix_288968_DEL_2': 'suffix_298482_DEL', - 'suffix_28909_DEL_2': 'suffix_29798_DEL', - 'suffix_289586_DEL_2': 'suffix_299115_DEL', - 'suffix_290193_DEL_2': 'suffix_299736_DEL', - 'suffix_29032_DEL_2': 'suffix_29921_DEL', - 'suffix_292554_DEL_2': 'suffix_302162_DEL', - 'suffix_293764_DEL_2': 'suffix_303412_DEL', - 'suffix_295098_DUP_2': 'suffix_304770_DUP', - 'suffix_295112_DUP_2': 'suffix_304785_DUP', - 'suffix_297809_DUP_2': 'suffix_307588_DUP', - 'suffix_297979_DEL_2': 'suffix_307770_DEL', - 'suffix_298560_DUP_2': 'suffix_308385_DUP', - 'suffix_299334_DEL_2': 'suffix_309189_DEL', - 'suffix_299424_DUP_2': 'suffix_309280_DUP', - 'suffix_299426_DUP_2': 'suffix_309282_DUP', - 'suffix_299461_DUP_2': 'suffix_309319_DUP', - 'suffix_299537_DUP_2': 'suffix_309398_DUP', - 'suffix_299973_DEL_2': 'suffix_309849_DEL', - 'suffix_300029_DEL_2': 'suffix_309906_DEL', - 'suffix_300096_DUP_2': 'suffix_309975_DUP', - 'suffix_300178_DEL_2': 'suffix_310060_DEL', - 'suffix_300705_DUP_2': 'suffix_310598_DUP', - 'suffix_300835_DUP_2': 'suffix_310730_DUP', - 'suffix_300947_DUP_2': 'suffix_310843_DUP', - 'suffix_301053_DUP_2': 'suffix_310954_DUP', - 'suffix_301313_DEL_2': 'suffix_311215_DEL', - 'suffix_302352_DEL_2': 'suffix_312295_DEL', - 'suffix_302614_DUP_2': 'suffix_312565_DUP', - 'suffix_305489_DEL_2': 'suffix_315530_DEL', - 'suffix_305497_DEL_2': 'suffix_315539_DEL', - 'suffix_305995_DEL_2': 'suffix_316050_DEL', - 'suffix_30805_DEL_2': 'suffix_31751_DEL', - 'suffix_310213_DUP_2': 'suffix_320398_DUP', - 'suffix_310801_DUP_2': 'suffix_321018_DUP', - 'suffix_312831_DUP_2': 'suffix_323123_DUP', - 'suffix_313834_DUP_2': 'suffix_324164_DUP', - 'suffix_314680_DUP_2': 'suffix_325048_DUP', - 'suffix_318209_DUP_2': 'suffix_328707_DUP', - 'suffix_319248_DUP_2': 'suffix_329772_DUP', - 'suffix_319459_DEL_2': 'suffix_329991_DEL', - 'suffix_320939_DUP_2': 'suffix_331503_DUP', - 'suffix_321448_DEL_2': 'suffix_332033_DEL', - 'suffix_321475_DEL_2': 'suffix_332061_DEL', - 'suffix_323409_DUP_2': 'suffix_334076_DUP', - 'suffix_323934_DUP_2': 'suffix_334633_DUP', - 'suffix_324816_DUP_2': 'suffix_335549_DUP', - 'suffix_32507_DUP_2': 'suffix_33497_DUP', - 'suffix_325590_DUP_2': 'suffix_336339_DUP', - 'suffix_325871_DUP_2': 'suffix_336630_DUP', - 'suffix_326049_DUP_2': 'suffix_336809_DUP', - 'suffix_326068_DUP_2': 'suffix_336828_DUP', - 'suffix_326213_DUP_2': 'suffix_336974_DUP', - 'suffix_326219_DEL_2': 'suffix_336980_DEL', - 'suffix_326226_DEL_2': 'suffix_336987_DEL', - 'suffix_326275_DUP_2': 'suffix_337036_DUP', - 'suffix_326360_DEL_2': 'suffix_337122_DEL', - 'suffix_326662_DEL_2': 'suffix_337431_DEL', - 'suffix_326846_DEL_2': 'suffix_337620_DEL', - 'suffix_327283_DEL_2': 'suffix_338071_DEL', - 'suffix_327813_DEL_2': 'suffix_338612_DEL', - 'suffix_328030_DUP_2': 'suffix_338831_DUP', - 'suffix_328354_DUP_2': 'suffix_339162_DUP', - 'suffix_328867_DUP_2': 'suffix_339705_DUP', - 'suffix_32906_DUP_2': 'suffix_33904_DUP', - 'suffix_32996_DUP_2': 'suffix_34000_DUP', - 'suffix_332611_DUP_2': 'suffix_343530_DUP', - 'suffix_332820_DEL_2': 'suffix_343748_DEL', - 'suffix_332823_DEL_2': 'suffix_343751_DEL', - 'suffix_333026_DEL_2': 'suffix_343960_DEL', - 'suffix_333055_DUP_2': 'suffix_343990_DUP', - 'suffix_333717_DUP_2': 'suffix_344664_DUP', - 'suffix_334188_DEL_2': 'suffix_345154_DEL', - 'suffix_334626_DEL_2': 'suffix_345601_DEL', - 'suffix_34446_DEL_2': 'suffix_35494_DEL', - 'suffix_35726_DEL_2': 'suffix_36817_DEL', - 'suffix_35799_DEL_2': 'suffix_36894_DEL', - 'suffix_3651_DEL_2': 'suffix_3783_DEL', - 'suffix_3691_DEL_2': 'suffix_3824_DEL', - 'suffix_37781_DUP_2': 'suffix_38929_DUP', - 'suffix_39085_DUP_2': 'suffix_40276_DUP', - 'suffix_42000_DUP_2': 'suffix_43285_DUP', - 'suffix_44539_DUP_2': 'suffix_45913_DUP', - 'suffix_44736_DUP_2': 'suffix_46123_DUP', - 'suffix_4505_DEL_2': 'suffix_4670_DEL', - 'suffix_46580_DUP_2': 'suffix_48024_DUP', - 'suffix_46882_DEL_2': 'suffix_48334_DEL', - 'suffix_4995_DUP_2': 'suffix_5177_DUP', - 'suffix_50873_DEL_2': 'suffix_52450_DEL', - 'suffix_51969_DUP_2': 'suffix_53573_DUP', - 'suffix_5276_DUP_2': 'suffix_5474_DUP', - 'suffix_57692_DUP_2': 'suffix_59439_DUP', - 'suffix_58774_DUP_2': 'suffix_60544_DUP', - 'suffix_58893_DEL_2': 'suffix_60667_DEL', - 'suffix_59352_DEL_2': 'suffix_61133_DEL', - 'suffix_59736_DUP_2': 'suffix_61529_DUP', - 'suffix_59942_DEL_2': 'suffix_61742_DEL', - 'suffix_6172_DUP_2': 'suffix_6414_DUP', - 'suffix_64006_DUP_2': 'suffix_65935_DUP', - 'suffix_65035_DUP_2': 'suffix_67010_DUP', - 'suffix_65938_DUP_2': 'suffix_67937_DUP', - 'suffix_6612_DEL_2': 'suffix_6868_DEL', - 'suffix_66192_DUP_2': 'suffix_68205_DUP', - 'suffix_66563_DEL_2': 'suffix_68588_DEL', - 'suffix_66964_DUP_2': 'suffix_68997_DUP', - 'suffix_70658_DEL_2': 'suffix_72772_DEL', - 'suffix_71031_DEL_2': 'suffix_73155_DEL', - 'suffix_72679_DUP_2': 'suffix_74850_DUP', - 'suffix_72691_DUP_2': 'suffix_74863_DUP', - 'suffix_72959_DUP_2': 'suffix_75144_DUP', - 'suffix_73683_DUP_2': 'suffix_75884_DUP', - 'suffix_76491_DEL_2': 'suffix_78773_DEL', - 'suffix_7806_DUP_2': 'suffix_8099_DUP', - 'suffix_81587_DUP_2': 'suffix_83995_DUP', - 'suffix_8201_DUP_2': 'suffix_8510_DUP', - 'suffix_82039_DUP_2': 'suffix_84458_DUP', - 'suffix_82642_DUP_2': 'suffix_85071_DUP', - 'suffix_837_DUP_2': 'suffix_862_DUP', - 'suffix_84346_DEL_2': 'suffix_86829_DEL', - 'suffix_84718_DUP_2': 'suffix_87225_DUP', - 'suffix_85814_DUP_2': 'suffix_88367_DUP', - 'suffix_86689_DUP_2': 'suffix_89280_DUP', - 'suffix_87664_DUP_2': 'suffix_90289_DUP', - 'suffix_88351_DEL_2': 'suffix_91000_DEL', - 'suffix_89655_DEL_2': 'suffix_92358_DEL', - 'suffix_90187_DEL_2': 'suffix_92909_DEL', - 'suffix_90707_DEL_2': 'suffix_93436_DEL', - 'suffix_90995_DEL_2': 'suffix_93733_DEL', - 'suffix_91277_DEL_2': 'suffix_94018_DEL', - 'suffix_91760_DUP_2': 'suffix_94518_DUP', - 'suffix_91769_DEL_2': 'suffix_94529_DEL', - 'suffix_94202_DEL_2': 'suffix_97018_DEL', - 'prefix_109408_DUP': 'suffix_220966_DUP', - 'prefix_109696_DUP': 'suffix_9533_DUP', - 'prefix_118254_DUP': 'suffix_132626_DUP', - 'prefix_120819_DUP': 'suffix_135755_DUP', - 'prefix_135048_DUP': 'suffix_151788_DUP', - 'prefix_136173_DEL': 'suffix_54386_DEL', - 'prefix_148570_DEL': 'suffix_107482_DEL', - 'prefix_150603_DUP': 'suffix_169662_DUP', - 'prefix_150646_DUP': 'suffix_343955_DUP', - 'prefix_175273_DUP': 'suffix_322926_DUP', - 'prefix_178590_DUP': 'suffix_201523_DUP', - 'prefix_18102_DUP': 'suffix_47262_DUP', - 'prefix_184122_DUP': 'suffix_118450_DUP', - 'prefix_191470_DUP': 'suffix_117959_DUP', - 'prefix_19177_DUP': 'suffix_21314_DUP', - 'prefix_193273_DUP': 'suffix_218537_DUP', - 'prefix_195055_DUP': 'suffix_277123_DUP', - 'prefix_195635_DUP': 'suffix_337960_DUP', - 'prefix_196043_DUP': 'suffix_223818_DUP', - 'prefix_196756_DUP': 'suffix_95953_DUP', - 'prefix_200666_DEL': 'suffix_4258_DEL', - 'prefix_212037_DEL': 'suffix_130018_DEL', - 'prefix_2145_DUP': 'suffix_2610_DUP', - 'prefix_22446_DUP': 'suffix_312822_DUP', - 'prefix_229750_DUP': 'suffix_190694_DUP', - 'prefix_237049_DUP': 'suffix_268708_DUP', - 'prefix_237750_DUP': 'suffix_42133_DUP', - 'prefix_239745_DEL': 'suffix_308552_DEL', - 'prefix_240072_DUP': 'suffix_108801_DUP', - 'prefix_240150_DEL': 'suffix_5244_DEL', - 'prefix_242658_DUP': 'suffix_84380_DUP', - 'prefix_246643_DEL': 'suffix_93326_DEL', - 'prefix_24840_DUP': 'suffix_882_DUP', - 'prefix_252587_DUP': 'suffix_158624_DUP', - 'prefix_254761_DUP': 'suffix_57665_DUP', - 'prefix_255040_DUP': 'suffix_106670_DUP', - 'prefix_261005_DEL': 'suffix_178167_DEL', - 'prefix_263591_DUP': 'suffix_319425_DUP', - 'prefix_263818_DEL': 'suffix_201219_DEL', - 'prefix_285913_DEL': 'suffix_74_DEL', - 'prefix_297312_DUP': 'suffix_337092_DUP', - 'prefix_30992_DEL': 'suffix_331860_DEL', - 'prefix_3358_DEL': 'suffix_71259_DEL', - 'prefix_33658_DUP': 'suffix_268605_DUP', - 'prefix_37118_DUP': 'suffix_248738_DUP', - 'prefix_39859_DUP': 'suffix_122375_DUP', - 'prefix_42735_DUP': 'suffix_155097_DUP', - 'prefix_48349_DUP': 'suffix_150163_DUP', - 'prefix_52511_DUP': 'suffix_234367_DUP', - 'prefix_54966_DEL': 'suffix_62104_DEL', - 'prefix_7639_DUP': 'suffix_307081_DUP', - 'prefix_74657_DUP': 'suffix_83373_DUP', - 'prefix_77210_DUP': 'suffix_241886_DUP', - 'prefix_7935_DUP': 'suffix_124466_DUP', - 'prefix_80037_DUP': 'suffix_332875_DUP', - 'prefix_82121_DUP': 'suffix_269588_DUP', - 'prefix_83942_DUP': 'suffix_322084_DUP', - 'prefix_84486_DUP': 'suffix_94510_DUP', - 'prefix_92979_DEL': 'suffix_103592_DEL', - 'prefix_92985_DEL': 'suffix_148088_DEL', - 'prefix_93168_DEL': 'suffix_184994_DEL', - 'prefix_932_DEL': 'suffix_334231_DEL', - 'prefix_9387_DUP': 'suffix_10911_DUP', - 'prefix_94035_DEL': 'suffix_262718_DEL', - 'prefix_94077_DEL': 'suffix_103402_DEL', - 'prefix_94079_DUP': 'suffix_115968_DUP', - 'prefix_94158_DEL': 'suffix_8045_DEL', - 'prefix_9428_DEL': 'suffix_10965_DEL', - 'prefix_94589_DEL': 'suffix_232242_DEL', - 'prefix_94688_DEL': 'suffix_329994_DEL', - 'prefix_94693_DUP': 'suffix_133009_DUP', - 'prefix_94714_DEL': 'suffix_86407_DEL', - 'prefix_94949_DEL': 'suffix_3640_DEL', - 'prefix_95467_DEL': 'suffix_106269_DEL', - 'prefix_95753_DEL': 'suffix_106601_DEL', - 'prefix_96081_DEL': 'suffix_39335_DEL', - 'prefix_96409_DEL': 'suffix_107290_DEL', - 'prefix_96417_DEL': 'suffix_107299_DEL', - 'prefix_96497_DUP': 'suffix_107390_DUP', - 'prefix_96678_DEL': 'suffix_107578_DEL', - 'prefix_96701_DUP': 'suffix_107599_DUP', - 'prefix_97034_DUP': 'suffix_107976_DUP', - 'prefix_97265_DUP': 'suffix_296214_DUP', - 'prefix_97383_DEL': 'suffix_269445_DEL', - 'prefix_97410_DEL': 'suffix_21697_DEL', - 'prefix_97624_DEL': 'suffix_19282_DEL', - 'prefix_98159_DUP': 'suffix_339672_DUP', - 'prefix_98246_DEL': 'suffix_343159_DEL', - 'prefix_98359_DUP': 'suffix_253054_DUP', - 'prefix_98374_DEL': 'suffix_303813_DEL', - 'prefix_98492_DEL': 'suffix_256422_DEL', - 'prefix_99210_DUP': 'suffix_110452_DUP', - 'prefix_99385_DEL': 'suffix_130483_DEL', - 'prefix_99429_DEL': 'suffix_316828_DEL', - 'prefix_99941_DEL': 'suffix_245189_DEL', - 'prefix_99962_DEL': 'suffix_11652_DEL', - 'prefix_99964_DUP': 'suffix_184974_DUP', - 'prefix_99980_DEL': 'suffix_291827_DEL', - 'prefix_99987_DUP': 'suffix_306785_DUP', - 'suffix_115271_DUP_2': 'suffix_118725_DUP', - 'suffix_147347_DUP_2': 'suffix_151788_DUP', - 'suffix_152903_DUP_2': 'suffix_157559_DUP', - 'suffix_16434_DEL_2': 'suffix_16950_DEL', - 'suffix_17973_DEL_2': 'suffix_18512_DEL', - 'suffix_199564_DUP_2': 'suffix_205825_DUP', - 'suffix_208055_DUP_2': 'suffix_214569_DUP', - 'suffix_208666_DEL_2': 'suffix_215204_DEL', - 'suffix_215449_DUP_2': 'suffix_222247_DUP', - 'suffix_24241_DUP_2': 'suffix_24952_DUP', - 'suffix_282215_DUP_2': 'suffix_291438_DUP', - 'suffix_78257_DEL_2': 'suffix_80567_DEL', - 'prefix_105041_DUP': 'suffix_157907_DUP', - 'prefix_105835_DUP': 'suffix_139575_DUP', - 'prefix_108273_DEL': 'suffix_302962_DEL', - 'prefix_110759_DUP': 'suffix_121735_DUP', - 'prefix_112991_DEL': 'suffix_19457_DEL', - 'prefix_116608_DUP': 'suffix_132952_DUP', - 'prefix_123515_DEL': 'suffix_138768_DEL', - 'prefix_133566_DUP': 'suffix_150246_DUP', - 'prefix_141809_DEL': 'suffix_300943_DEL', - 'prefix_142858_DEL': 'suffix_128588_DEL', - 'prefix_146871_DUP': 'suffix_133106_DUP', - 'prefix_149568_DEL': 'suffix_118635_DEL', - 'prefix_167368_DEL': 'suffix_189133_DEL', - 'prefix_176845_DUP': 'suffix_211875_DUP', - 'prefix_177271_DUP': 'suffix_312822_DUP', - 'prefix_181179_DUP': 'suffix_234798_DUP', - 'prefix_185619_DEL': 'suffix_217950_DEL', - 'prefix_185910_DEL': 'suffix_291085_DEL', - 'prefix_192960_DEL': 'suffix_290358_DEL', - 'prefix_219036_DUP': 'suffix_341481_DUP', - 'prefix_21518_DUP': 'suffix_218342_DUP', - 'prefix_22434_DEL': 'suffix_189133_DEL', - 'prefix_22509_DUP': 'suffix_261218_DUP', - 'prefix_230654_DEL': 'suffix_106308_DEL', - 'prefix_237348_DUP': 'suffix_155009_DUP', - 'prefix_239011_DUP': 'suffix_107426_DUP', - 'prefix_240074_DUP': 'suffix_109378_DUP', - 'prefix_248113_DUP': 'suffix_291388_DUP', - 'prefix_252320_DUP': 'suffix_286050_DUP', - 'prefix_252636_DUP': 'suffix_337479_DUP', - 'prefix_29923_DEL': 'suffix_157806_DEL', - 'prefix_30391_DEL': 'suffix_230937_DEL', - 'prefix_30532_DUP': 'suffix_86319_DUP', - 'prefix_32634_DUP': 'suffix_95910_DUP', - 'prefix_32918_DEL': 'suffix_36811_DEL', - 'prefix_37021_DUP': 'suffix_123374_DUP', - 'prefix_39925_DEL': 'suffix_107137_DEL', - 'prefix_45770_DUP': 'suffix_329746_DUP', - 'prefix_45776_DEL': 'suffix_167381_DEL', - 'prefix_46296_DUP': 'suffix_255601_DUP', - 'prefix_47514_DUP': 'suffix_53429_DUP', - 'prefix_47557_DUP': 'suffix_247215_DUP', - 'prefix_47647_DUP': 'suffix_311519_DUP', - 'prefix_49053_DEL': 'suffix_33001_DEL', - 'prefix_53399_DUP': 'suffix_335231_DUP', - 'prefix_72156_DUP': 'suffix_122944_DUP', - 'prefix_7322_DUP': 'suffix_189132_DUP', - 'prefix_75522_DUP': 'suffix_84376_DUP', - 'prefix_78549_DEL': 'suffix_50090_DEL', - 'prefix_80929_DEL': 'suffix_90588_DEL', - 'prefix_81289_DUP': 'suffix_90965_DUP', - 'prefix_8811_DUP': 'suffix_148957_DUP', - 'prefix_91635_DEL': 'suffix_293519_DEL', - 'prefix_98234_DEL': 'suffix_109388_DEL', - 'prefix_99986_DUP': 'suffix_306740_DUP', - 'suffix_10162_DEL_2': 'suffix_10535_DEL', - 'suffix_119700_DUP_2': 'suffix_123318_DUP', - 'suffix_146970_DUP_2': 'suffix_151397_DUP', - 'suffix_189219_DEL_2': 'suffix_195097_DEL', - 'suffix_200704_DUP_2': 'suffix_206987_DUP', - 'suffix_208083_DEL_2': 'suffix_214598_DEL', - 'suffix_208097_DEL_2': 'suffix_214612_DEL', - 'suffix_212079_DUP_2': 'suffix_218732_DUP', - 'suffix_24668_DUP_2': 'suffix_25398_DUP', - 'suffix_247327_DUP_2': 'suffix_255366_DUP', - 'suffix_248140_DUP_2': 'suffix_256203_DUP', - 'suffix_26120_DUP_2': 'suffix_26896_DUP', - 'suffix_26153_DUP_2': 'suffix_26929_DUP', - 'suffix_28933_DEL_2': 'suffix_29823_DEL', - 'suffix_83854_DUP_2': 'suffix_86319_DUP', - 'prefix_113219_DUP': 'suffix_345434_DUP', - 'prefix_173077_DEL': 'suffix_195402_DEL', - 'prefix_11978_DEL': 'suffix_283553_DEL', - 'prefix_284589_DUP': 'suffix_289594_DUP', - 'prefix_4812_DUP': 'suffix_5671_DUP', - 'suffix_326634_DEL_2': 'suffix_337403_DEL', - 'suffix_333975_DUP_2': 'suffix_344929_DUP', - 'prefix_102605_DEL': 'suffix_265004_DEL', - 'prefix_103754_DEL': 'suffix_308671_DEL', - 'prefix_105176_DEL': 'suffix_617_DEL', - 'prefix_105255_DEL': 'suffix_212161_DEL', - 'prefix_105413_DUP': 'suffix_16872_DUP', - 'prefix_108429_DEL': 'suffix_276230_DEL', - 'prefix_11302_DEL': 'suffix_211819_DEL', - 'prefix_115700_DUP': 'suffix_201547_DUP', - 'prefix_117580_DUP': 'suffix_181916_DUP', - 'prefix_119226_DEL': 'suffix_223666_DEL', - 'prefix_119993_DUP': 'suffix_62592_DUP', - 'prefix_121412_DEL': 'suffix_136441_DEL', - 'prefix_12154_DEL': 'suffix_16787_DEL', - 'prefix_12156_DEL': 'suffix_16888_DEL', - 'prefix_12407_DEL': 'suffix_193951_DEL', - 'prefix_126083_DUP': 'suffix_93868_DUP', - 'prefix_132933_DUP': 'suffix_178504_DUP', - 'prefix_132951_DUP': 'suffix_201246_DUP', - 'prefix_138735_DUP': 'suffix_155567_DUP', - 'prefix_140549_DUP': 'suffix_157451_DUP', - 'prefix_141948_DEL': 'suffix_159338_DEL', - 'prefix_146744_DEL': 'suffix_31511_DEL', - 'prefix_147468_DUP': 'suffix_166040_DUP', - 'prefix_152051_DUP': 'suffix_136974_DUP', - 'prefix_15323_DUP': 'suffix_62805_DUP', - 'prefix_164074_DEL': 'suffix_2290_DEL', - 'prefix_167_DEL': 'suffix_46122_DEL', - 'prefix_168590_DUP': 'suffix_288536_DUP', - 'prefix_168606_DUP': 'suffix_157956_DUP', - 'prefix_171981_DUP': 'suffix_35868_DUP', - 'prefix_173688_DUP': 'suffix_54837_DUP', - 'prefix_174135_DEL': 'suffix_16791_DEL', - 'prefix_175272_DUP': 'suffix_322830_DUP', - 'prefix_175274_DUP': 'suffix_322963_DUP', - 'prefix_183_DEL': 'suffix_106660_DEL', - 'prefix_184814_DEL': 'suffix_195404_DEL', - 'prefix_18485_DEL': 'suffix_29789_DEL', - 'prefix_185069_DEL': 'suffix_132891_DEL', - 'prefix_185206_DEL': 'suffix_1868_DEL', - 'prefix_185338_DEL': 'suffix_290358_DEL', - 'prefix_191249_DEL': 'suffix_150483_DEL', - 'prefix_191318_DEL': 'suffix_20510_DEL', - 'prefix_191655_DUP': 'suffix_327391_DUP', - 'prefix_191946_DEL': 'suffix_216855_DEL', - 'prefix_194528_DUP': 'suffix_220210_DUP', - 'prefix_195246_DUP': 'suffix_293034_DUP', - 'prefix_198967_DUP': 'suffix_244057_DUP', - 'prefix_198991_DUP': 'suffix_244754_DUP', - 'prefix_206418_DUP': 'suffix_234368_DUP', - 'prefix_20_DUP': 'suffix_268896_DUP', - 'prefix_216861_DEL': 'suffix_246030_DEL', - 'prefix_22176_DUP': 'suffix_47721_DUP', - 'prefix_22336_DUP': 'suffix_159422_DUP', - 'prefix_22604_DEL': 'suffix_106759_DEL', - 'prefix_2317_DEL': 'suffix_176841_DEL', - 'prefix_233367_DUP': 'suffix_100639_DUP', - 'prefix_23494_DEL': 'suffix_333136_DEL', - 'prefix_236689_DUP': 'suffix_268271_DUP', - 'prefix_236881_DUP': 'suffix_251030_DUP', - 'prefix_23854_DEL': 'suffix_120387_DEL', - 'prefix_240210_DUP': 'suffix_184198_DUP', - 'prefix_241766_DEL': 'suffix_274245_DEL', - 'prefix_242193_DUP': 'suffix_317066_DUP', - 'prefix_242258_DUP': 'suffix_257305_DUP', - 'prefix_24253_DEL': 'suffix_212318_DEL', - 'prefix_243424_DUP': 'suffix_276101_DUP', - 'prefix_243699_DUP': 'suffix_276379_DUP', - 'prefix_24375_DUP': 'suffix_258696_DUP', - 'prefix_24557_DUP': 'suffix_47694_DUP', - 'prefix_250938_DUP': 'suffix_105404_DUP', - 'prefix_250978_DEL': 'suffix_114000_DEL', - 'prefix_251024_DEL': 'suffix_292979_DEL', - 'prefix_252358_DUP': 'suffix_69555_DUP', - 'prefix_252448_DUP': 'suffix_210958_DUP', - 'prefix_253050_DUP': 'suffix_11042_DUP', - 'prefix_255274_DEL': 'suffix_74443_DEL', - 'prefix_255883_DEL': 'suffix_105498_DEL', - 'prefix_257310_DUP': 'suffix_291433_DUP', - 'prefix_260697_DUP': 'suffix_101357_DUP', - 'prefix_262045_DUP': 'suffix_125836_DUP', - 'prefix_266484_DUP': 'suffix_301782_DUP', - 'prefix_26953_DEL': 'suffix_307435_DEL', - 'prefix_272536_DUP': 'suffix_308571_DUP', - 'prefix_273661_DEL': 'suffix_309764_DEL', - 'prefix_277674_DEL': 'suffix_126886_DEL', - 'prefix_27955_DEL': 'suffix_118689_DEL', - 'prefix_280210_DEL': 'suffix_63525_DEL', - 'prefix_23608_DEL': 'suffix_141141_DEL', - 'prefix_284014_DUP': 'suffix_217043_DUP', - 'prefix_297304_DEL': 'suffix_337079_DEL', - 'prefix_298624_DUP': 'suffix_338928_DUP', - 'prefix_29886_DEL': 'suffix_122545_DEL', - 'prefix_30066_DEL': 'suffix_105693_DEL', - 'prefix_30121_DUP': 'suffix_82107_DUP', - 'prefix_31125_DUP': 'suffix_34864_DUP', - 'prefix_31175_DEL': 'suffix_334478_DEL', - 'prefix_32814_DEL': 'suffix_290329_DEL', - 'prefix_3353_DUP': 'suffix_4034_DUP', - 'prefix_3417_DUP': 'suffix_106659_DUP', - 'prefix_34842_DUP': 'suffix_110095_DUP', - 'prefix_352_DEL': 'suffix_195699_DEL', - 'prefix_36954_DUP': 'suffix_322053_DUP', - 'prefix_36966_DUP': 'suffix_137770_DUP', - 'prefix_37088_DUP': 'suffix_246551_DUP', - 'prefix_37312_DEL': 'suffix_269442_DEL', - 'prefix_37395_DEL': 'suffix_268115_DEL', - 'prefix_37462_DUP': 'suffix_155684_DUP', - 'prefix_37511_DUP': 'suffix_88367_DUP', - 'prefix_38100_DUP': 'suffix_280153_DUP', - 'prefix_41266_DEL': 'suffix_46106_DEL', - 'prefix_44220_DEL': 'suffix_29831_DEL', - 'prefix_45074_DEL': 'suffix_102910_DEL', - 'prefix_4522_DEL': 'suffix_246030_DEL', - 'prefix_463_DUP': 'suffix_291433_DUP', - 'prefix_46468_DUP': 'suffix_273617_DUP', - 'prefix_46530_DEL': 'suffix_258864_DEL', - 'prefix_47637_DUP': 'suffix_309976_DUP', - 'prefix_51641_DEL': 'suffix_58293_DEL', - 'prefix_53677_DUP': 'suffix_34864_DUP', - 'prefix_54008_DUP': 'suffix_285632_DUP', - 'prefix_54421_DUP': 'suffix_47721_DUP', - 'prefix_55677_DUP': 'suffix_62795_DUP', - 'prefix_65173_DUP': 'suffix_290375_DUP', - 'prefix_56142_DUP': 'suffix_147501_DUP', - 'prefix_58202_DUP': 'suffix_116683_DUP', - 'prefix_5881_DUP': 'suffix_230728_DUP', - 'prefix_62733_DUP': 'suffix_228856_DUP', - 'prefix_628_DEL': 'suffix_195672_DEL', - 'prefix_6327_DEL': 'suffix_283625_DEL', - 'prefix_63755_DUP': 'suffix_258929_DUP', - 'prefix_6487_DEL': 'suffix_337048_DEL', - 'prefix_6596_DEL': 'suffix_35678_DEL', - 'prefix_66180_DEL': 'suffix_42022_DEL', - 'prefix_6683_DUP': 'suffix_189168_DUP', - 'prefix_69844_DEL': 'suffix_139225_DEL', - 'prefix_71567_DUP': 'suffix_297455_DUP', - 'prefix_71575_DUP': 'suffix_292697_DUP', - 'prefix_71627_DUP': 'suffix_101357_DUP', - 'prefix_72141_DUP': 'suffix_334049_DUP', - 'prefix_74496_DUP': 'suffix_37213_DUP', - 'prefix_74947_DEL': 'suffix_1698_DEL', - 'prefix_77626_DEL': 'suffix_35767_DEL', - 'prefix_778_DUP': 'suffix_166040_DUP', - 'prefix_78301_DEL': 'suffix_336999_DEL', - 'prefix_78948_DUP': 'suffix_308282_DUP', - 'prefix_79561_DUP': 'suffix_216297_DUP', - 'prefix_79966_DUP': 'suffix_202096_DUP', - 'prefix_8308_DUP': 'suffix_62795_DUP', - 'prefix_83433_DEL': 'suffix_307445_DEL', - 'prefix_83930_DUP': 'suffix_32721_DUP', - 'prefix_86900_DUP': 'suffix_47150_DUP', - 'prefix_89846_DEL': 'suffix_234997_DEL', - 'prefix_92729_DEL': 'suffix_287108_DEL', - 'prefix_93407_DEL': 'suffix_336228_DEL', - 'prefix_93530_DUP': 'suffix_324867_DUP', - 'prefix_93992_DUP': 'suffix_181474_DUP', - 'prefix_95800_DUP': 'suffix_106659_DUP', - 'prefix_96646_DUP': 'suffix_212303_DUP', - 'prefix_97428_DEL': 'suffix_335378_DEL', - 'prefix_9785_DEL': 'suffix_336447_DEL', - 'prefix_9871_DEL': 'suffix_13825_DEL', - 'suffix_106521_DEL_2': 'suffix_109744_DEL', - 'suffix_111044_DUP_2': 'suffix_114370_DUP', - 'suffix_115236_DEL_2': 'suffix_118689_DEL', - 'suffix_118984_DUP_2': 'suffix_122590_DUP', - 'suffix_123189_DEL_2': 'suffix_126886_DEL', - 'suffix_128305_DUP_2': 'suffix_132156_DUP', - 'suffix_128994_DEL_2': 'suffix_132877_DEL', - 'suffix_143611_DEL_2': 'suffix_147926_DEL', - 'suffix_145008_DUP_2': 'suffix_149379_DUP', - 'suffix_145792_DUP_2': 'suffix_150187_DUP', - 'suffix_146879_DUP_2': 'suffix_151303_DUP', - 'suffix_152658_DUP_2': 'suffix_157304_DUP', - 'suffix_15417_DEL_2': 'suffix_15906_DEL', - 'suffix_1560_DEL_2': 'suffix_1619_DEL', - 'suffix_159007_DUP_2': 'suffix_163879_DUP', - 'suffix_16278_DUP_2': 'suffix_16792_DUP', - 'suffix_16287_DUP_2': 'suffix_16801_DUP', - 'suffix_16320_DUP_2': 'suffix_16834_DUP', - 'suffix_179450_DEL_2': 'suffix_184949_DEL', - 'suffix_188093_DEL_2': 'suffix_193931_DEL', - 'suffix_189764_DUP_2': 'suffix_195660_DUP', - 'suffix_189797_DEL_2': 'suffix_195693_DEL', - 'suffix_189963_DEL_2': 'suffix_195862_DEL', - 'suffix_190204_DEL_2': 'suffix_196111_DEL', - 'suffix_190596_DEL_2': 'suffix_196510_DEL', - 'suffix_198910_DUP_2': 'suffix_205165_DUP', - 'suffix_198945_DUP_2': 'suffix_205200_DUP', - 'suffix_205378_DUP_2': 'suffix_211816_DUP', - 'suffix_205382_DEL_2': 'suffix_211819_DEL', - 'suffix_206291_DEL_2': 'suffix_212760_DEL', - 'suffix_208232_DEL_2': 'suffix_214749_DEL', - 'suffix_208718_DUP_2': 'suffix_215260_DUP', - 'suffix_210692_DEL_2': 'suffix_217291_DEL', - 'suffix_210883_DUP_2': 'suffix_217485_DUP', - 'suffix_211544_DUP_2': 'suffix_218173_DUP', - 'suffix_216501_DEL_2': 'suffix_223347_DEL', - 'suffix_216580_DEL_2': 'suffix_223427_DEL', - 'suffix_233162_DEL_2': 'suffix_240724_DEL', - 'suffix_238770_DUP_2': 'suffix_246551_DUP', - 'suffix_239364_DEL_2': 'suffix_247162_DEL', - 'suffix_253024_DUP_2': 'suffix_261270_DUP', - 'suffix_26252_DEL_2': 'suffix_27028_DEL', - 'suffix_26422_DEL_2': 'suffix_27202_DEL', - 'suffix_273614_DEL_2': 'suffix_282591_DEL', - 'suffix_274216_DEL_2': 'suffix_283203_DEL', - 'suffix_285251_DEL_2': 'suffix_294612_DEL', - 'suffix_285526_DUP_2': 'suffix_294904_DUP', - 'suffix_286293_DEL_2': 'suffix_295704_DEL', - 'suffix_28918_DEL_2': 'suffix_29808_DEL', - 'suffix_290688_DEL_2': 'suffix_300251_DEL', - 'suffix_291524_DUP_2': 'suffix_301110_DUP', - 'suffix_291969_DUP_2': 'suffix_301562_DUP', - 'suffix_299400_DUP_2': 'suffix_309256_DUP', - 'suffix_303591_DEL_2': 'suffix_313583_DEL', - 'suffix_30447_DUP_2': 'suffix_31375_DUP', - 'suffix_304499_DEL_2': 'suffix_314515_DEL', - 'suffix_3148_DEL_2': 'suffix_3268_DEL', - 'suffix_324233_DUP_2': 'suffix_334946_DUP', - 'suffix_32624_DUP_2': 'suffix_33615_DUP', - 'suffix_35346_DEL_2': 'suffix_36419_DEL', - 'suffix_35506_DEL_2': 'suffix_36591_DEL', - 'suffix_38424_DUP_2': 'suffix_39596_DUP', - 'suffix_38845_DEL_2': 'suffix_40030_DEL', - 'suffix_39353_DUP_2': 'suffix_40557_DUP', - 'suffix_41150_DUP_2': 'suffix_42404_DUP', - 'suffix_47113_DUP_2': 'suffix_48575_DUP', - 'suffix_48026_DUP_2': 'suffix_49547_DUP', - 'suffix_49445_DUP_2': 'suffix_50997_DUP', - 'suffix_61720_DUP_2': 'suffix_63567_DUP', - 'suffix_62316_DUP_2': 'suffix_64183_DUP', - 'suffix_9094_DUP_2': 'suffix_9431_DUP', - 'suffix_914_DEL_2': 'suffix_925_DEL', - 'suffix_92273_DUP_2': 'suffix_95045_DUP', - 'suffix_93367_DUP_2': 'suffix_96167_DUP', - 'prefix_156333_DUP': 'suffix_176635_DUP', - 'prefix_1723_DUP': 'suffix_176578_DUP', - 'prefix_172949_DEL': 'suffix_195223_DUP', - 'prefix_18_DUP': 'suffix_20_DUP', - 'prefix_192882_DUP': 'suffix_225192_DUP', - 'prefix_230001_DEL': 'suffix_106669_DEL', - 'prefix_241273_DUP': 'suffix_273630_DUP', - 'prefix_260458_DUP': 'suffix_295039_DUP', - 'prefix_26552_DEL': 'suffix_118629_DEL', - 'prefix_268993_DUP': 'suffix_304512_DUP', - 'prefix_34289_DUP': 'suffix_122919_DUP', - 'prefix_42135_DUP': 'suffix_47150_DUP', - 'prefix_447_DUP': 'suffix_195223_DUP', - 'prefix_5926_DUP': 'suffix_277846_DUP', - 'prefix_6582_DEL': 'suffix_212509_DEL', - 'prefix_65866_DEL': 'suffix_282591_DEL', - 'prefix_92476_DUP': 'suffix_110991_DUP', - 'prefix_10078_DEL': 'suffix_195635_DEL', - 'prefix_106132_DEL': 'suffix_195692_DEL', - 'prefix_120338_DUP': 'suffix_135104_DUP', - 'prefix_143062_DUP': 'suffix_160819_DUP', - 'prefix_167275_DUP': 'suffix_308395_DUP', - 'prefix_191895_DEL': 'suffix_216785_DEL', - 'prefix_191896_DEL': 'suffix_216785_DEL', - 'prefix_23369_DEL': 'suffix_296578_DEL', - 'prefix_244929_DUP': 'suffix_277868_DUP', - 'prefix_42634_DUP': 'suffix_166401_DUP', - 'prefix_91448_DUP': 'suffix_148093_DUP', - }, -} SV_DROPPED_IDS = { 'cluster_6_last_call_cnv_17479_DUP', 'cluster_1_last_call_cnv_30127_DEL', 'cluster_19_COHORT_cnv_23176_DEL', 'phase2_DEL_chrX_1149', 'prefix_121357_DUP', 'prefix_73945_DEL', 'phase2_DUP_chr9_1663', 'phase2_CPX_chr20_4', @@ -2328,47 +294,11 @@ def _resolve_reloaded_svs(cls, variant_ids): if not missing_with_search_data: return - gcnv_id_map = cls._load_gcnv_id_map() missing_by_sample_type = defaultdict(list) - update_variants_by_sample_type = defaultdict(dict) for variant in missing_with_search_data: variant_id = variant['variant_id'] sample_type = variant['family__individual__sample__sample_type'] - update_id = SV_ID_UPDATE_MAP[sample_type].get(variant_id) - if not update_id and sample_type == 'WES' : - suffix = next((suff for suff in ['_DEL', '_DUP'] if variant_id.endswith(suff)), None) - if suffix: - base_id = gcnv_id_map.get(variant_id.rsplit(suffix)[0]) - if base_id: - update_id = f'{base_id}{suffix}' - if update_id: - update_variants_by_sample_type[sample_type][variant_id] = update_id - elif re.match(r'.*_(DEL|DUP)_\d+', variant_id): - update_variants_by_sample_type[sample_type][variant_id] = variant_id.rsplit('_', 1)[0] - else: - missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" ) - - for sample_type, variant_id_updates in sorted(update_variants_by_sample_type.items()): - logger.info(f'Mapping reloaded SV_{sample_type} IDs to latest version') - failed_mapping = cls._set_variant_keys( - list(variant_id_updates.values()), f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}', - genome_version=GENOME_VERSION_GRCh38, variant_id_updates=variant_id_updates, - ) - if failed_mapping: - logger.info(f'{len(failed_mapping)} variants failed ID mapping: {list(failed_mapping)}') + missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" ) for sample_type, variants in missing_by_sample_type.items(): logger.info(f'{len(variants)} remaining SV {sample_type} variants {", ".join(variants)}') - - @staticmethod - def _load_gcnv_id_map(): - file_content = file_iter(GCNV_CALLSET_PATH) - header = next(file_content).split('\t') - variant_name_idx = header.index('variant_name') - old_id_idx = header.index('any_ovl') - id_map = {} - for raw_row in file_content: - row = raw_row.split('\t') - for old_id in row[old_id_idx].split(';'): - id_map[old_id.strip()] = row[variant_name_idx].strip() - return id_map diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index be47bf5bb5..047c889cb4 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -39,21 +39,14 @@ def test_command(self, mock_subprocess): ('Updated batch of 2', None), ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None), ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None), - ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None), - ('Mapping reloaded SV_WES IDs to latest version', None), - ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), ('Found 1 keys', None), ('Updated batch of 1', None), - ('Updated keys for 1 SV_WES (GRCh38) variants', None), - ('Mapping reloaded SV_WGS IDs to latest version', None), - ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None), + ('Updated keys for 1 SV_WGS (GRCh38) variants', None), + ('No key found for 1 variants', None), + ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), ('Found 1 keys', None), ('Updated batch of 1', None), - ('Updated keys for 1 SV_WGS (GRCh38) variants', None), + ('Updated keys for 1 SV_WES (GRCh38) variants', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), ('Found 1 keys', None), ('Updated batch of 1', None), @@ -70,9 +63,9 @@ def test_command(self, mock_subprocess): {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, 'saved_variant_json': {}}, {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000002_na19675': mock.ANY, 'I000017_na20889': mock.ANY}, 'saved_variant_json': {}}, {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000018_na21234': {'sampleId': 'NA20885', 'ab': 0.0, 'gq': 99.0, 'numAlt': 1}}, 'saved_variant_json': {}}, - {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_70191_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}}, {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'dataset_type': 'MITO', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, - {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}}, {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY}, @@ -121,12 +114,8 @@ def test_command(self, mock_subprocess): ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), ('Found 0 keys', None), ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None), - ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None), - ('Mapping reloaded SV_WGS IDs to latest version', None), - ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None), - ('Found 0 keys', None), - ("1 variants failed ID mapping: ['suffix_19107_DEL']", None), ('1 remaining SV WES variants prefix_19107_DEL - 12', None), + ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), From 35cabcc40d0a814ad009ba0532da8eb111044365 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 26 Sep 2025 17:05:00 -0400 Subject: [PATCH 67/71] remove unused import --- clickhouse_search/management/commands/set_saved_variant_key.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 6b429e54b1..8a283d54fb 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand from django.db.models import F import logging -import re from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 From 6a796f3a760713f3a4f87303b25b2e29b04aea75 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 29 Sep 2025 11:04:39 -0400 Subject: [PATCH 68/71] remove unused code --- .../management/commands/set_saved_variant_key.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 8a283d54fb..49a6d0c0d8 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -214,7 +214,7 @@ def handle(self, *args, **options): logger.info('Done') @staticmethod - def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_GRCh38, variant_id_updates=None): + def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_GRCh38): if not variants_ids: return set() logger.info(f'Finding keys for {len(variants_ids)} {dataset_type} (GRCh{genome_version}) variant ids') @@ -224,13 +224,8 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ return set(variants_ids) mapped_variant_ids = list(variant_key_map.keys()) - if variant_id_updates: - reverse_lookup = {v: k for k, v in variant_id_updates.items()} - mapped_variant_ids = [reverse_lookup[vid] for vid in mapped_variant_ids] update_fields = ['key', 'dataset_type'] - if variant_id_updates: - update_fields.append('variant_id') total_num_updated = 0 for i in range(0, len(mapped_variant_ids), BATCH_SIZE): batch_ids = mapped_variant_ids[i:i + BATCH_SIZE] @@ -238,8 +233,6 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ family__project__genome_version=genome_version, variant_id__in=batch_ids, ) for variant in saved_variants: - if variant_id_updates: - variant.variant_id = variant_id_updates[variant.variant_id] variant.key = variant_key_map[variant.variant_id] variant.dataset_type = dataset_type num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields) From a1fd37e0c3ed83204727c8c897358980a9b39907 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 29 Sep 2025 11:11:33 -0400 Subject: [PATCH 69/71] fix unit test image --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fb021f24f5..71c309be6e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,7 +41,7 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: bitnami/clickhouse:latest + image: bitnamilegacy/clickhouse:latest ports: - 9000:9000 # Native client interface volumes: From c8a36319771374b18627e7c19c218051af6f3609 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 29 Sep 2025 14:03:44 -0400 Subject: [PATCH 70/71] use correct field for querying sample remap --- seqr/views/apis/data_manager_api.py | 4 ++-- seqr/views/apis/data_manager_api_tests.py | 6 +++--- seqr/views/utils/airtable_utils.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index cd70fb083a..f4daeb7b92 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -389,8 +389,8 @@ def _get_valid_search_individuals(project, airtable_samples, vcf_samples, datase } def fetch_missing_vcf_samples(missing_vcf_samples): samples = _get_dataset_type_samples_for_matched_pdos( - LOADABLE_PDO_STATUSES + AVAILABLE_PDO_STATUSES, **get_sample_kwargs, - additional_sample_filters={'VCFIDWithMismatch': sorted(missing_vcf_samples)}, + LOADABLE_PDO_STATUSES + AVAILABLE_PDO_STATUSES, **get_sample_kwargs, sample_fields=['VCFIDWithMismatch'], + additional_sample_filters={'SeqrIDWithMismatch': sorted(missing_vcf_samples)}, ) vcf_sample_id_map.update({ s['sample_id']: s['VCFIDWithMismatch'] for s in samples diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 72b73d34ac..b4a4392c65 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1887,8 +1887,8 @@ def _assert_expected_airtable_call(self, required_sample_field, project_guid, ca def _assert_expected_airtable_vcf_id_call(self, required_sample_field=None, additional_vcf_ids='', **kwargs): self._assert_expected_airtable_call( - required_sample_field, 'R0004_non_analyst_project', **kwargs, additional_fields=['VCFIDWithMismatch'], - additional_filter=f"OR(VCFIDWithMismatch='NA21234'{additional_vcf_ids})", + required_sample_field, 'R0004_non_analyst_project', **kwargs, additional_fields=['VCFIDWithMismatch', 'SeqrIDWithMismatch'], + additional_filter=f"OR(SeqrIDWithMismatch='NA21234'{additional_vcf_ids})", additional_pdo_statuses=",SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';'))", ) @@ -1942,7 +1942,7 @@ def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_m body['sampleType'] = 'WGS' self.assertEqual(len(responses.calls), 1) self._assert_expected_airtable_vcf_id_call( - required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",VCFIDWithMismatch='NA21987'", + required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",SeqrIDWithMismatch='NA21987'", ) responses.calls.reset() diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py index 0031ed61b5..c8d458f1be 100644 --- a/seqr/views/utils/airtable_utils.py +++ b/seqr/views/utils/airtable_utils.py @@ -171,7 +171,7 @@ def get_samples_for_sample_ids(self, sample_ids, fields): records_by_id.update(self._get_samples_for_id_field(missing, 'SeqrCollaboratorSampleID', fields)) return records_by_id - def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, additional_sample_filters=None, project_guid=None, required_sample_fields=None): + def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, sample_fields=None, additional_sample_filters=None, project_guid=None, required_sample_fields=None): pdo_fields = pdo_fields or [] additional_and_filters = [ f'LEN({{{required_sample_field}}})>0' for required_sample_field in required_sample_fields or [] @@ -182,7 +182,7 @@ def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, additional sample_records = self.fetch_records( 'Samples', fields=[ 'CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', *pdo_fields, - *(additional_sample_filters or {}).keys(), + *(sample_fields or []), *(additional_sample_filters or {}).keys(), ], or_filters={'PDOStatus': pdo_statuses}, and_filters={'SeqrProject': f'{BASE_URL}project/{project_guid}/project_page'} if project_guid else {}, From 6fada8809e366ae443dc3d4096b0b0d1add31d4f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 29 Sep 2025 14:28:57 -0400 Subject: [PATCH 71/71] bump changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 289ff71d1c..098f304084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,11 @@ # _seqr_ Changes ## dev -* Store gene ids on SavedVariant model (REQUIRES DB MIGRATION) ## 9/18/25 +* Store gene ids on SavedVariant model (REQUIRES DB MIGRATION) + +## 9/29/25 * Deprecate hail backend support ## 8/7/25