From 294675217a71812a77ece466f38297acdd172ad2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 Aug 2025 12:56:35 -0400 Subject: [PATCH 01/10] post process individual guids for all data types; --- clickhouse_search/managers.py | 2 +- clickhouse_search/search.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 199d83b2d5..3bb52cfc5e 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -282,7 +282,7 @@ def _conditional_selected_transcript_values(self, query, prefix=''): def _genotype_override_values(self, query, prefix=''): genotype_override_fields = query.model.GENOTYPE_OVERRIDE_FIELDS - if not genotype_override_fields: + if not (genotype_override_fields and query.query.annotations.get('genotypes')): return {} index_map = { diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f1e82f99b8..6f42e1aa2a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -45,8 +45,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] family_guid = sample_data['family_guids'][0] - is_multi_project = len(sample_data['project_guids']) > 1 - skip_individual_guid = is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS + skip_individual_guid = len(sample_data['project_guids']) > 1 dataset_results = [] if inheritance_mode != COMPOUND_HET: @@ -63,7 +62,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, **search) + results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, **search) cache_results = get_clickhouse_cache_results(results, sort, family_guid) previous_search_results.update(cache_results) @@ -86,7 +85,7 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -107,6 +106,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b families = snv_indel_families.intersection(sv_families) if not families: continue + logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) entries = entry_cls.objects.search({ **snv_indel_sample_data, From f66939fe026162be22e180309c668e407dfb2586 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 Aug 2025 13:34:42 -0400 Subject: [PATCH 02/10] remove unneccessary map from carrier expression --- clickhouse_search/managers.py | 38 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 3bb52cfc5e..1778538323 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -767,15 +767,17 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None - inheritance_q, quality_q, gt_filter, family_missing_type_samples = self._get_inheritance_quality_qs( + inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, - custom_affected=(inheritance_filter or {}).get('affected') or {}, + annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {}, ) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) if clinvar_override_q: q |= clinvar_override_q entries = entries.filter(q) + if unaffected_samples: + entries = entries.annotate(carriers=self._carriers_expression(unaffected_samples)) if multi_sample_type_families: if gt_filter: @@ -798,17 +800,20 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if quality_q is not None: entries = entries.filter(quality_q) - return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) + return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) - def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): + def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected): samples_by_gt = defaultdict(list) affected_samples = [] + unaffected_samples = [] family_missing_type_samples = defaultdict(lambda: defaultdict(list)) for sample in sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')): affected_samples.append(sample['sample_id']) + if affected == UNAFFECTED and annotate_carriers: + unaffected_samples.append(sample['sample_id']) if (inheritance_mode and inheritance_mode != ANY_AFFECTED) or individual_genotype_filter: for gt in self.genotype_lookup[genotype]: samples_by_gt[gt].append(sample['sample_id']) @@ -832,7 +837,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) - return inheritance_q, quality_q, gt_filter, family_missing_type_samples + return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None @@ -893,10 +898,7 @@ def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samp ) return entries - def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None): - carriers_expression = self._carriers_expression(sample_data) if annotate_carriers else None - if carriers_expression: - entries = entries.annotate(carriers=carriers_expression) + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -921,7 +923,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an **{'genotypes' if genotype_sample_data else 'familyGenotypes': GroupArrayArray(self.genotype_expression(genotype_sample_data))}, **{col: GroupArrayArray(col) for col in genotype_override_annotations} ) - if carriers_expression: + if 'carriers' in entries.query.annotations: map_field = models.MapField(models.StringField(), models.ArrayField(models.StringField())) if multi_sample_type_families: family_carriers = ArrayFold( @@ -942,7 +944,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an if multi_sample_type_families: entries = self._multi_sample_type_filtered_entries(entries) else: - if carriers_expression: + if 'carriers' in entries.query.annotations: fields.append('carriers') if annotate_hom_alts: fields.append('has_hom_alt') @@ -983,20 +985,10 @@ def genotype_expression(self, sample_data=None): conditions=[{1: (None, 'notEmpty({field})')}] ) - def _carriers_expression(self, sample_data): - family_carriers = defaultdict(set) - for s in sample_data['samples']: - if s['affected'] == UNAFFECTED: - family_carriers[s['family_guid']].add(f"'{s['sample_id']}'") - if not any(family_carriers.values()): - return None - - carrier_map = [ - f"'{family_guid}', [{', '.join(samples)}]" for family_guid, samples in family_carriers.items() - ] + def _carriers_expression(self, unaffected_samples): return ArrayMap( ArrayFilter('calls', conditions=[{ - 'sampleId': (", ".join(carrier_map), 'has(map({value})[family_guid], {field})'), + 'sampleId': (unaffected_samples, 'has({value}, {field})'), 'gt': (0, '{field} > {value}'), }]), mapped_expression='x.sampleId', From 8618aff133bf3d2582c5f6ef94232afbaac08477 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 Aug 2025 14:14:39 -0400 Subject: [PATCH 03/10] filter gcnv by gene id --- clickhouse_search/managers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1778538323..48940b9cb4 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1033,7 +1033,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, variant_ids=None, padded_interval=None, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, @@ -1049,6 +1049,10 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N if exclude_intervals: intervals = None else: + if gene_ids and 'geneIds' in self.call_fields: + entries = entries.filter(calls__array_all={ + 'geneIds': (gene_ids, 'or(isNull({field}), hasAny({value}, {field}))'), + }) chromosomes = {chrom for chrom, _, _ in list((gene_intervals or {}).values()) + (intervals or [])} intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] gene_intervals = None From 1845fe37647ffbde9a4a3037069c23d89e810299 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 Aug 2025 15:38:38 -0400 Subject: [PATCH 04/10] fix mme submission with inavlid gene --- ui/shared/components/panel/variants/FamilyVariantTags.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/shared/components/panel/variants/FamilyVariantTags.jsx b/ui/shared/components/panel/variants/FamilyVariantTags.jsx index 9adede7048..8ea1516f94 100644 --- a/ui/shared/components/panel/variants/FamilyVariantTags.jsx +++ b/ui/shared/components/panel/variants/FamilyVariantTags.jsx @@ -221,12 +221,12 @@ const MatchmakerLabel = ({ variant, family, mmeSubmissionsByGuid, genesById }) = const variantSubmissions = (Array.isArray(variant) ? variant.reduce( (acc, { mmeSubmissions = [] }) => ([...acc, ...mmeSubmissions]), [], ) : (variant.mmeSubmissions || [])).map( - ({ submissionGuid, geneId }) => ({ gene: genesById[geneId], submission: mmeSubmissionsByGuid[submissionGuid] }), + ({ submissionGuid, geneId }) => ({ gene: genesById[geneId] || { geneId }, submission: mmeSubmissionsByGuid[submissionGuid] }), ).filter(({ submission }) => family.individualGuids.includes(submission.individualGuid)) return variantSubmissions.length ? ( `${gene.geneSymbol} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`, + ({ gene, submission }) => `${gene.geneSymbol || gene.geneId} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`, ))].join('; ')} trigger={