diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py index 07329222b7..0e61e7b6ab 100644 --- a/clickhouse_search/backend/functions.py +++ b/clickhouse_search/backend/functions.py @@ -131,7 +131,9 @@ def process_lhs(self, compiler, connection): def process_rhs(self, compiler, connection): rhs, rhs_params = super().process_rhs(compiler, connection) - return f'bitmapBuild{rhs.split("::")[0]}', rhs_params + cast_type = self.lhs.output_field.base_field.cast_db_type(connection) + rhs = rhs.split("::")[0].replace('%s', f'%s::{cast_type}') + return f'bitmapBuild{rhs}', rhs_params class DictGet(Func): diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json index 55afe33016..11f38a06d1 100644 --- a/clickhouse_search/fixtures/clickhouse_search.json +++ b/clickhouse_search/fixtures/clickhouse_search.json @@ -987,6 +987,7 @@ "family_guid": "F000014_14", "xpos": 1009310023, "filters": ["HIGH_SR_BACKGROUND"], + "geneId_ids": [71], "sign": 1, "calls": [ ["NA21234", 1, 2, 0, false, true, null], @@ -1003,6 +1004,7 @@ "family_guid": "F000014_14", "xpos": 1009380254, "filters": ["HIGH_SR_BACKGROUND"], + "geneId_ids": [71], "sign": 1, "calls": [ ["NA21234", 1, null, 0, true, false, null], @@ -1019,6 +1021,7 @@ "family_guid": "F000014_14", "xpos": 13063036002, "filters": ["HIGH_SR_BACKGROUND"], + "geneId_ids": [75], "sign": 1, "calls": [ ["NA21234", 1, null, 62, false, true, null], @@ -1035,6 +1038,7 @@ "family_guid": "F000014_14", "xpos": 14106694244, "filters": [], + "geneId_ids": [76], "sign": 1, "calls": [ ["NA21234", 2, 0, 99, false, true, null], @@ -1526,5 +1530,35 @@ "gencode_gene_type": "protein_coding", "gencode_release": 27 } +}, +{ + "model": "reference_data.geneinfo", + "pk": 75, + "fields": { + "gene_id": "ENSG00000083544", + "gene_symbol": "CTAGE6", + "chrom_grch38": "7", + "start_grch38": 91500851, + "end_grch38": 91525764, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } +}, +{ + "model": "reference_data.geneinfo", + "pk": 76, + "fields": { + "gene_id": "ENSG00000184986", + "gene_symbol": "CTAGE6", + "chrom_grch38": "7", + "start_grch38": 91500851, + "end_grch38": 91525764, + "strand_grch38": "+", + "coding_region_size_grch38": 0, + "gencode_gene_type": "protein_coding", + "gencode_release": 27 + } } ] \ No newline at end of file diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 199d83b2d5..e9fbf20b55 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -285,18 +285,19 @@ def _genotype_override_values(self, query, prefix=''): if not genotype_override_fields: return {} + genotype_field = next(field for field in ['genotypes', 'familyGenotypes'] if field in query.query.annotations) index_map = { - field: i+1 for i, (field, _) in enumerate(query.query.annotations['genotypes'].output_field.base_fields) + field: i+1 for i, (field, _) in enumerate(query.query.annotations[genotype_field].output_field.base_fields) } override_field_map = {field: col for col, (field, _) in genotype_override_fields.items()} genotype_fields = [ self._genotype_override_expression(index_map[field], override_field_map[field], index_map['cn']) if field in override_field_map else (f'ifNull(x.{index_map[field]}, 0)' if field == 'numAlt' else f'x.{index_map[field]}') - for (field, _) in query.query.annotations['genotypes'].output_field.base_fields + for (field, _) in query.query.annotations[genotype_field].output_field.base_fields ] return { - 'genotypes': ArrayMap('genotypes', mapped_expression=f"tuple({', '.join(genotype_fields)})"), + genotype_field: ArrayMap(genotype_field, mapped_expression=f"tuple({', '.join(genotype_fields)})"), 'transcripts': F(query.GENOTYPE_GENE_CONSEQUENCE_FIELD), **{col: F(f'sample_{col}') for col in genotype_override_fields if col != 'geneIds'}, } @@ -624,7 +625,6 @@ def has_annotation(self, field): class EntriesManager(SearchQuerySet): - MAX_XPOS_FILTER_INTERVALS = 500 GENOTYPE_LOOKUP = { REF_REF: [0], REF_ALT: [1], @@ -767,15 +767,17 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit clinvar_override_q = AnnotationsQuerySet._clinvar_path_q( pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range), ) if self._has_clinvar() else None - inheritance_q, quality_q, gt_filter, family_missing_type_samples = self._get_inheritance_quality_qs( + inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs( sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, - custom_affected=(inheritance_filter or {}).get('affected') or {}, + annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {}, ) if quality_filter.get('vcf_filter'): q = Q(filters__len=0) if clinvar_override_q: q |= clinvar_override_q entries = entries.filter(q) + if unaffected_samples: + entries = entries.annotate(carriers=self._carriers_expression(unaffected_samples)) if multi_sample_type_families: if gt_filter: @@ -798,17 +800,20 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit if quality_q is not None: entries = entries.filter(quality_q) - return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) + return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families) - def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected): + def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected): samples_by_gt = defaultdict(list) affected_samples = [] + unaffected_samples = [] family_missing_type_samples = defaultdict(lambda: defaultdict(list)) for sample in sample_data['samples']: affected = custom_affected.get(sample['individual_guid']) or sample['affected'] genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter) if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')): affected_samples.append(sample['sample_id']) + if affected == UNAFFECTED and annotate_carriers: + unaffected_samples.append(sample['sample_id']) if (inheritance_mode and inheritance_mode != ANY_AFFECTED) or individual_genotype_filter: for gt in self.genotype_lookup[genotype]: samples_by_gt[gt].append(sample['sample_id']) @@ -832,7 +837,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q) - return inheritance_q, quality_q, gt_filter, family_missing_type_samples + return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter): genotype = None @@ -893,10 +898,7 @@ def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samp ) return entries - def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None): - carriers_expression = self._carriers_expression(sample_data) if annotate_carriers else None - if carriers_expression: - entries = entries.annotate(carriers=carriers_expression) + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -921,7 +923,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an **{'genotypes' if genotype_sample_data else 'familyGenotypes': GroupArrayArray(self.genotype_expression(genotype_sample_data))}, **{col: GroupArrayArray(col) for col in genotype_override_annotations} ) - if carriers_expression: + if 'carriers' in entries.query.annotations: map_field = models.MapField(models.StringField(), models.ArrayField(models.StringField())) if multi_sample_type_families: family_carriers = ArrayFold( @@ -942,7 +944,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an if multi_sample_type_families: entries = self._multi_sample_type_filtered_entries(entries) else: - if carriers_expression: + if 'carriers' in entries.query.annotations: fields.append('carriers') if annotate_hom_alts: fields.append('has_hom_alt') @@ -983,20 +985,10 @@ def genotype_expression(self, sample_data=None): conditions=[{1: (None, 'notEmpty({field})')}] ) - def _carriers_expression(self, sample_data): - family_carriers = defaultdict(set) - for s in sample_data['samples']: - if s['affected'] == UNAFFECTED: - family_carriers[s['family_guid']].add(f"'{s['sample_id']}'") - if not any(family_carriers.values()): - return None - - carrier_map = [ - f"'{family_guid}', [{', '.join(samples)}]" for family_guid, samples in family_carriers.items() - ] + def _carriers_expression(self, unaffected_samples): return ArrayMap( ArrayFilter('calls', conditions=[{ - 'sampleId': (", ".join(carrier_map), 'has(map({value})[family_guid], {field})'), + 'sampleId': (unaffected_samples, 'has({value}, {field})'), 'gt': (0, '{field} > {value}'), }]), mapped_expression='x.sampleId', @@ -1041,7 +1033,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, variant_ids=None, padded_interval=None, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, @@ -1057,19 +1049,24 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N if exclude_intervals: intervals = None else: - chromosomes = {chrom for chrom, _, _ in list((gene_intervals or {}).values()) + (intervals or [])} + if gene_ids and 'geneIds' in self.call_fields: + entries = entries.filter(calls__array_all={'OR': [ + {'geneIds': (gene_ids, 'hasAny({value}, {field})')}, + {'gt': (None, 'isNull({field})')}, + ]}) + intervals = list((gene_intervals or {}).values()) + (intervals or []) + gene_intervals = None + chromosomes = {chrom for chrom, _, _ in (intervals or [])} intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] - gene_intervals = None if not (gene_intervals or intervals): return entries locus_q = None if gene_intervals: - has_entry_genes = hasattr(self.model, 'is_annotated_in_any_gene') - if has_entry_genes and not intervals: + if hasattr(self.model, 'is_annotated_in_any_gene') and not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) - if (not has_entry_genes) or exclude_intervals or len(gene_intervals) < self.MAX_XPOS_FILTER_INTERVALS: + if (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS: intervals = list((gene_intervals or {}).values()) + (intervals or []) else: locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index b9dc348dd3..0898f60849 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -88,6 +88,7 @@ class Migration(migrations.Migration): ('project_guid', clickhouse_backend.models.StringField(low_cardinality=True)), ('family_guid', clickhouse_backend.models.StringField()), ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()), + ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())), ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))), ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('cn', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('newCall', clickhouse_backend.models.BoolField(blank=True, null=True)), ('prevCall', clickhouse_backend.models.BoolField(blank=True, null=True)), ('prevNumAlt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True))]))), ('sign', clickhouse_backend.models.Int8Field()), diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index c7e25c23d3..3e9d0688f9 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -558,6 +558,8 @@ class Meta(BaseClinvarJoin.Meta): class BaseEntries(FixtureLoadableClickhouseModel): + MAX_XPOS_FILTER_INTERVALS = 500 + project_guid = models.StringField(low_cardinality=True) family_guid = models.StringField() xpos = UInt64FieldDeltaCodecField() @@ -645,6 +647,7 @@ class Meta(BaseEntries.Meta): ) class EntriesSv(BaseEntries): + MAX_XPOS_FILTER_INTERVALS = 0 SAMPLE_TYPE = Sample.SAMPLE_TYPE_WGS CALL_FIELDS = [ ('sampleId', models.StringField()), @@ -659,6 +662,7 @@ class EntriesSv(BaseEntries): # primary_key is not enforced by clickhouse, but setting it here prevents django adding an id column key = ForeignKey('AnnotationsSv', db_column='key', primary_key=True, on_delete=CASCADE) calls = models.ArrayField(NamedTupleField(CALL_FIELDS)) + geneId_ids = models.ArrayField(models.UInt32Field()) class Meta(BaseEntries.Meta): db_table = 'GRCh38/SV/entries' diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f1e82f99b8..6f42e1aa2a 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -45,8 +45,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type] family_guid = sample_data['family_guids'][0] - is_multi_project = len(sample_data['project_guids']) > 1 - skip_individual_guid = is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS + skip_individual_guid = len(sample_data['project_guids']) > 1 dataset_results = [] if inheritance_mode != COMPOUND_HET: @@ -63,7 +62,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any( dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type ): - results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, **search) + results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, **search) cache_results = get_clickhouse_cache_results(results, sort, family_guid) previous_search_results.update(cache_results) @@ -86,7 +85,7 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, @@ -107,6 +106,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b families = snv_indel_families.intersection(sv_families) if not families: continue + logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) entries = entry_cls.objects.search({ **snv_indel_sample_data, diff --git a/ui/shared/components/panel/variants/FamilyVariantTags.jsx b/ui/shared/components/panel/variants/FamilyVariantTags.jsx index 9adede7048..8ea1516f94 100644 --- a/ui/shared/components/panel/variants/FamilyVariantTags.jsx +++ b/ui/shared/components/panel/variants/FamilyVariantTags.jsx @@ -221,12 +221,12 @@ const MatchmakerLabel = ({ variant, family, mmeSubmissionsByGuid, genesById }) = const variantSubmissions = (Array.isArray(variant) ? variant.reduce( (acc, { mmeSubmissions = [] }) => ([...acc, ...mmeSubmissions]), [], ) : (variant.mmeSubmissions || [])).map( - ({ submissionGuid, geneId }) => ({ gene: genesById[geneId], submission: mmeSubmissionsByGuid[submissionGuid] }), + ({ submissionGuid, geneId }) => ({ gene: genesById[geneId] || { geneId }, submission: mmeSubmissionsByGuid[submissionGuid] }), ).filter(({ submission }) => family.individualGuids.includes(submission.individualGuid)) return variantSubmissions.length ? ( `${gene.geneSymbol} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`, + ({ gene, submission }) => `${gene.geneSymbol || gene.geneId} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`, ))].join('; ')} trigger={