Dev

clickhouse_search/backend/functions.py

@@ -131,7 +131,9 @@ def process_lhs(self, compiler, connection):
 
     def process_rhs(self, compiler, connection):
         rhs, rhs_params = super().process_rhs(compiler, connection)
-        return f'bitmapBuild{rhs.split("::")[0]}', rhs_params
+        cast_type = self.lhs.output_field.base_field.cast_db_type(connection)
+        rhs = rhs.split("::")[0].replace('%s', f'%s::{cast_type}')
+        return f'bitmapBuild{rhs}', rhs_params
 
 
 class DictGet(Func):

clickhouse_search/fixtures/clickhouse_search.json

@@ -987,6 +987,7 @@
       "family_guid": "F000014_14",
       "xpos": 1009310023,
       "filters": ["HIGH_SR_BACKGROUND"],
+      "geneId_ids": [71],
       "sign": 1,
       "calls": [
         ["NA21234", 1, 2, 0, false, true, null],
@@ -1003,6 +1004,7 @@
       "family_guid": "F000014_14",
       "xpos": 1009380254,
       "filters": ["HIGH_SR_BACKGROUND"],
+      "geneId_ids": [71],
       "sign": 1,
       "calls": [
         ["NA21234", 1, null, 0, true, false, null],
@@ -1019,6 +1021,7 @@
       "family_guid": "F000014_14",
       "xpos": 13063036002,
       "filters": ["HIGH_SR_BACKGROUND"],
+      "geneId_ids": [75],
       "sign": 1,
       "calls": [
         ["NA21234", 1, null, 62, false, true, null],
@@ -1035,6 +1038,7 @@
       "family_guid": "F000014_14",
       "xpos": 14106694244,
       "filters": [],
+      "geneId_ids": [76],
       "sign": 1,
       "calls": [
         ["NA21234", 2, 0, 99, false, true, null],
@@ -1526,5 +1530,35 @@
         "gencode_gene_type": "protein_coding",
         "gencode_release": 27
     }
+},
+{
+    "model": "reference_data.geneinfo",
+    "pk": 75,
+    "fields": {
+        "gene_id": "ENSG00000083544",
+        "gene_symbol": "CTAGE6",
+        "chrom_grch38": "7",
+        "start_grch38": 91500851,
+        "end_grch38": 91525764,
+        "strand_grch38": "+",
+        "coding_region_size_grch38": 0,
+        "gencode_gene_type": "protein_coding",
+        "gencode_release": 27
+    }
+},
+{
+    "model": "reference_data.geneinfo",
+    "pk": 76,
+    "fields": {
+        "gene_id": "ENSG00000184986",
+        "gene_symbol": "CTAGE6",
+        "chrom_grch38": "7",
+        "start_grch38": 91500851,
+        "end_grch38": 91525764,
+        "strand_grch38": "+",
+        "coding_region_size_grch38": 0,
+        "gencode_gene_type": "protein_coding",
+        "gencode_release": 27
+    }
 }
 ]

clickhouse_search/managers.py

@@ -285,18 +285,19 @@ def _genotype_override_values(self, query, prefix=''):
         if not genotype_override_fields:
             return {}
 
+        genotype_field = next(field for field in ['genotypes', 'familyGenotypes'] if field in query.query.annotations)
         index_map = {
-            field: i+1 for i, (field, _) in enumerate(query.query.annotations['genotypes'].output_field.base_fields)
+            field: i+1 for i, (field, _) in enumerate(query.query.annotations[genotype_field].output_field.base_fields)
         }
         override_field_map = {field: col for col, (field, _) in genotype_override_fields.items()}
         genotype_fields = [
             self._genotype_override_expression(index_map[field], override_field_map[field], index_map['cn'])
             if field in override_field_map else (f'ifNull(x.{index_map[field]}, 0)' if field == 'numAlt' else f'x.{index_map[field]}')
-            for (field, _) in query.query.annotations['genotypes'].output_field.base_fields
+            for (field, _) in query.query.annotations[genotype_field].output_field.base_fields
         ]
 
         return {
-            'genotypes': ArrayMap('genotypes', mapped_expression=f"tuple({', '.join(genotype_fields)})"),
+            genotype_field: ArrayMap(genotype_field, mapped_expression=f"tuple({', '.join(genotype_fields)})"),
             'transcripts': F(query.GENOTYPE_GENE_CONSEQUENCE_FIELD),
             **{col: F(f'sample_{col}') for col in genotype_override_fields if col != 'geneIds'},
         }
@@ -624,7 +625,6 @@ def has_annotation(self, field):
 
 
 class EntriesManager(SearchQuerySet):
-    MAX_XPOS_FILTER_INTERVALS = 500
     GENOTYPE_LOOKUP = {
         REF_REF: [0],
         REF_ALT: [1],
@@ -767,15 +767,17 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
             clinvar_override_q = AnnotationsQuerySet._clinvar_path_q(
                pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range),
             ) if self._has_clinvar() else None
-            inheritance_q, quality_q, gt_filter, family_missing_type_samples = self._get_inheritance_quality_qs(
+            inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs(
                sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q,
-                custom_affected=(inheritance_filter or {}).get('affected') or {},
+                annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {},
             )
             if quality_filter.get('vcf_filter'):
                 q = Q(filters__len=0)
                 if clinvar_override_q:
                     q |= clinvar_override_q
                 entries = entries.filter(q)
+            if unaffected_samples:
+                entries = entries.annotate(carriers=self._carriers_expression(unaffected_samples))
 
        if multi_sample_type_families:
            if gt_filter:
@@ -798,17 +800,20 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
        if quality_q is not None:
            entries = entries.filter(quality_q)
 
-       return self._annotate_calls(entries, sample_data, annotate_carriers, annotate_hom_alts, skip_individual_guid, multi_sample_type_families)
+       return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families)
 
-    def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, custom_affected):
+    def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected):
         samples_by_gt = defaultdict(list)
         affected_samples = []
+        unaffected_samples = []
         family_missing_type_samples = defaultdict(lambda: defaultdict(list))
         for sample in sample_data['samples']:
             affected = custom_affected.get(sample['individual_guid']) or sample['affected']
             genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter)
             if affected == AFFECTED and (inheritance_mode == ANY_AFFECTED or quality_filter.get('affected_only')):
                 affected_samples.append(sample['sample_id'])
+            if affected == UNAFFECTED and annotate_carriers:
+                unaffected_samples.append(sample['sample_id'])
             if (inheritance_mode and inheritance_mode != ANY_AFFECTED) or individual_genotype_filter:
                 for gt in self.genotype_lookup[genotype]:
                     samples_by_gt[gt].append(sample['sample_id'])
@@ -832,7 +837,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i
 
         quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q)
 
-        return inheritance_q, quality_q, gt_filter, family_missing_type_samples
+        return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples
 
     def _sample_genotype(self, sample, affected, inheritance_mode, individual_genotype_filter):
         genotype = None
@@ -893,10 +898,7 @@ def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samp
             )
         return entries
 
-    def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None):
-        carriers_expression = self._carriers_expression(sample_data) if annotate_carriers else None
-        if carriers_expression:
-            entries = entries.annotate(carriers=carriers_expression)
+    def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, skip_individual_guid=False, multi_sample_type_families=None):
         if annotate_hom_alts:
             entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)}))
 
@@ -921,7 +923,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an
                 **{'genotypes' if genotype_sample_data else 'familyGenotypes': GroupArrayArray(self.genotype_expression(genotype_sample_data))},
                 **{col: GroupArrayArray(col) for col in genotype_override_annotations}
             )
-            if carriers_expression:
+            if 'carriers' in entries.query.annotations:
                 map_field = models.MapField(models.StringField(), models.ArrayField(models.StringField()))
                 if multi_sample_type_families:
                     family_carriers = ArrayFold(
@@ -942,7 +944,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_carriers=False, an
             if multi_sample_type_families:
                 entries = self._multi_sample_type_filtered_entries(entries)
         else:
-            if carriers_expression:
+            if 'carriers' in entries.query.annotations:
                 fields.append('carriers')
             if annotate_hom_alts:
                 fields.append('has_hom_alt')
@@ -983,20 +985,10 @@ def genotype_expression(self, sample_data=None):
             conditions=[{1: (None, 'notEmpty({field})')}]
         )
 
-    def _carriers_expression(self, sample_data):
-        family_carriers = defaultdict(set)
-        for s in sample_data['samples']:
-            if s['affected'] == UNAFFECTED:
-                family_carriers[s['family_guid']].add(f"'{s['sample_id']}'")
-        if not any(family_carriers.values()):
-            return None
-
-        carrier_map = [
-            f"'{family_guid}', [{', '.join(samples)}]" for family_guid, samples in family_carriers.items()
-        ]
+    def _carriers_expression(self, unaffected_samples):
         return ArrayMap(
             ArrayFilter('calls', conditions=[{
-                'sampleId': (", ".join(carrier_map), 'has(map({value})[family_guid], {field})'),
+                'sampleId': (unaffected_samples, 'has({value}, {field})'),
                 'gt': (0, '{field} > {value}'),
             }]),
             mapped_expression='x.sampleId',
@@ -1041,7 +1033,7 @@ def _family_passes_expression(pass_field):
             mapped_expression='x.1', output_field=models.ArrayField(models.StringField()),
         )
 
-    def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, variant_ids=None, padded_interval=None, **kwargs):
+    def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, **kwargs):
         entries = self
         if variant_ids:
             # although technically redundant, the interval query is applied to the entries table before join and reduces the join size,
@@ -1057,19 +1049,24 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N
             if exclude_intervals:
                 intervals = None
             else:
-                chromosomes = {chrom for chrom, _, _ in list((gene_intervals or {}).values()) + (intervals or [])}
+                if gene_ids and 'geneIds' in self.call_fields:
+                    entries = entries.filter(calls__array_all={'OR': [
+                        {'geneIds': (gene_ids, 'hasAny({value}, {field})')},
+                        {'gt': (None, 'isNull({field})')},
+                    ]})
+                    intervals = list((gene_intervals or {}).values()) + (intervals or [])
+                    gene_intervals = None
+                chromosomes = {chrom for chrom, _, _ in (intervals or [])}
                 intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes]
-            gene_intervals = None
 
         if not (gene_intervals or intervals):
             return entries
 
         locus_q = None
         if gene_intervals:
-            has_entry_genes = hasattr(self.model, 'is_annotated_in_any_gene')
-            if has_entry_genes and not intervals:
+            if hasattr(self.model, 'is_annotated_in_any_gene') and not intervals:
                 entries = entries.filter(is_annotated_in_any_gene=Value(True))
-            if (not has_entry_genes) or exclude_intervals or len(gene_intervals) < self.MAX_XPOS_FILTER_INTERVALS:
+            if (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS:
                 intervals = list((gene_intervals or {}).values()) + (intervals or [])
             else:
                 locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys()))

clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py

@@ -88,6 +88,7 @@ class Migration(migrations.Migration):
                 ('project_guid', clickhouse_backend.models.StringField(low_cardinality=True)),
                 ('family_guid', clickhouse_backend.models.StringField()),
                 ('xpos', clickhouse_search.backend.fields.UInt64FieldDeltaCodecField()),
+                ('geneId_ids', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.UInt32Field())),
                 ('filters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(low_cardinality=True))),
                 ('calls', clickhouse_backend.models.ArrayField(base_field=clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sampleId', clickhouse_backend.models.StringField()), ('gt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True)), ('cn', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('gq', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('newCall', clickhouse_backend.models.BoolField(blank=True, null=True)), ('prevCall', clickhouse_backend.models.BoolField(blank=True, null=True)), ('prevNumAlt', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')], null=True))]))),
                 ('sign', clickhouse_backend.models.Int8Field()),

clickhouse_search/models.py

@@ -558,6 +558,8 @@ class Meta(BaseClinvarJoin.Meta):
 
 
 class BaseEntries(FixtureLoadableClickhouseModel):
+    MAX_XPOS_FILTER_INTERVALS = 500
+
     project_guid = models.StringField(low_cardinality=True)
     family_guid = models.StringField()
     xpos = UInt64FieldDeltaCodecField()
@@ -645,6 +647,7 @@ class Meta(BaseEntries.Meta):
         )
 
 class EntriesSv(BaseEntries):
+    MAX_XPOS_FILTER_INTERVALS = 0
     SAMPLE_TYPE = Sample.SAMPLE_TYPE_WGS
     CALL_FIELDS = [
         ('sampleId', models.StringField()),
@@ -659,6 +662,7 @@ class EntriesSv(BaseEntries):
     # primary_key is not enforced by clickhouse, but setting it here prevents django adding an id column
     key = ForeignKey('AnnotationsSv', db_column='key', primary_key=True, on_delete=CASCADE)
     calls = models.ArrayField(NamedTupleField(CALL_FIELDS))
+    geneId_ids = models.ArrayField(models.UInt32Field())
 
     class Meta(BaseEntries.Meta):
         db_table = 'GRCh38/SV/entries'

clickhouse_search/search.py

@@ -45,8 +45,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno
         entry_cls = ENTRY_CLASS_MAP[genome_version][dataset_type]
         annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][dataset_type]
         family_guid = sample_data['family_guids'][0]
-        is_multi_project = len(sample_data['project_guids']) > 1
-        skip_individual_guid = is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS
+        skip_individual_guid = len(sample_data['project_guids']) > 1
 
         dataset_results = []
         if inheritance_mode != COMPOUND_HET:
@@ -63,7 +62,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno
     if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any(
         dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type
     ):
-        results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, **search)
+        results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, **search)
 
     cache_results = get_clickhouse_cache_results(results, sort, family_guid)
     previous_search_results.update(cache_results)
@@ -86,7 +85,7 @@ def _evaluate_results(result_q, is_comp_het=False):
         raise InvalidSearchException('This search returned too many results')
     return results
 
-def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs):
+def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs):
     if annotations_secondary:
         annotations = {
             **annotations,
@@ -107,6 +106,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b
         families = snv_indel_families.intersection(sv_families)
         if not families:
             continue
+        logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user)
 
         entries = entry_cls.objects.search({
             **snv_indel_sample_data,

ui/shared/components/panel/variants/FamilyVariantTags.jsx

@@ -221,12 +221,12 @@ const MatchmakerLabel = ({ variant, family, mmeSubmissionsByGuid, genesById }) =
   const variantSubmissions = (Array.isArray(variant) ? variant.reduce(
     (acc, { mmeSubmissions = [] }) => ([...acc, ...mmeSubmissions]), [],
   ) : (variant.mmeSubmissions || [])).map(
-    ({ submissionGuid, geneId }) => ({ gene: genesById[geneId], submission: mmeSubmissionsByGuid[submissionGuid] }),
+    ({ submissionGuid, geneId }) => ({ gene: genesById[geneId] || { geneId }, submission: mmeSubmissionsByGuid[submissionGuid] }),
   ).filter(({ submission }) => family.individualGuids.includes(submission.individualGuid))
   return variantSubmissions.length ? (
     <Popup
       content={[...new Set(variantSubmissions.map(
-        ({ gene, submission }) => `${gene.geneSymbol} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`,
+        ({ gene, submission }) => `${gene.geneSymbol || gene.geneId} submitted ${new Date(submission.lastModifiedDate).toLocaleDateString()}`,
       ))].join('; ')}
       trigger={<Label
         as={NavLink}