Dev

clickhouse_search/backend/functions.py

@@ -4,6 +4,7 @@
 
 from clickhouse_search.backend.fields import NestedField
 
+
 class Array(Func):
     function = 'array'
 
@@ -124,9 +125,9 @@ def as_sql(self, compiler, connection, *args, **kwargs):
 
 
 class ArrayIndex(IndexTransform):
-    def __init__(self, index, *args, **kwargs):
+    def __init__(self, index, *args, base_field=None, **kwargs):
         super().__init__(index+1, None, *args, **kwargs)
-        self.base_field = self.source_expressions[0].output_field.base_field
+        self.base_field = base_field or self.source_expressions[0].output_field.base_field
 
 
 @ArrayField.register_lookup

seqr/utils/search/constants.py → clickhouse_search/constants.py

@@ -1,10 +1,6 @@
 from seqr.models import Individual
 
-VCF_FILE_EXTENSIONS = ('.vcf', '.vcf.gz', '.vcf.bgz')
-
 MAX_VARIANTS = 10000
-MAX_EXPORT_VARIANTS = 1000
-MAX_NO_LOCATION_COMP_HET_FAMILIES = 100
 
 XPOS_SORT_KEY = 'xpos'
 PATHOGENICTY_SORT_KEY = 'pathogenicity'
@@ -65,7 +61,6 @@
 SV_CONSEQUENCES_FIELD = 'structural_consequence'
 SV_TYPE_FILTER_FIELD = 'structural'
 SV_ANNOTATION_TYPES = {SV_CONSEQUENCES_FIELD, SV_TYPE_FILTER_FIELD, NEW_SV_FIELD}
-ALL_DATA_TYPES = 'ALL'
 
 CLINVAR_KEY = 'clinvar'
 CLINVAR_PATH_FILTER = 'pathogenic'

clickhouse_search/fixtures/clickhouse_saved_variants.json

@@ -94,6 +94,30 @@
         [1, ["frameshift_variant"], "ENSG00000228198"]
       ]
     }
+}, {
+    "model": "clickhouse_search.variantdetailsgrch37snvindel",
+    "pk": 113,
+    "fields": {
+      "key": 113,
+      "variant_id": "15-62456358-A-C",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "transcripts": []
+    }
+}, {
+    "model": "clickhouse_search.variantdetailsgrch37snvindel",
+    "pk": 114,
+    "fields": {
+      "key": 114,
+      "variant_id": "15-62456406-G-A",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "transcripts": []
+    }
 }, {
     "model": "clickhouse_search.entriesgrch37snvindel",
     "pk": 101,

clickhouse_search/fixtures/clickhouse_search.json

@@ -88,6 +88,18 @@
       "sorted_motif_feature_consequences": [],
       "sorted_regulatory_feature_consequences": []
     }
+}, {
+    "model": "clickhouse_search.variantssnvindel",
+    "pk": 23,
+    "fields": {
+      "key": 23,
+      "sorted_transcript_consequences": [
+        [null, 1, ["missense_variant"], false, null, "ENSG00000227232", false],
+        [null, 1, ["intron_variant"], false, null, "ENSG00000268903", false]
+      ],
+      "sorted_motif_feature_consequences": [],
+      "sorted_regulatory_feature_consequences": []
+    }
 }, {
     "model": "clickhouse_search.keylookupsnvindel",
     "pk": "1-10439-AC-A",
@@ -125,6 +137,12 @@
       "key": 22
     }
 }, {
+    "model": "clickhouse_search.keylookupsnvindel",
+    "pk": "21-3343353-GAGA-G",
+    "fields": {
+      "key": 23
+    }
+},{
     "model": "clickhouse_search.clinvarallvariantssnvindel",
     "pk": "1-10439-AC-A",
     "fields": {
@@ -860,6 +878,24 @@
         ["NA21654", 0, 0, 0, 24]
       ]
     }
+}, {
+    "model": "clickhouse_search.entriessnvindel",
+    "pk": 23,
+    "fields": {
+      "key": 23,
+      "project_guid": "R0001_1kg",
+      "family_guid": "F000001_1",
+      "sample_type": "WES",
+      "xpos": 21003343353,
+      "is_gnomad_gt_5_percent": false,
+      "is_annotated_in_any_gene": true,
+      "geneId_ids": [2],
+      "filters": [],
+      "sign": 1,
+      "calls": [
+        ["NA19678", 1, 0, 0.16, 45]
+      ]
+    }
 }, {
     "model": "clickhouse_search.variantsgrch37snvindel",
     "pk": 10,
@@ -1418,9 +1454,9 @@
       "geneId_ids": [76],
       "sign": 1,
       "calls": [
-        ["NA21234", 2, 0, 99, false, true, null],
         ["NA21987", 1, 1, 99, false, false, 2],
-        ["NA21654", 1, 1, 99, false, false, 2]
+        ["NA21654", 1, 1, 99, false, false, 2],
+        ["NA21234", 2, 0, 99, false, true, null]
       ]
     }
 }, {
@@ -1589,9 +1625,9 @@
       "filters": [],
       "sign": 1,
       "calls": [
-        ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false],
+        ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false],
         ["HG00732", null, null, null, null, null, null, null, [], null, null, null],
-        ["HG00733", 1, 3, 29, false, 38721781, 38734440, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false]
+        ["HG00731", 1, 3, 28, false, 38721781, 38735703, 7, ["ENSG00000275023", "ENSG00000277258", "ENSG00000277972"], false, true, false]
       ]
     }
 }, {

clickhouse_search/fixtures/clickhouse_transcripts.json

@@ -235,5 +235,48 @@
         ]
       ]
     }
+}, {
+    "model": "clickhouse_search.variantdetailssnvindel",
+    "pk": 23,
+    "fields": {
+      "key": 23,
+      "variant_id": "21-3343353-GAGA-G",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "sorted_motif_feature_consequences": [],
+      "sorted_regulatory_feature_consequences": [],
+      "transcripts": [
+        [
+          [null], "T/I", "protein_coding", 1, "aCc/aTc", ["missense_variant"], [1, 12], "ENSG00000227232",
+          "ENST00000257261.10:c.131C>T", "ENSP00000257261.6:p.Thr44Ile", [null, null], [null, []], "missense_variant",
+          null, null, "NM_001281501.1", [false], "ENST00000257261", 0, [null, null, null, [
+            null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null
+          ], null]
+        ],
+        [
+          [null], null, "protein_coding", null, null, ["intron_variant"], [null, null], "ENSG00000268903",
+          "ENST00000426137.1:c.-64+100G>A", null, [1, 5], [null, []], "intron_variant", null, null, null, [false],
+          "ENST00000426137", 0, [null, null, null, [
+            null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null
+          ], null]
+        ]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.variantdetailssnvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "variant_id": "1-248367227-TC-T",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "sorted_motif_feature_consequences": [],
+      "sorted_regulatory_feature_consequences": [],
+      "transcripts": []
+    }
 }
 ]

clickhouse_search/management/commands/reload_clinvar_all_variants.py

@@ -99,7 +99,7 @@ def parse_positions(classified_record_node: xml.etree.ElementTree.Element) -> di
                 }
     return positions
 
-def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element) -> [str, list[str]]:
+def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.ElementTree.Element, allele_id: int, unenumerated_value_alerts: list) -> [str, list[str]]:
     pathogenicity_node = classified_record_node.find(
         'Classifications/GermlineClassification/Description',
     )
@@ -133,15 +133,23 @@ def parse_pathogenicity_and_assertions(classified_record_node: xml.etree.Element
         assertions = [a.strip() for a in pathogenicity_string.split(';')]
 
     enumerated_assertions = set(CLINVAR_ASSERTIONS)
+    filtered_assertions = []
     for assertion in assertions:
         if assertion not in enumerated_assertions:
-            raise CommandError(f'Found an un-enumerated clinvar assertion: {assertion}')
-
-    return pathogenicity, assertions
+            unenumerated_value_alerts.append({
+                'type': 'Assertion',
+                'allele_id': allele_id,
+                'value': assertion,
+            })
+        else:
+            filtered_assertions.append(assertion)
+    return pathogenicity, filtered_assertions
 
 def parse_conflicting_pathogenicities(
     classified_record_node: xml.etree.ElementTree.Element,
     pathogenicity: str,
+    allele_id: int,
+    unenumerated_value_alerts: list,
 ) -> list[[str, int]]:
     if pathogenicity == CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY:
         conflicting_pathogenicities_node = classified_record_node.find(
@@ -159,10 +167,17 @@ def parse_conflicting_pathogenicities(
         conflicting_pathogenicities_node.text
     )
     enumerated_pathogenicities = set(CLINVAR_PATHOGENICITIES)
-    for (pathogenicity, _) in conflicting_pathogenicities:
+    filtered_pathogenicities = []
+    for (pathogenicity, count) in conflicting_pathogenicities:
         if pathogenicity not in enumerated_pathogenicities:
-            raise CommandError(f'Found an un-enumerated conflicting pathogenicity: {pathogenicity}')
-    return conflicting_pathogenicities
+            unenumerated_value_alerts.append({
+                'type': 'Conflicting Pathogenicity',
+                'allele_id': allele_id,
+                'value': pathogenicity,
+            })
+        else:
+            filtered_pathogenicities.append((pathogenicity, count))
+    return filtered_pathogenicities
 
 def parse_gold_stars(classified_record_node: xml) -> Optional[int]:
     review_status_node = classified_record_node.find(
@@ -189,7 +204,7 @@ def parse_submitters_and_conditions(classified_record_node: xml) -> [list[str],
     })
     return submitters, conditions
 
-def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]:
+def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, unenumerated_value_alerts: Optional[list] = None) -> tuple[models.ClickhouseModel, models.ClickhouseModel, models.ClickhouseModel]:
     # Cannot use regular bool-falseyness here, as:
     # "An element with no child elements (even if it exists and has text) will be falsey."
     classified_record_node = elem.find('ClassifiedRecord')
@@ -202,8 +217,8 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str)
     if not positions:
         return None, None, None
 
-    pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node)
-    conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity)
+    pathogenicity, assertions = parse_pathogenicity_and_assertions(classified_record_node, allele_id, unenumerated_value_alerts)
+    conflicting_pathogenicities = parse_conflicting_pathogenicities(classified_record_node, pathogenicity, allele_id, unenumerated_value_alerts)
     gold_stars = parse_gold_stars(classified_record_node)
     submitters, conditions = parse_submitters_and_conditions(classified_record_node)
     # Note: this manipulation to an enumerated pathogenicty happens after we parse conflicting pathogenicities.
@@ -247,6 +262,7 @@ def handle(self, *args, **options):
             ClinvarAllVariantsSnvIndel: [],
             ClinvarAllVariantsMito: [],
         }
+        unenumerated_value_alerts = []
         new_version = None
         with tempfile.TemporaryDirectory() as tmpdir:
             with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \
@@ -276,7 +292,7 @@ def handle(self, *args, **options):
                                     )
                     # Handle parsing variants
                     if event == 'end' and elem.tag == 'VariationArchive' and new_version:
-                        for obj in extract_variant_info(elem, new_version):
+                        for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts):
                             for model, batch in model_to_batch.items():
                                 if isinstance(obj, model):
                                     batch.append(obj)
@@ -316,4 +332,10 @@ def handle(self, *args, **options):
                 version=new_version
             )
         slack_message = f'Successfully updated Clinvar ClickHouse tables to {new_version}.'
+        if unenumerated_value_alerts:
+            slack_message += f'\n\nFound {len(unenumerated_value_alerts)} unenumerated value(s) during parsing:'
+            for alert in unenumerated_value_alerts[:10]:
+                slack_message += f"\n- {alert['type']}: '{alert['value']}' (Allele ID: {alert['allele_id']})"
+            if len(unenumerated_value_alerts) > 10:
+                slack_message += f"\n• ... and {len(unenumerated_value_alerts) - 10} more"
         safe_post_to_slack(SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, slack_message)