Dev

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## dev
 
+## 8/7/25
+* Deprecate support for lifting existing projects from GRCH37 to GRCh38
+
 ## 4/4/25
 * Refactor manage commands to update reference data. May require changes to custom cron jobs. (REQUIRES DB MIGRATION)
 

clickhouse_search/backend/fields.py

@@ -50,7 +50,8 @@ def _from_db_value(self, value, expression, connection, format_item=None):
         if self.group_by_key:
             group_value = defaultdict(list)
             for item in value:
-                group_value[item[self.group_by_key]].append(item)
+                group_key = item[self.group_by_key]
+                group_value['null' if group_key is None else group_key].append(item)
             if self.flatten_groups:
                 value = {k: v[0] if len(v) == 1 else v for k, v in group_value.items()}
             else:

clickhouse_search/backend/functions.py

@@ -120,7 +120,7 @@ class DictGet(Func):
 
 class If(Func):
     function = 'if'
-    template = '%(function)s(%(condition)s, %(expressions)s)'
+    template = '%(function)s(%(condition)s%(expressions)s)'
 
 
 class MapLookup(Func):

clickhouse_search/fixtures/clickhouse_saved_variants.json

@@ -0,0 +1,333 @@
+[{
+    "model": "clickhouse_search.keylookupgrch37snvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "variant_id": "1-248367227-TC-T"
+    }
+}, {
+    "model": "clickhouse_search.annotationsgrch37snvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "xpos": 1248367227,
+      "chrom": "1",
+      "pos": 248367227,
+      "ref": "TC",
+      "alt": "T",
+      "variant_id": "1-248367227-TC-T",
+      "rsid": null,
+      "caid": "CA1501729",
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "hgmd": [null, null],
+      "predictions": [27.2, null, null, null, null, null, null, null, null, null, null, null, null],
+      "populations": [
+        [66593, 0.63, 104352, 0.81988, 0, 22269, 22162],
+        [137532, 0.63542, 216442, 0.82261, 0, 45869],
+        [14649, 0.61365, 23872, 0.82844, 0, 4584],
+        [65461, 0.52132, 125568, 33149, 16156]
+      ],
+      "sorted_transcript_consequences": [
+        [1, [], "ENSG00000240361"],
+        [1, [], "ENSG00000135953"]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.transcriptsgrch37snvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "transcripts": [
+        [null, "protein_coding", 1, "Gtg/Atg", ["intron_variant"], "ENSG00000240361", "ENST00000262738.3:c.3955G>A", "ENST00000505820.2:c.1586-17C>G", [null, []], "intron_variant", "ENST00000505820", 0],
+        [null, "protein_coding", 1, null, [], "ENSG00000135953", null, null, [null, []], null, "ENST00000371839", 1]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.clinvargrch37snvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "allele_id": 12345,
+      "conflicting_pathogenicities": [],
+      "gold_stars": null,
+      "submitters": [],
+      "conditions": [],
+      "assertions": [],
+      "pathogenicity": "Uncertain_significance"
+    }
+}, {
+    "model": "clickhouse_search.entriesgrch37snvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+       "project_guid": "R0001_1kg",
+      "family_guid": "F000002_2",
+      "sample_type": "WGS",
+      "xpos": 1248367227,
+      "is_gnomad_gt_5_percent": false,
+      "filters": [],
+      "sign": 1,
+      "calls": [
+        ["HG00733", 1, 99, 0.6530612111091614, 49],
+        ["HG00731", 2, 48, 1, 16]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.keylookupgrch37snvindel",
+    "pk": 101,
+    "fields": {
+      "key": 101,
+      "variant_id": "2-103343353-GAGA-G"
+    }
+}, {
+    "model": "clickhouse_search.annotationsgrch37snvindel",
+    "pk": 101,
+    "fields": {
+      "key": 101,
+      "xpos": 2103343353,
+      "chrom": "2",
+      "pos": 103343353,
+      "ref": "GAGA",
+      "alt": "G",
+      "variant_id": "2-103343353-GAGA-G",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "hgmd": [null, null],
+      "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null],
+      "populations": [
+        [0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [4, 0.000044254640670260414, 90386, 0.00011118523252662271, 0, 0],
+        [13, 0.00004911409996566363, 264690, 13, 0]
+      ],
+      "sorted_transcript_consequences": [
+        [1, ["inframe_deletion"], "ENSG00000135953"],
+        [1, ["frameshift_variant"], "ENSG00000228198"]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.entriesgrch37snvindel",
+    "pk": 101,
+    "fields": {
+      "key": 101,
+      "project_guid": "R0001_1kg",
+      "family_guid": "F000002_2",
+      "sample_type": "WES",
+      "xpos": 2103343353,
+      "is_gnomad_gt_5_percent": false,
+      "filters": [],
+      "sign": 1,
+      "calls": [
+        ["HG00731", 2, 99, 0, 67],
+        ["HG00732", 1, 96, 0, 42],
+        ["HG00733", 0, 96, 0, 42]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.annotationsgrch37snvindel",
+    "pk": 123,
+    "fields": {
+      "key": 123,
+      "xpos": 261413835,
+      "chrom": "2",
+      "pos": 61413835,
+      "ref": "AAAG",
+      "alt": "A",
+      "variant_id": "2-61413835-AAAG-A",
+      "rsid": null,
+      "caid": null,
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "hgmd": [null, null],
+      "predictions": [21.9, null, null, null, null, null, null, null, null, null, null, null, null],
+      "populations": [
+        [0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0]
+      ],
+      "sorted_transcript_consequences": []
+    }
+}, {
+    "model": "clickhouse_search.keylookupsnvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "variant_id": "1-248367227-TC-T"
+    }
+}, {
+    "model": "clickhouse_search.annotationssnvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "xpos": 1248367227,
+      "chrom": "1",
+      "pos": 248367227,
+      "ref": "TC",
+      "alt": "T",
+      "variant_id": "1-248367227-TC-T",
+      "rsid": null,
+      "caid": "CA1501729",
+      "lifted_over_chrom": "1",
+      "lifted_over_pos": null,
+      "hgmd": [null, null],
+      "screen_region_type": null,
+      "predictions": [null, null, null, null, null, null, null, null, null, null, null, null, null, null],
+      "populations": [
+        [0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [4, 0.000044254640670260414, 90386, 0.00011118523252662271, 0, 0],
+        [13, 0.00004911409996566363, 264690, 13, 0]
+      ],
+      "sorted_transcript_consequences": [],
+      "sorted_motif_feature_consequences": [],
+      "sorted_regulatory_feature_consequences": []
+    }
+}, {
+    "model": "clickhouse_search.entriessnvindel",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "project_guid": "R0004_non_analyst_project",
+      "family_guid": "F000014_14",
+      "sample_type": "WGS",
+      "xpos": 1248367227,
+      "is_gnomad_gt_5_percent": false,
+      "filters": [],
+      "sign": 1,
+      "calls": [
+        ["NA21234", 2, 99, 0, 49]
+      ]
+    }
+}, {
+    "model": "clickhouse_search.keylookupsv",
+    "pk": 101,
+    "fields": {
+      "key": 101,
+      "variant_id": "suffix_19107_DEL"
+    }
+}, {
+    "model": "clickhouse_search.annotationssv",
+    "pk": 101,
+    "fields": {
+      "key": 101,
+      "xpos": 1249045487,
+      "chrom": "1",
+      "end_chrom": null,
+      "pos": 249045487,
+      "end": 249045898,
+      "variant_id": "suffix_19107_DEL",
+      "lifted_over_chrom": null,
+      "lifted_over_pos": null,
+      "rg37_locus_end": [null, null],
+      "bothsides_support": false,
+      "algorithms": "manta",
+      "predictions": [0.374],
+      "populations": [
+        [0, 0, 0, ""]
+      ],
+      "sorted_gene_consequences": [
+        ["ENSG00000240361", "LOF"],
+        ["ENSG00000135953", "LOF"],
+        ["ENSG00000223972", "LOF"]
+      ],
+      "sv_type": "DEL",
+      "sv_type_detail": null,
+      "cpx_intervals": [],
+      "sv_source_detail": [null]
+    }
+},{
+    "model": "clickhouse_search.keylookupgcnv",
+    "pk": 111,
+    "fields": {
+      "key": 111,
+      "variant_id": "prefix_70191_DEL"
+    }
+}, {
+    "model": "clickhouse_search.annotationsgcnv",
+    "pk": 111,
+    "fields": {
+      "key": 111,
+      "xpos": 1249045487,
+      "chrom": "1",
+      "pos": 249045487,
+      "end": 249045898,
+      "variant_id": "prefix_70191_DEL",
+      "lifted_over_chrom": "2",
+      "lifted_over_pos": null,
+      "rg37_locus_end": [null, null],
+      "num_exon": 2,
+      "predictions": [0.374],
+      "populations": [
+        [2, 0.063, 32, 0, 0]
+      ],
+      "sorted_gene_consequences": [
+        ["ENSG00000240361", "LOF"],
+        ["ENSG00000135953", "LOF"],
+        ["ENSG00000223972", "LOF"]
+      ],
+      "sv_type": "DUP"
+    }
+}, {
+    "model": "clickhouse_search.annotationsgcnv",
+    "pk": 123,
+    "fields": {
+      "key": 123,
+      "xpos": 2061413835,
+      "chrom": "2",
+      "pos": 61413835,
+      "end": 61414175,
+      "variant_id": "batch_123_DUP",
+      "lifted_over_chrom": "2",
+      "lifted_over_pos": 61413835,
+      "rg37_locus_end": [2, 61414175],
+      "num_exon": 0,
+      "predictions": [21.9],
+      "populations": [
+        [2, 0.063, 32, 0, 0]
+      ],
+      "sorted_gene_consequences": [
+        ["ENSG00000240361", "LOF"]
+      ],
+      "sv_type": "DUP"
+    }
+}, {
+    "model": "clickhouse_search.keylookupmito",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "variant_id": "M-14783-T-C"
+    }
+}, {
+    "model": "clickhouse_search.annotationsmito",
+    "pk": 100,
+    "fields": {
+      "key": 100,
+      "xpos": 25000014783,
+      "pos": 14783,
+      "ref": "T",
+      "alt": "C",
+      "variant_id": "M-14783-T-C",
+      "rsid": "rs193302982",
+      "lifted_over_pos": 14783,
+      "common_low_heteroplasmy": true,
+      "mitomap_pathogenic": true,
+      "predictions": [null, true, null, null, null, null, 0.7514],
+      "populations": [
+        [3118, 0.05535, 56336],
+        [3, 0.00005, 56336, 1],
+        [9573, 0.04885, 195983],
+        [18, 0.00009, 195983, 0.96269]
+      ],
+      "sorted_transcript_consequences": [
+        [
+          "L", "protein_coding", 1, "Tta/Cta", ["synonymous_variant"], "ENSG00000198727", "ENST00000361789.2:c.37T>C",
+          "ENSP00000354554.2:p.Leu13=", [null, []], "synonymous_variant", "ENST00000361789", 0
+        ]
+      ]
+    }
+}
+]