diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index fb021f24f5..71c309be6e 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -41,7 +41,7 @@ jobs:
# Maps tcp port 5432 on service container to the host
- 5432:5432
clickhouse:
- image: bitnami/clickhouse:latest
+ image: bitnamilegacy/clickhouse:latest
ports:
- 9000:9000 # Native client interface
volumes:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3045cb08ca..098f304084 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,10 @@
## dev
## 9/18/25
-# Deprecate hail backend support
+* Store gene ids on SavedVariant model (REQUIRES DB MIGRATION)
+
+## 9/29/25
+* Deprecate hail backend support
## 8/7/25
* Deprecate support for lifting existing projects from GRCH37 to GRCh38
diff --git a/clickhouse_search/backend/functions.py b/clickhouse_search/backend/functions.py
index 0e61e7b6ab..ab1835fde6 100644
--- a/clickhouse_search/backend/functions.py
+++ b/clickhouse_search/backend/functions.py
@@ -7,6 +7,12 @@
class Array(Func):
function = 'array'
+ def _resolve_output_field(self):
+ output_field = super()._resolve_output_field()
+ if not isinstance(output_field, ArrayField):
+ output_field = ArrayField(base_field=output_field)
+ return output_field
+
class ArrayConcat(Func):
function = 'arrayConcat'
diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json
index 6a7e7706e7..cc94a9fa12 100644
--- a/clickhouse_search/fixtures/clickhouse_saved_variants.json
+++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json
@@ -213,7 +213,7 @@
"pk": 101,
"fields": {
"key": 101,
- "variant_id": "suffix_19107_DEL"
+ "variant_id": "suffix_19107_DEL_013746"
}
}, {
"model": "clickhouse_search.annotationssv",
@@ -225,7 +225,7 @@
"end_chrom": null,
"pos": 249045487,
"end": 249045898,
- "variant_id": "suffix_19107_DEL",
+ "variant_id": "suffix_19107_DEL_013746",
"lifted_over_chrom": null,
"lifted_over_pos": null,
"rg37_locus_end": [null, null],
@@ -250,7 +250,7 @@
"pk": 111,
"fields": {
"key": 111,
- "variant_id": "prefix_70191_DEL"
+ "variant_id": "prefix_19107_DEL"
}
}, {
"model": "clickhouse_search.annotationsgcnv",
@@ -261,7 +261,7 @@
"chrom": "1",
"pos": 249045487,
"end": 249045898,
- "variant_id": "prefix_70191_DEL",
+ "variant_id": "prefix_19107_DEL",
"lifted_over_chrom": "2",
"lifted_over_pos": null,
"rg37_locus_end": [null, null],
diff --git a/clickhouse_search/fixtures/clickhouse_search.json b/clickhouse_search/fixtures/clickhouse_search.json
index 3260b68c68..477191cf51 100644
--- a/clickhouse_search/fixtures/clickhouse_search.json
+++ b/clickhouse_search/fixtures/clickhouse_search.json
@@ -48,7 +48,7 @@
"populations": [
[35805, 0.29499999, 121372, 0.41530353, 0, 24061, 5872],
[72672, 0.28899795, 251462, 0.4116475, 0, 11567],
- [0, 0, 0, 0, 0, 0],
+ [927, 0.03444932, 26912, 0.04027665, 0, 0],
[65154, 0.246152, 264690, 47604, 8775]
],
"sorted_transcript_consequences": [
@@ -124,7 +124,7 @@
],
"sorted_transcript_consequences": [
[null, null, ["splice_donor_variant"], true, null, "ENSG00000097046"],
- [null, 1, ["missense_variant"], false, null, "ENSG00000097046"],
+ [0.1, 1, ["missense_variant"], false, null, "ENSG00000097046"],
[null, null, ["missense_variant"], false, null, "ENSG00000097046"]
],
"sorted_motif_feature_consequences": [
@@ -681,7 +681,7 @@
],
"sign": 1,
"calls": [
- ["HG00733", 1, 99, 0.6530612111091614, 49],
+ ["HG00733", null, 0, null, null],
["HG00731", 2, 48, 1, 16]
]
}
diff --git a/clickhouse_search/fixtures/clickhouse_transcripts.json b/clickhouse_search/fixtures/clickhouse_transcripts.json
index e7a3352963..27894fd71a 100644
--- a/clickhouse_search/fixtures/clickhouse_transcripts.json
+++ b/clickhouse_search/fixtures/clickhouse_transcripts.json
@@ -113,7 +113,7 @@
], null]
],
[
- [null], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046",
+ [0.1], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046",
"ENST00000350997.12:c.375+139G>A", null, [1, 11], [null, []], "missense_variant", null, "NM_013402.7",
"NM_013402.7", [false], "ENST00000350997", 1, [null, null, null, [
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null
diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py
index 07df47e39b..49a6d0c0d8 100644
--- a/clickhouse_search/management/commands/set_saved_variant_key.py
+++ b/clickhouse_search/management/commands/set_saved_variant_key.py
@@ -4,2051 +4,16 @@
from django.core.management.base import BaseCommand
from django.db.models import F
import logging
-import re
from clickhouse_search.search import get_clickhouse_key_lookup
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
from seqr.models import SavedVariant, Sample
-from seqr.utils.file_utils import file_iter
from seqr.utils.search.utils import parse_variant_id
logger = logging.getLogger(__name__)
BATCH_SIZE = 10000
-GCNV_CALLSET_PATH = 'gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz'
-SV_ID_UPDATE_MAP = {
- 'WGS': {
- 'CMG.phase1_CMG_DEL_chr10_2038': 'phase2_DEL_chr10_4611',
- 'CMG.phase1_CMG_DEL_chr11_2963': 'phase2_DEL_chr11_5789',
- 'CMG.phase1_CMG_DEL_chr13_153': 'phase2_DEL_chr13_378',
- 'cohort_2911.chr1.final_cleanup_BND_chr1_1167': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1376',
- 'cohort_2911.chr1.final_cleanup_BND_chr1_1017': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1208',
- 'cohort_2911.chr1.final_cleanup_BND_chr1_2837': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_3326',
- 'cohort_2911.chr1.final_cleanup_DEL_chr1_12237': 'phase2_DEL_chr1_9347',
- 'cohort_2911.chr1.final_cleanup_DEL_chr1_2953': 'phase2_DEL_chr1_2503',
- 'cohort_2911.chrX.final_cleanup_CPX_chrX_20': 'cohort_2911.chrX.final_cleanup_CPX_chrX_19',
- 'phase2_DUP_chr1_1164': 'phase4_all_batches.chr1.final_cleanup_DUP_chr1_1666',
- 'phase2_CPX_chr1_73': 'phase4_all_batches.chr1.final_cleanup_CPX_chr1_85',
- 'cohort_2911.chr1.final_cleanup_DEL_chr1_3844': 'phase2_DEL_chr1_3148',
- 'cohort_2911.chr1.final_cleanup_DEL_chr1_4181': 'phase2_DEL_chr1_3389',
- 'cohort_2911.chr1.final_cleanup_DEL_chr1_6414': 'phase2_DEL_chr1_5030',
- 'cohort_2911.chr10.final_cleanup_DEL_chr10_6275': 'phase2_DEL_chr10_4600',
- 'cohort_2911.chr10.final_cleanup_DEL_chr10_7793': 'phase2_DEL_chr10_5695',
- 'cohort_2911.chr10.final_cleanup_DUP_chr10_1491': 'phase2_DUP_chr10_1234',
- 'cohort_2911.chr10.final_cleanup_DUP_chr10_3659': 'phase2_DUP_chr10_3132',
- 'cohort_2911.chr11.final_cleanup_DUP_chr11_3003': 'phase2_DUP_chr11_2757',
- 'cohort_2911.chr12.final_cleanup_DEL_chr12_244': 'phase2_DEL_chr12_207',
- 'cohort_2911.chr12.final_cleanup_DEL_chr12_5405': 'phase2_DEL_chr12_4072',
- 'cohort_2911.chr12.final_cleanup_INS_chr12_42': 'phase4_all_batches.chr12.final_cleanup_INS_chr12_61',
- 'cohort_2911.chr14.final_cleanup_DEL_chr14_1513': 'phase2_DEL_chr14_1140',
- 'cohort_2911.chr14.final_cleanup_DEL_chr14_2595': 'phase2_DEL_chr14_1998',
- 'cohort_2911.chr15.final_cleanup_DEL_chr15_3281': 'phase2_DEL_chr15_2486',
- 'cohort_2911.chr16.final_cleanup_DEL_chr16_5308': 'phase2_DEL_chr16_4074',
- 'cohort_2911.chr17.final_cleanup_BND_chr17_209': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_250',
- 'cohort_2911.chr16.final_cleanup_BND_chr16_635': 'phase4_all_batches.chr16.final_cleanup_BND_chr16_744',
- 'cohort_2911.chr17.final_cleanup_CPX_chr17_79': 'phase2_CPX_chr17_46',
- 'cohort_2911.chr17.final_cleanup_DEL_chr17_306': 'phase2_DEL_chr17_271',
- 'cohort_2911.chr17.final_cleanup_DEL_chr17_3161': 'phase2_DEL_chr17_2627',
- 'cohort_2911.chr17.final_cleanup_DEL_chr17_4111': 'phase2_DEL_chr17_3390',
- 'cohort_2911.chr17.final_cleanup_DEL_chr17_5157': 'phase2_DEL_chr17_4242',
- 'cohort_2911.chr18.final_cleanup_DEL_chr18_3519': 'phase2_DEL_chr18_2585',
- 'cohort_2911.chr19.final_cleanup_BND_chr19_162': 'phase4_all_batches.chr19.final_cleanup_BND_chr19_185',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_3395': 'phase2_DEL_chrX_2020',
- 'cohort_2911.chr19.final_cleanup_DEL_chr19_1825': 'phase2_DEL_chr19_1567',
- 'cohort_2911.chr19.final_cleanup_DEL_chr19_3787': 'phase2_DEL_chr19_3203',
- 'cohort_2911.chr19.final_cleanup_DEL_chr19_807': 'phase2_DEL_chr19_692',
- 'cohort_2911.chr19.final_cleanup_DEL_chr19_971': 'phase2_DEL_chr19_832',
- 'cohort_2911.chr2.final_cleanup_DEL_chr2_7485': 'phase2_DEL_chr2_5601',
- 'cohort_2911.chr20.final_cleanup_DEL_chr20_37': 'phase2_DEL_chr20_25',
- 'cohort_2911.chr20.final_cleanup_INV_chr20_6': 'phase2_INV_chr20_3',
- 'cohort_2911.chr22.final_cleanup_DEL_chr22_1440': 'phase2_DEL_chr22_1210',
- 'cohort_2911.chr3.final_cleanup_DUP_chr3_566': 'phase2_DUP_chr3_522',
- 'cohort_2911.chr4.final_cleanup_DEL_chr4_5267': 'phase2_DEL_chr4_3794',
- 'cohort_2911.chr6.final_cleanup_DEL_chr6_9094': 'phase2_DEL_chr6_6868',
- 'cohort_2911.chr7.final_cleanup_DEL_chr7_1540': 'phase2_DEL_chr7_1311',
- 'cohort_2911.chr8.final_cleanup_DEL_chr8_75': 'phase2_DEL_chr8_67',
- 'cohort_2911.chr8.final_cleanup_DEL_chr8_7934': 'phase2_DEL_chr8_5754',
- 'cohort_2911.chr9.final_cleanup_DEL_chr9_6983': 'phase2_DEL_chr9_5130',
- 'cohort_2911.chrX.final_cleanup_BND_chrX_166': 'phase4_all_batches.chrX.final_cleanup_BND_chrX_227',
- 'cohort_2911.chrX.final_cleanup_CPX_chrX_106': 'cohort_2911.chrX.final_cleanup_CPX_chrX_105',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1861': 'phase2_DEL_chrX_1151',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1922': 'phase2_DEL_chrX_1184',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1936': 'phase2_DEL_chrX_1194',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1974': 'phase2_DEL_chrX_1222',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_2607': 'phase2_DEL_chrX_1550',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_2882': 'phase2_DEL_chrX_1715',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_3450': 'phase2_DEL_chrX_2061',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_6616': 'phase2_DEL_chrX_3880',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_7868': 'phase2_DEL_chrX_4582',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_7872': 'phase2_DEL_chrX_4585',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_7882': 'phase2_DEL_chrX_4594',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_1102': 'phase2_DUP_chrX_1000',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_1791': 'phase2_DUP_chrX_1595',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_2682': 'phase2_DUP_chrX_2394',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_3233': 'phase2_DUP_chrX_2875',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_3564': 'phase2_DUP_chrX_3160',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_3599': 'phase2_DUP_chrX_3192',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_3621': 'phase2_DUP_chrX_3211',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_600': 'phase2_DUP_chrX_554',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_656': 'phase2_DUP_chrX_606',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_818': 'phase2_DUP_chrX_743',
- 'phase2_BND_chr17_485': 'phase4_all_batches.chr17.final_cleanup_BND_chr17_710',
- 'phase2_BND_chr1_1262': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_1669',
- 'phase2_BND_chr1_19': 'phase4_all_batches.chr1.final_cleanup_BND_chr1_25',
- 'phase2_BND_chr4_1666': 'phase4_all_batches.chr4.final_cleanup_BND_chr4_2264',
- 'phase2_BND_chr5_1124': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1476',
- 'phase2_DUP_chr11_942': 'phase4_all_batches.chr11.final_cleanup_DUP_chr11_1405',
- 'cohort_2911.chr19.final_cleanup_DEL_chr19_3909': 'phase2_DEL_chr19_3310',
- 'cohort_2911.chr19.final_cleanup_DUP_chr19_1707': 'phase2_DUP_chr19_1571',
- 'cohort_2911.chr2.final_cleanup_DUP_chr2_3291': 'phase2_DUP_chr2_2862',
- 'cohort_2911.chr3.final_cleanup_DEL_chr3_8602': 'phase2_DEL_chr3_6419',
- 'cohort_2911.chr3.final_cleanup_BND_chr3_2956': 'phase4_all_batches.chr3.final_cleanup_BND_chr3_3398',
- 'cohort_2911.chr6.final_cleanup_DEL_chr6_7703': 'phase2_DEL_chr6_5832',
- 'cohort_2911.chrX.final_cleanup_CPX_chrX_69': 'phase4_all_batches.chrX.final_cleanup_CPX_chrX_63',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1879': 'phase2_DEL_chrX_1160',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_3396': 'phase2_DEL_chrX_2020',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_6041': 'phase2_DEL_chrX_3547',
- 'phase2_CTX_chr3_2': 'phase4_all_batches.chr3.final_cleanup_CTX_chr3_3',
- 'phase2_DEL_chr11_3941': 'phase4_all_batches.chr11.final_cleanup_DEL_chr11_6231',
- 'phase2_DEL_chr16_453': 'phase2_DEL_chr16_452',
- 'phase2_DEL_chr16_516': 'phase2_DEL_chr16_517',
- 'phase2_DEL_chr20_2752': 'phase4_all_batches.chr20.final_cleanup_DEL_chr20_4722',
- 'phase2_DUP_chr16_1292': 'phase4_all_batches.chr16.final_cleanup_DUP_chr16_1975',
- 'cohort_2911.chr2.final_cleanup_DEL_chr2_10322': 'phase2_DEL_chr2_7665',
- 'cohort_2911.chr9.final_cleanup_DEL_chr9_2715': 'phase2_DEL_chr9_2002',
- 'cohort_2911.chrX.final_cleanup_DEL_chrX_1905': 'phase2_DEL_chrX_1175',
- 'cohort_2911.chrX.final_cleanup_DUP_chrX_1533': 'phase4_all_batches.chrX.final_cleanup_DUP_chrX_2546',
- 'phase2_CPX_chr1_28': 'cohort_2911.chr1.final_cleanup_CPX_chr1_49',
- 'phase2_DEL_chr18_2435': 'cohort_2911.chr18.final_cleanup_DEL_chr18_3308',
- 'phase2_DUP_chr8_4806': 'phase2_DUP_chr8_4807',
- 'cohort_2911.chr2.final_cleanup_BND_chr2_3079': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_3571',
- 'cohort_2911.chr2.final_cleanup_BND_chr2_166': 'phase4_all_batches.chr2.final_cleanup_BND_chr2_192',
- 'phase2_BND_chr13_369': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_478',
- 'phase2_BND_chr7_230': 'phase4_all_batches.chr7.final_cleanup_BND_chr7_300',
- 'phase2_BND_chr8_238': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_320',
- 'phase2_BND_chr8_53': 'phase4_all_batches.chr8.final_cleanup_BND_chr8_72',
- 'phase2_BND_chr9_489': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_663',
- 'cohort_2911.chr5.final_cleanup_BND_chr5_1116': 'phase4_all_batches.chr5.final_cleanup_BND_chr5_1251',
- 'cohort_2911.chr9.final_cleanup_BND_chr9_764': 'phase4_all_batches.chr9.final_cleanup_BND_chr9_849',
- 'phase2_BND_chr13_368': 'phase4_all_batches.chr13.final_cleanup_BND_chr13_479',
- },
- 'WES': {
- 'R4_variant_7334_DUP_08162023': 'R4_variant_7334_DUP',
- 'prefix_112949_DEL': 'suffix_210553_DEL',
- 'prefix_131670_DUP': 'suffix_251025_DUP',
- 'suffix_19443_DEL_2': 'suffix_20030_DEL',
- 'prefix_184342_DEL': 'suffix_194439_DEL',
- 'prefix_255018_DEL': 'suffix_155939_DEL',
- 'prefix_188042_DUP': 'suffix_107531_DUP',
- 'suffix_104367_DUP_2': 'suffix_107531_DUP',
- 'prefix_59865_DEL': 'suffix_124465_DEL',
- 'suffix_120814_DEL_2': 'suffix_124465_DEL',
- 'prefix_152595_DEL': 'suffix_286580_DEL',
- 'suffix_277504_DEL_2': 'suffix_286581_DEL',
- 'prefix_185630_DEL': 'suffix_217490_DEL',
- 'suffix_210888_DEL_2': 'suffix_217490_DEL',
- 'prefix_126071_DUP': 'suffix_48694_DUP',
- 'suffix_47226_DUP_2': 'suffix_48694_DUP',
- 'prefix_177696_DUP': 'suffix_23_DUP',
- 'prefix_33049_DEL': 'suffix_123373_DEL',
- 'suffix_119753_DEL_2': 'suffix_123373_DEL',
- 'prefix_6968_DUP': 'suffix_8158_DUP',
- 'prefix_111128_DEL': 'suffix_124250_DEL',
- 'prefix_265455_DEL': 'suffix_124250_DEL',
- 'prefix_133233_DEL': 'suffix_149757_DEL',
- 'suffix_145373_DEL_2': 'suffix_149757_DEL',
- 'prefix_104962_DEL': 'suffix_262703_DEL',
- 'prefix_230669_DUP': 'suffix_37212_DUP',
- 'prefix_195634_DUP': 'suffix_337680_DUP',
- 'suffix_326904_DUP_2': 'suffix_337680_DUP',
- 'prefix_252896_DUP': 'suffix_154760_DUP',
- 'prefix_252895_DUP': 'suffix_150484_DUP',
- 'prefix_192808_DUP': 'suffix_182662_DUP',
- 'prefix_31312_DUP': 'suffix_343278_DUP',
- 'suffix_332369_DUP_2': 'suffix_343278_DUP',
- 'prefix_201559_DEL': 'suffix_228780_DEL',
- 'prefix_173086_DEL': 'suffix_195404_DEL',
- 'prefix_230567_DEL': 'suffix_261336_DEL',
- 'prefix_25206_DEL': 'suffix_27995_DEL',
- 'prefix_72517_DEL': 'suffix_27995_DEL',
- 'prefix_194936_DUP': 'suffix_220746_DUP',
- 'prefix_117065_DEL': 'suffix_131048_DEL',
- 'suffix_127235_DEL_2': 'suffix_131048_DEL',
- 'prefix_168621_DEL': 'suffix_336993_DEL',
- 'prefix_297236_DEL': 'suffix_336993_DEL',
- 'prefix_198998_DUP': 'suffix_225862_DUP',
- 'prefix_239300_DUP': 'suffix_271302_DUP',
- 'prefix_236836_DEL': 'suffix_268435_DEL',
- 'prefix_31131_DUP': 'suffix_34872_DUP',
- 'prefix_238314_DEL': 'suffix_336979_DEL',
- 'prefix_75798_DUP': 'suffix_84697_DUP',
- 'prefix_261715_DEL': 'suffix_337689_DEL',
- 'prefix_183989_DEL': 'suffix_4946_DEL',
- 'prefix_26448_DEL': 'suffix_43066_DEL',
- 'suffix_41786_DEL_2': 'suffix_43066_DEL',
- 'prefix_103176_DEL': 'suffix_114811_DEL',
- 'suffix_111479_DEL_2': 'suffix_114811_DEL',
- 'prefix_120528_DEL': 'suffix_135375_DEL',
- 'prefix_296302_DEL': 'suffix_335637_DEL',
- 'prefix_108274_DEL': 'suffix_121089_DEL',
- 'prefix_46615_DEL': 'suffix_118481_DEL',
- 'prefix_185022_DUP': 'suffix_12708_DUP',
- 'suffix_12284_DUP_2': 'suffix_12708_DUP',
- 'prefix_252585_DUP': 'suffix_309117_DUP',
- 'suffix_299264_DUP_2': 'suffix_309117_DUP',
- 'prefix_80031_DUP': 'suffix_241944_DUP',
- 'suffix_234357_DUP_2': 'suffix_241944_DUP',
- 'prefix_117179_DEL': 'suffix_131305_DEL',
- 'prefix_71499_DUP': 'suffix_330986_DUP',
- 'suffix_320433_DUP_2': 'suffix_330986_DUP',
- 'prefix_24788_DUP': 'suffix_336670_DUP',
- 'suffix_325911_DUP_2': 'suffix_336670_DUP',
- 'prefix_192365_DUP': 'suffix_139090_DUP',
- 'prefix_255360_DUP': 'suffix_337082_DUP',
- 'prefix_237614_DEL': 'suffix_269379_DEL',
- 'prefix_284003_DEL': 'suffix_321174_DEL',
- 'suffix_12426_DEL_2': 'suffix_12852_DEL',
- 'prefix_9852_DEL': 'suffix_12852_DEL',
- 'suffix_50042_DEL_2': 'suffix_51609_DEL',
- 'prefix_223675_DEL': 'suffix_51609_DEL',
- 'prefix_175156_DEL': 'suffix_54122_DEL',
- 'suffix_52499_DEL_2': 'suffix_54122_DEL',
- 'prefix_3138_DEL': 'suffix_109381_DEL',
- 'suffix_106163_DEL_2': 'suffix_109381_DEL',
- 'prefix_255276_DEL': 'suffix_93464_DEL',
- 'prefix_152594_DEL': 'suffix_286542_DEL',
- 'prefix_77396_DEL': 'suffix_287070_DEL',
- 'prefix_152599_DEL': 'suffix_287163_DEL',
- 'prefix_152600_DEL': 'suffix_287381_DEL',
- 'prefix_108317_DEL': 'suffix_161791_DEL',
- 'prefix_201562_DEL': 'suffix_168210_DEL',
- 'suffix_163193_DEL_2': 'suffix_168210_DEL',
- 'prefix_246635_DEL': 'suffix_92687_DEL',
- 'suffix_89974_DEL_2': 'suffix_92687_DEL',
- 'prefix_246642_DEL': 'suffix_93299_DEL',
- 'suffix_90573_DEL_2': 'suffix_93299_DEL',
- 'prefix_246674_DUP': 'suffix_126676_DUP',
- 'suffix_122983_DUP_2': 'suffix_126676_DUP',
- 'prefix_199389_DEL': 'suffix_94004_DEL',
- 'prefix_3593_DEL': 'suffix_27094_DEL',
- 'prefix_100725_DEL': 'suffix_336955_DEL',
- 'prefix_297227_DEL': 'suffix_336980_DEL',
- 'prefix_179648_DEL': 'suffix_202719_DEL',
- 'prefix_143182_DEL': 'suffix_100756_DEL',
- 'prefix_34890_DEL': 'suffix_38919_DEL',
- 'prefix_36703_DEL': 'suffix_39815_DEL',
- 'prefix_164553_DEL': 'suffix_185929_DEL',
- 'prefix_132206_DEL': 'suffix_282692_DEL',
- 'suffix_273714_DEL_2': 'suffix_282692_DEL',
- 'prefix_144442_DEL': 'suffix_282716_DEL',
- 'suffix_273738_DEL_2': 'suffix_282716_DEL',
- 'suffix_273732_DEL_2': 'suffix_282710_DEL',
- 'prefix_6326_DEL': 'suffix_282710_DEL',
- 'prefix_178200_DEL': 'suffix_118870_DEL',
- 'suffix_115409_DEL_2': 'suffix_118870_DEL',
- 'prefix_254507_DUP': 'suffix_283948_DUP',
- 'prefix_254510_DUP': 'suffix_39298_DUP',
- 'prefix_13260_DEL': 'suffix_38202_DEL',
- 'suffix_37073_DEL_2': 'suffix_38202_DEL',
- 'prefix_278497_DEL': 'suffix_338533_DEL',
- 'suffix_327734_DEL_2': 'suffix_338533_DEL',
- 'prefix_242131_DEL': 'suffix_176483_DEL',
- 'suffix_171236_DEL_2': 'suffix_176483_DEL',
- 'suffix_32210_DEL_2': 'suffix_33192_DEL',
- 'prefix_37286_DEL': 'suffix_33192_DEL',
- 'prefix_113302_DUP': 'suffix_236396_DUP',
- 'suffix_228996_DUP_2': 'suffix_236396_DUP',
- 'prefix_154396_DEL': 'suffix_174270_DEL',
- 'prefix_71769_DEL': 'suffix_227830_DEL',
- 'suffix_220764_DEL_2': 'suffix_227830_DEL',
- 'prefix_44455_DEL': 'suffix_50096_DEL',
- 'prefix_135473_DEL': 'suffix_152259_DEL',
- 'prefix_83881_DUP': 'suffix_93771_DUP',
- 'prefix_185180_DEL': 'suffix_190807_DEL',
- 'prefix_230340_DEL': 'suffix_217490_DEL',
- 'prefix_98781_DUP': 'suffix_118080_DUP',
- 'suffix_333515_DEL_2': 'suffix_344457_DEL',
- 'prefix_50567_DEL': 'suffix_344457_DEL',
- 'suffix_243997_DEL_2': 'suffix_251931_DEL',
- 'prefix_185688_DEL': 'suffix_251931_DEL',
- 'prefix_185697_DEL': 'suffix_252985_DEL',
- 'suffix_245019_DEL_2': 'suffix_252985_DEL',
- 'prefix_185700_DEL': 'suffix_253165_DEL',
- 'suffix_245194_DEL_2': 'suffix_253165_DEL',
- 'prefix_164600_DUP': 'suffix_33272_DUP',
- 'suffix_32287_DUP_2': 'suffix_33272_DUP',
- 'prefix_239826_DUP': 'suffix_339672_DUP',
- 'suffix_328837_DUP_2': 'suffix_339672_DUP',
- 'prefix_239847_DUP': 'suffix_341812_DUP',
- 'suffix_330933_DUP_2': 'suffix_341812_DUP',
- 'suffix_217083_DUP_2': 'suffix_223945_DUP',
- 'prefix_78332_DUP': 'suffix_223945_DUP',
- 'prefix_81687_DUP': 'suffix_132894_DUP',
- 'suffix_129007_DUP_2': 'suffix_132894_DUP',
- 'suffix_150605_DEL_2': 'suffix_155170_DEL',
- 'prefix_47681_DEL': 'suffix_155170_DEL',
- 'suffix_119752_DUP_2': 'suffix_123374_DUP',
- 'prefix_75741_DUP': 'suffix_123374_DUP',
- 'prefix_17370_DUP': 'suffix_19297_DUP',
- 'prefix_54006_DUP': 'suffix_19297_DUP',
- 'prefix_137115_DUP': 'suffix_67761_DUP',
- 'suffix_65769_DUP_2': 'suffix_67761_DUP',
- 'prefix_127018_DUP': 'suffix_183995_DUP',
- 'prefix_179697_DUP': 'suffix_183995_DUP',
- 'prefix_65335_DEL': 'suffix_334692_DEL',
- 'prefix_26523_DEL': 'suffix_92931_DEL',
- 'suffix_90209_DEL_2': 'suffix_92931_DEL',
- 'prefix_121355_DEL': 'suffix_310958_DEL',
- 'prefix_3970_DEL': 'suffix_311681_DEL',
- 'prefix_16643_DEL': 'suffix_337172_DEL',
- 'suffix_326410_DEL_2': 'suffix_337172_DEL',
- 'prefix_98251_DEL': 'suffix_179586_DEL',
- 'suffix_174257_DEL_2': 'suffix_179586_DEL',
- 'prefix_302982_DUP': 'suffix_344928_DUP',
- 'prefix_78630_DUP': 'suffix_344928_DUP',
- 'prefix_248115_DEL': 'suffix_344177_DEL',
- 'prefix_60051_DUP': 'suffix_338928_DUP',
- 'suffix_328124_DUP_2': 'suffix_338928_DUP',
- 'prefix_202120_DUP': 'suffix_10024_DUP',
- 'prefix_137202_DEL': 'suffix_105069_DEL',
- 'suffix_101981_DEL_2': 'suffix_105069_DEL',
- 'suffix_312453_DEL_2': 'suffix_322732_DEL',
- 'prefix_101760_DEL': 'suffix_322732_DEL',
- 'prefix_164827_DUP': 'suffix_67293_DUP',
- 'prefix_164831_DEL': 'suffix_223792_DEL',
- 'suffix_216934_DEL_2': 'suffix_223792_DEL',
- 'prefix_47704_DEL': 'suffix_24051_DEL',
- 'prefix_108297_DEL': 'suffix_155176_DEL',
- 'prefix_284632_DEL': 'suffix_155176_DEL',
- 'prefix_37083_DUP': 'suffix_246174_DUP',
- 'prefix_37090_DUP': 'suffix_246928_DUP',
- 'prefix_37105_DUP': 'suffix_247737_DUP',
- 'prefix_164627_DEL': 'suffix_308855_DEL',
- 'prefix_164630_DUP': 'suffix_309488_DUP',
- 'prefix_247841_DUP': 'suffix_310746_DUP',
- 'prefix_164644_DUP': 'suffix_311682_DUP',
- 'suffix_301772_DUP_2': 'suffix_311682_DUP',
- 'prefix_137780_DEL': 'suffix_223761_DEL',
- 'prefix_12281_DEL': 'suffix_30168_DEL',
- 'suffix_29274_DEL_2': 'suffix_30168_DEL',
- 'prefix_202150_DEL': 'suffix_5892_DEL',
- 'prefix_249235_DUP': 'suffix_178512_DUP',
- 'suffix_173214_DUP_2': 'suffix_178512_DUP',
- 'prefix_286336_DUP': 'suffix_137695_DUP',
- 'suffix_133674_DUP_2': 'suffix_137695_DUP',
- 'prefix_202153_DUP': 'suffix_217142_DUP',
- 'suffix_210547_DUP_2': 'suffix_217142_DUP',
- 'suffix_101293_DUP_2': 'suffix_104364_DUP',
- 'prefix_240446_DUP': 'suffix_104364_DUP',
- 'suffix_326211_DEL_2': 'suffix_336972_DEL',
- 'suffix_214414_DUP_2': 'suffix_221162_DUP',
- 'suffix_264403_DUP_2': 'suffix_273062_DUP',
- 'suffix_92903_DEL_2': 'suffix_95683_DEL',
- 'suffix_326915_DEL_2': 'suffix_337695_DEL',
- 'suffix_432_DEL_2': 'suffix_442_DEL',
- 'suffix_96995_DEL_2': 'suffix_99914_DEL',
- 'suffix_16343_DEL_2': 'suffix_16857_DEL',
- 'suffix_239134_DUP_2': 'suffix_246928_DUP',
- 'suffix_239928_DUP_2': 'suffix_247737_DUP',
- 'suffix_299627_DUP_2': 'suffix_309488_DUP',
- 'suffix_300851_DUP_2': 'suffix_310746_DUP',
- 'prefix_100003_DEL': 'suffix_183787_DEL',
- 'prefix_101443_DEL': 'suffix_308476_DEL',
- 'prefix_10035_DEL': 'suffix_191074_DEL',
- 'prefix_102085_DUP': 'suffix_258879_DUP',
- 'prefix_102166_DUP': 'suffix_308496_DUP',
- 'prefix_102435_DEL': 'suffix_4265_DEL',
- 'prefix_102446_DEL': 'suffix_8242_DEL',
- 'prefix_102604_DEL': 'suffix_114190_DEL',
- 'prefix_10313_DEL': 'suffix_282612_DEL',
- 'prefix_10397_DEL': 'suffix_299122_DEL',
- 'prefix_104124_DEL': 'suffix_115920_DEL',
- 'prefix_104792_DEL': 'suffix_116623_DEL',
- 'prefix_105204_DEL': 'suffix_321184_DEL',
- 'prefix_10520_DEL': 'suffix_12114_DEL',
- 'prefix_105408_DEL': 'suffix_124299_DEL',
- 'prefix_105567_DEL': 'suffix_137041_DEL',
- 'prefix_105575_DEL': 'suffix_132895_DEL',
- 'prefix_105957_DEL': 'suffix_117939_DEL',
- 'prefix_106328_DUP': 'suffix_118372_DUP',
- 'prefix_10674_DEL': 'suffix_62051_DEL',
- 'prefix_106904_DUP': 'suffix_119175_DUP',
- 'prefix_10706_DUP': 'suffix_12310_DUP',
- 'prefix_107471_DEL': 'suffix_119959_DEL',
- 'prefix_107474_DEL': 'suffix_107226_DEL',
- 'prefix_107640_DEL': 'suffix_42123_DEL',
- 'prefix_107682_DEL': 'suffix_328436_DEL',
- 'prefix_107713_DEL': 'suffix_295713_DEL',
- 'prefix_107739_DEL': 'suffix_120359_DEL',
- 'prefix_108245_DUP': 'suffix_345422_DUP',
- 'prefix_108246_DUP': 'suffix_345434_DUP',
- 'prefix_108445_DEL': 'suffix_193406_DEL',
- 'prefix_108449_DUP': 'suffix_306110_DUP',
- 'prefix_108682_DEL': 'suffix_121584_DEL',
- 'prefix_108787_DUP': 'suffix_121697_DUP',
- 'prefix_109461_DUP': 'suffix_69248_DUP',
- 'prefix_109954_DEL': 'suffix_136917_DEL',
- 'prefix_11087_DEL': 'suffix_12711_DEL',
- 'prefix_111170_DUP': 'suffix_124298_DUP',
- 'prefix_111287_DUP': 'suffix_124448_DUP',
- 'prefix_111457_DEL': 'suffix_325622_DEL',
- 'prefix_112147_DEL': 'suffix_176498_DEL',
- 'prefix_112170_DEL': 'suffix_189247_DEL',
- 'prefix_112473_DEL': 'suffix_189253_DEL',
- 'prefix_112579_DUP': 'suffix_218784_DUP',
- 'prefix_112737_DEL': 'suffix_322979_DEL',
- 'prefix_113007_DEL': 'suffix_189763_DEL',
- 'prefix_113045_DEL': 'suffix_182363_DEL',
- 'prefix_113061_DUP': 'suffix_258764_DUP',
- 'prefix_113221_DUP': 'suffix_345607_DUP',
- 'prefix_113248_DEL': 'suffix_220751_DEL',
- 'prefix_113388_DEL': 'suffix_49218_DEL',
- 'prefix_114137_DUP': 'suffix_181479_DUP',
- 'prefix_114155_DEL': 'suffix_203405_DEL',
- 'prefix_114307_DUP': 'suffix_127847_DUP',
- 'prefix_114524_DEL': 'suffix_236802_DEL',
- 'prefix_115161_DEL': 'suffix_337104_DEL',
- 'prefix_115528_DEL': 'suffix_345416_DEL',
- 'prefix_115594_DEL': 'suffix_185881_DEL',
- 'prefix_1164_DEL': 'suffix_1403_DEL',
- 'prefix_116701_DEL': 'suffix_130655_DEL',
- 'prefix_11784_DEL': 'suffix_193515_DEL',
- 'prefix_11793_DEL': 'suffix_261170_DEL',
- 'prefix_118448_DUP': 'suffix_132890_DUP',
- 'prefix_118567_DUP': 'suffix_257299_DUP',
- 'prefix_118926_DEL': 'suffix_272807_DEL',
- 'prefix_119331_DEL': 'suffix_133934_DEL',
- 'prefix_119619_DUP': 'suffix_251043_DUP',
- 'prefix_12014_DEL': 'suffix_171415_DEL',
- 'prefix_120661_DUP': 'suffix_253757_DUP',
- 'prefix_120683_DUP': 'suffix_299274_DUP',
- 'prefix_121292_DUP': 'suffix_136324_DUP',
- 'prefix_122017_DUP': 'suffix_169951_DUP',
- 'prefix_122819_DEL': 'suffix_137963_DEL',
- 'prefix_122974_DUP': 'suffix_138130_DUP',
- 'prefix_123007_DUP': 'suffix_138159_DUP',
- 'prefix_123041_DEL': 'suffix_335303_DEL',
- 'prefix_123977_DEL': 'suffix_139305_DEL',
- 'prefix_12470_DEL': 'suffix_226823_DEL',
- 'prefix_124928_DEL': 'suffix_140340_DEL',
- 'prefix_12605_DEL': 'suffix_241864_DEL',
- 'prefix_127020_DUP': 'suffix_202584_DUP',
- 'prefix_127505_DUP': 'suffix_252433_DUP',
- 'prefix_12830_DEL': 'suffix_282628_DEL',
- 'prefix_13020_DEL': 'suffix_14730_DEL',
- 'prefix_13052_DEL': 'suffix_311669_DEL',
- 'prefix_131669_DEL': 'suffix_33717_DEL',
- 'prefix_131746_DEL': 'suffix_147895_DEL',
- 'prefix_131753_DEL': 'suffix_111130_DEL',
- 'prefix_13269_DUP': 'suffix_14995_DUP',
- 'prefix_133114_DUP': 'suffix_217164_DUP',
- 'prefix_133204_DUP': 'suffix_49743_DUP',
- 'prefix_133405_DEL': 'suffix_342958_DEL',
- 'prefix_13359_DEL': 'suffix_15085_DEL',
- 'prefix_13369_DEL': 'suffix_52450_DEL',
- 'prefix_133839_DUP': 'suffix_82443_DUP',
- 'prefix_134071_DEL': 'suffix_307896_DEL',
- 'prefix_134074_DEL': 'suffix_308281_DEL',
- 'prefix_134075_DEL': 'suffix_308321_DEL',
- 'prefix_134411_DEL': 'suffix_196304_DEL',
- 'prefix_134571_DEL': 'suffix_150896_DEL',
- 'prefix_134858_DEL': 'suffix_151606_DEL',
- 'prefix_135555_DEL': 'suffix_152343_DEL',
- 'prefix_136351_DUP': 'suffix_153160_DUP',
- 'prefix_137771_DUP': 'suffix_290375_DUP',
- 'prefix_138469_DUP': 'suffix_211854_DUP',
- 'prefix_138595_DUP': 'suffix_97099_DUP',
- 'prefix_138886_DUP': 'suffix_118224_DUP',
- 'prefix_139433_DEL': 'suffix_21321_DEL',
- 'prefix_139627_DEL': 'suffix_228579_DEL',
- 'prefix_141117_DEL': 'suffix_75812_DEL',
- 'prefix_141394_DEL': 'suffix_212333_DEL',
- 'prefix_141485_DEL': 'suffix_158734_DEL',
- 'prefix_141766_DUP': 'suffix_250308_DUP',
- 'prefix_142256_DEL': 'suffix_151736_DEL',
- 'prefix_142808_DEL': 'suffix_50260_DEL',
- 'prefix_143266_DEL': 'suffix_283626_DEL',
- 'prefix_143618_DEL': 'suffix_336978_DEL',
- 'prefix_144319_DEL': 'suffix_12781_DEL',
- 'prefix_144571_DEL': 'suffix_225727_DEL',
- 'prefix_144878_DEL': 'suffix_162955_DEL',
- 'prefix_144935_DEL': 'suffix_88144_DEL',
- 'prefix_14542_DEL': 'suffix_29900_DEL',
- 'prefix_145614_DEL': 'suffix_64370_DEL',
- 'prefix_145631_DUP': 'suffix_213097_DUP',
- 'prefix_145739_DUP': 'suffix_29512_DUP',
- 'prefix_145751_DUP': 'suffix_142338_DUP',
- 'prefix_145859_DEL': 'suffix_236767_DEL',
- 'prefix_146138_DEL': 'suffix_341905_DEL',
- 'prefix_146698_DUP': 'suffix_75558_DUP',
- 'prefix_146725_DEL': 'suffix_92679_DEL',
- 'prefix_146819_DEL': 'suffix_204036_DEL',
- 'prefix_146982_DUP': 'suffix_80598_DUP',
- 'prefix_147832_DUP': 'suffix_36481_DUP',
- 'prefix_147905_DEL': 'suffix_116249_DEL',
- 'prefix_14586_DEL': 'suffix_189167_DEL',
- 'prefix_150663_DUP': 'suffix_169723_DUP',
- 'prefix_150988_DEL': 'suffix_170148_DEL',
- 'prefix_151487_DEL': 'suffix_87868_DEL',
- 'prefix_151896_DUP': 'suffix_298612_DUP',
- 'prefix_151897_DUP': 'suffix_67741_DUP',
- 'prefix_151958_DEL': 'suffix_72228_DEL',
- 'prefix_152188_DEL': 'suffix_258374_DEL',
- 'prefix_152243_DEL': 'suffix_171706_DEL',
- 'prefix_152598_DEL': 'suffix_287152_DEL',
- 'prefix_153441_DUP': 'suffix_181703_DUP',
- 'prefix_15373_DEL': 'suffix_17189_DEL',
- 'prefix_155426_DEL': 'suffix_175579_DEL',
- 'prefix_156182_DEL': 'suffix_12082_DEL',
- 'prefix_157522_DEL': 'suffix_222475_DEL',
- 'prefix_157813_DUP': 'suffix_178342_DUP',
- 'prefix_158523_DUP': 'suffix_313465_DUP',
- 'prefix_158661_DEL': 'suffix_88654_DEL',
- 'prefix_160612_DEL': 'suffix_248116_DEL',
- 'prefix_163949_DUP': 'suffix_344732_DUP',
- 'prefix_164522_DEL': 'suffix_185883_DEL',
- 'prefix_164622_DEL': 'suffix_186016_DEL',
- 'prefix_164629_DUP': 'suffix_309441_DUP',
- 'prefix_164642_DUP': 'suffix_311548_DUP',
- 'prefix_164777_DEL': 'suffix_186196_DEL',
- 'prefix_165076_DEL': 'suffix_328506_DEL',
- 'prefix_165078_DEL': 'suffix_328786_DEL',
- 'prefix_1657_DEL': 'suffix_2017_DEL',
- 'prefix_167202_DEL': 'suffix_336972_DEL',
- 'prefix_168614_DUP': 'suffix_190522_DUP',
- 'prefix_168800_DUP': 'suffix_122588_DUP',
- 'prefix_169366_DEL': 'suffix_191327_DEL',
- 'prefix_170220_DUP': 'suffix_122478_DUP',
- 'prefix_170275_DEL': 'suffix_329992_DEL',
- 'prefix_170760_DEL': 'suffix_192854_DEL',
- 'prefix_171486_DEL': 'suffix_193608_DEL',
- 'prefix_17150_DEL': 'suffix_98009_DEL',
- 'prefix_17230_DEL': 'suffix_336965_DEL',
- 'prefix_17231_DEL': 'suffix_337022_DEL',
- 'prefix_172982_DUP': 'suffix_195278_DUP',
- 'prefix_17317_DEL': 'suffix_275059_DEL',
- 'prefix_173203_DEL': 'suffix_195570_DEL',
- 'prefix_173318_DEL': 'suffix_173502_DEL',
- 'prefix_173699_DUP': 'suffix_127421_DUP',
- 'prefix_173819_DEL': 'suffix_344967_DEL',
- 'prefix_174099_DEL': 'suffix_151801_DEL',
- 'prefix_174146_DEL': 'suffix_193980_DEL',
- 'prefix_174221_DUP': 'suffix_125584_DUP',
- 'prefix_174279_DEL': 'suffix_157330_DEL',
- 'prefix_174372_DEL': 'suffix_262729_DEL',
- 'prefix_174562_DEL': 'suffix_174111_DEL',
- 'prefix_174564_DEL': 'suffix_1324_DEL',
- 'prefix_175140_DEL': 'suffix_272786_DEL',
- 'prefix_175243_DUP': 'suffix_322125_DUP',
- 'prefix_175281_DUP': 'suffix_323384_DUP',
- 'prefix_175291_DUP': 'suffix_323971_DUP',
- 'prefix_176490_DUP': 'suffix_337504_DUP',
- 'prefix_176785_DEL': 'suffix_256937_DEL',
- 'prefix_177587_DUP': 'suffix_56526_DUP',
- 'prefix_177834_DUP': 'suffix_276088_DUP',
- 'prefix_178234_DEL': 'suffix_170832_DEL',
- 'prefix_178374_DUP': 'suffix_173115_DUP',
- 'prefix_178921_DEL': 'suffix_201891_DEL',
- 'prefix_179719_DEL': 'suffix_326378_DEL',
- 'prefix_179787_DEL': 'suffix_337951_DEL',
- 'prefix_179804_DEL': 'suffix_289797_DEL',
- 'prefix_180023_DEL': 'suffix_298661_DEL',
- 'prefix_180149_DUP': 'suffix_203322_DUP',
- 'prefix_180338_DEL': 'suffix_45778_DEL',
- 'prefix_180351_DEL': 'suffix_107299_DEL',
- 'prefix_180464_DEL': 'suffix_342917_DEL',
- 'prefix_180500_DEL': 'suffix_183250_DEL',
- 'prefix_180680_DUP': 'suffix_82979_DUP',
- 'prefix_180721_DEL': 'suffix_4948_DEL',
- 'prefix_181685_DEL': 'suffix_42286_DEL',
- 'prefix_181724_DUP': 'suffix_343954_DUP',
- 'prefix_18172_DUP': 'suffix_94217_DUP',
- 'prefix_182085_DUP': 'suffix_248675_DUP',
- 'prefix_182185_DUP': 'suffix_185453_DUP',
- 'prefix_182381_DEL': 'suffix_65928_DEL',
- 'prefix_182507_DEL': 'suffix_162956_DEL',
- 'prefix_182_DEL': 'suffix_106425_DEL',
- 'prefix_183723_DUP': 'suffix_207297_DUP',
- 'prefix_184282_DUP': 'suffix_301766_DUP',
- 'prefix_184370_DEL': 'suffix_156979_DEL',
- 'prefix_184996_DUP': 'suffix_328901_DUP',
- 'prefix_185023_DEL': 'suffix_15090_DEL',
- 'prefix_185054_DEL': 'suffix_280761_DEL',
- 'prefix_185055_DEL': 'suffix_132024_DEL',
- 'prefix_185138_DEL': 'suffix_127756_DEL',
- 'prefix_185141_DEL': 'suffix_134624_DEL',
- 'prefix_185531_DEL': 'suffix_132027_DEL',
- 'prefix_185615_DEL': 'suffix_217486_DEL',
- 'prefix_185627_DEL': 'suffix_224393_DEL',
- 'prefix_185677_DEL': 'suffix_173146_DEL',
- 'prefix_185750_DUP': 'suffix_102606_DUP',
- 'prefix_185860_DEL': 'suffix_241831_DEL',
- 'prefix_185877_DEL': 'suffix_302969_DEL',
- 'prefix_185935_DEL': 'suffix_173921_DEL',
- 'prefix_186018_DEL': 'suffix_217498_DEL',
- 'prefix_186031_DEL': 'suffix_44013_DEL',
- 'prefix_156507_DEL': 'suffix_176830_DEL',
- 'prefix_186032_DUP': 'suffix_2023_DUP',
- 'prefix_186063_DEL': 'suffix_209934_DEL',
- 'prefix_186624_DUP': 'suffix_210575_DUP',
- 'prefix_186854_DUP': 'suffix_210807_DUP',
- 'prefix_187881_DEL': 'suffix_92683_DEL',
- 'prefix_190360_DEL': 'suffix_214876_DEL',
- 'prefix_191004_DUP': 'suffix_215642_DUP',
- 'prefix_191257_DEL': 'suffix_228609_DEL',
- 'prefix_191292_DUP': 'suffix_234879_DUP',
- 'prefix_191354_DUP': 'suffix_187230_DUP',
- 'prefix_191471_DEL': 'suffix_208014_DEL',
- 'prefix_191474_DEL': 'suffix_280450_DEL',
- 'prefix_191494_DUP': 'suffix_202343_DUP',
- 'prefix_191709_DEL': 'suffix_171576_DEL',
- 'prefix_191829_DUP': 'suffix_217491_DUP',
- 'prefix_191939_DEL': 'suffix_62014_DEL',
- 'prefix_192002_DUP': 'suffix_271568_DUP',
- 'prefix_192090_DEL': 'suffix_132549_DEL',
- 'prefix_192468_DUP': 'suffix_217487_DUP',
- 'prefix_192692_DEL': 'suffix_217799_DEL',
- 'prefix_192750_DEL': 'suffix_217878_DEL',
- 'prefix_192794_DEL': 'suffix_103053_DEL',
- 'prefix_192809_DEL': 'suffix_242803_DEL',
- 'prefix_192811_DUP': 'suffix_288571_DUP',
- 'prefix_192812_DUP': 'suffix_288581_DUP',
- 'prefix_192881_DEL': 'suffix_203645_DEL',
- 'prefix_193000_DEL': 'suffix_327418_DEL',
- 'prefix_193117_DEL': 'suffix_105935_DEL',
- 'prefix_193570_DEL': 'suffix_218935_DEL',
- 'prefix_193750_DEL': 'suffix_171289_DEL',
- 'prefix_193822_DEL': 'suffix_219267_DEL',
- 'prefix_193980_DEL': 'suffix_173513_DEL',
- 'prefix_193998_DEL': 'suffix_342920_DEL',
- 'prefix_193_DEL': 'suffix_175579_DEL',
- 'prefix_195154_DUP': 'suffix_262756_DUP',
- 'prefix_195332_DUP': 'suffix_301004_DUP',
- 'prefix_195356_DEL': 'suffix_67397_DEL',
- 'prefix_195394_DUP': 'suffix_32943_DUP',
- 'prefix_195409_DEL': 'suffix_345343_DEL',
- 'prefix_195446_DUP': 'suffix_218272_DUP',
- 'prefix_195686_DEL': 'suffix_36573_DEL',
- 'prefix_195893_DEL': 'suffix_216802_DEL',
- 'prefix_196343_DEL': 'suffix_187682_DEL',
- 'prefix_196653_DEL': 'suffix_287837_DEL',
- 'prefix_197018_DEL': 'suffix_176475_DEL',
- 'prefix_197343_DEL': 'suffix_329991_DEL',
- 'prefix_197345_DUP': 'suffix_217953_DUP',
- 'prefix_197514_DEL': 'suffix_224017_DEL',
- 'prefix_197601_DUP': 'suffix_92304_DUP',
- 'prefix_197689_DEL': 'suffix_4035_DEL',
- 'prefix_19776_DUP': 'suffix_22049_DUP',
- 'prefix_197839_DEL': 'suffix_37637_DEL',
- 'prefix_197846_DEL': 'suffix_201717_DEL',
- 'prefix_197848_DUP': 'suffix_285533_DUP',
- 'prefix_197857_DEL': 'suffix_33261_DEL',
- 'prefix_197858_DEL': 'suffix_33273_DEL',
- 'prefix_1981_DEL': 'suffix_2420_DEL',
- 'prefix_198681_DEL': 'suffix_225482_DEL',
- 'prefix_198960_DEL': 'suffix_214208_DEL',
- 'prefix_198962_DUP': 'suffix_298013_DUP',
- 'prefix_199230_DUP': 'suffix_219340_DUP',
- 'prefix_199526_DEL': 'suffix_244411_DEL',
- 'prefix_199572_DEL': 'suffix_273734_DEL',
- 'prefix_199578_DEL': 'suffix_132651_DEL',
- 'prefix_201088_DEL': 'suffix_343155_DEL',
- 'prefix_201199_DUP': 'suffix_136419_DUP',
- 'prefix_201221_DEL': 'suffix_79200_DEL',
- 'prefix_201288_DEL': 'suffix_228487_DEL',
- 'prefix_201337_DEL': 'suffix_228541_DEL',
- 'prefix_201565_DEL': 'suffix_168578_DEL',
- 'prefix_202110_DEL': 'suffix_177304_DEL',
- 'prefix_202154_DUP': 'suffix_217485_DUP',
- 'prefix_202256_DEL': 'suffix_331004_DEL',
- 'prefix_202259_DUP': 'suffix_163215_DUP',
- 'prefix_203581_DEL': 'suffix_231189_DEL',
- 'prefix_20401_DEL': 'suffix_22739_DEL',
- 'prefix_204188_DUP': 'suffix_1547_DUP',
- 'prefix_204826_DUP': 'suffix_117758_DUP',
- 'prefix_20504_DUP': 'suffix_61796_DUP',
- 'prefix_205313_DEL': 'suffix_233124_DEL',
- 'prefix_205330_DEL': 'suffix_233146_DEL',
- 'prefix_20552_DEL': 'suffix_162107_DEL',
- 'prefix_20587_DEL': 'suffix_177607_DEL',
- 'prefix_206642_DUP': 'suffix_191476_DUP',
- 'prefix_20842_DUP': 'suffix_247208_DUP',
- 'prefix_208997_DUP': 'suffix_68391_DUP',
- 'prefix_209842_DEL': 'suffix_238144_DEL',
- 'prefix_210581_DEL': 'suffix_282703_DEL',
- 'prefix_210630_DEL': 'suffix_126646_DEL',
- 'prefix_210904_DUP': 'suffix_239309_DUP',
- 'prefix_211888_DEL': 'suffix_337006_DEL',
- 'prefix_212443_DEL': 'suffix_19298_DEL',
- 'prefix_215700_DUP': 'suffix_244607_DUP',
- 'prefix_216096_DEL': 'suffix_271336_DEL',
- 'prefix_21663_DUP': 'suffix_30036_DUP',
- 'prefix_21679_DUP': 'suffix_343012_DUP',
- 'prefix_21680_DEL': 'suffix_344961_DEL',
- 'prefix_218702_DUP': 'suffix_248113_DUP',
- 'prefix_219145_DEL': 'suffix_198229_DEL',
- 'prefix_221051_DEL': 'suffix_32473_DEL',
- 'prefix_22118_DEL': 'suffix_291073_DEL',
- 'prefix_222234_DEL': 'suffix_176476_DEL',
- 'prefix_22347_DUP': 'suffix_336271_DUP',
- 'prefix_223818_DEL': 'suffix_253681_DEL',
- 'prefix_223972_DEL': 'suffix_253871_DEL',
- 'prefix_224087_DEL': 'suffix_254000_DEL',
- 'prefix_22411_DEL': 'suffix_329985_DEL',
- 'prefix_22607_DEL': 'suffix_107271_DEL',
- 'prefix_22608_DEL': 'suffix_107295_DEL',
- 'prefix_22727_DEL': 'suffix_93545_DEL',
- 'prefix_227942_DUP': 'suffix_258345_DUP',
- 'prefix_228561_DEL': 'suffix_247820_DEL',
- 'prefix_228798_DEL': 'suffix_224733_DEL',
- 'prefix_229038_DUP': 'suffix_40155_DUP',
- 'prefix_229254_DUP': 'suffix_345612_DUP',
- 'prefix_229345_DUP': 'suffix_178578_DUP',
- 'prefix_229543_DUP': 'suffix_260172_DUP',
- 'prefix_229784_DEL': 'suffix_4824_DEL',
- 'prefix_229791_DUP': 'suffix_188332_DUP',
- 'prefix_229805_DUP': 'suffix_301762_DUP',
- 'prefix_229839_DEL': 'suffix_39713_DEL',
- 'prefix_229851_DEL': 'suffix_134949_DEL',
- 'prefix_229879_DUP': 'suffix_12928_DUP',
- 'prefix_230003_DEL': 'suffix_107348_DEL',
- 'prefix_230020_DUP': 'suffix_118182_DUP',
- 'prefix_230025_DUP': 'suffix_223544_DUP',
- 'prefix_230090_DUP': 'suffix_332572_DUP',
- 'prefix_230178_DUP': 'suffix_302641_DUP',
- 'prefix_230183_DUP': 'suffix_74958_DUP',
- 'prefix_230251_DEL': 'suffix_133150_DEL',
- 'prefix_230271_DEL': 'suffix_161946_DEL',
- 'prefix_230308_DEL': 'suffix_244610_DEL',
- 'prefix_230350_DEL': 'suffix_252059_DEL',
- 'prefix_230445_DEL': 'suffix_19341_DEL',
- 'prefix_230454_DUP': 'suffix_313052_DUP',
- 'prefix_230603_DEL': 'suffix_77668_DEL',
- 'prefix_230616_DEL': 'suffix_316424_DEL',
- 'prefix_230738_DEL': 'suffix_261522_DEL',
- 'prefix_231752_DEL': 'suffix_262711_DEL',
- 'prefix_231770_DEL': 'suffix_262731_DEL',
- 'prefix_231952_DUP': 'suffix_322130_DUP',
- 'prefix_232113_DUP': 'suffix_85765_DUP',
- 'prefix_232394_DEL': 'suffix_308668_DEL',
- 'prefix_232438_DEL': 'suffix_263531_DEL',
- 'prefix_232444_DEL': 'suffix_263537_DEL',
- 'prefix_232452_DUP': 'suffix_80443_DUP',
- 'prefix_232540_DUP': 'suffix_134297_DUP',
- 'prefix_232595_DEL': 'suffix_189246_DEL',
- 'prefix_232637_DEL': 'suffix_182330_DEL',
- 'prefix_232677_DEL': 'suffix_55000_DEL',
- 'prefix_232776_DUP': 'suffix_262197_DUP',
- 'prefix_232899_DUP': 'suffix_329033_DUP',
- 'prefix_232900_DUP': 'suffix_329084_DUP',
- 'prefix_233045_DEL': 'suffix_150737_DEL',
- 'prefix_233119_DEL': 'suffix_264248_DEL',
- 'prefix_233311_DEL': 'suffix_241927_DEL',
- 'prefix_233321_DEL': 'suffix_344725_DEL',
- 'prefix_233717_DUP': 'suffix_201544_DUP',
- 'prefix_234863_DEL': 'suffix_185898_DEL',
- 'prefix_23489_DEL': 'suffix_331839_DEL',
- 'prefix_234953_DEL': 'suffix_109390_DEL',
- 'prefix_234990_DEL': 'suffix_200659_DEL',
- 'prefix_236431_DUP': 'suffix_267998_DUP',
- 'prefix_237073_DUP': 'suffix_319276_DUP',
- 'prefix_237178_DEL': 'suffix_93427_DEL',
- 'prefix_237197_DEL': 'suffix_65316_DEL',
- 'prefix_237515_DUP': 'suffix_269265_DUP',
- 'prefix_237761_DEL': 'suffix_218136_DEL',
- 'prefix_237780_DUP': 'suffix_237634_DUP',
- 'prefix_237791_DEL': 'suffix_37586_DEL',
- 'prefix_237797_DUP': 'suffix_279247_DUP',
- 'prefix_237833_DUP': 'suffix_118074_DUP',
- 'prefix_237857_DEL': 'suffix_217856_DEL',
- 'prefix_237872_DUP': 'suffix_54996_DUP',
- 'prefix_237874_DEL': 'suffix_172470_DEL',
- 'prefix_237991_DUP': 'suffix_138522_DUP',
- 'prefix_238021_DUP': 'suffix_136966_DUP',
- 'prefix_238147_DUP': 'suffix_270013_DUP',
- 'prefix_238938_DEL': 'suffix_270899_DEL',
- 'prefix_238945_DUP': 'suffix_308856_DUP',
- 'prefix_238961_DEL': 'suffix_310956_DEL',
- 'prefix_239339_DEL': 'suffix_255507_DEL',
- 'prefix_240008_DEL': 'suffix_272099_DEL',
- 'prefix_240186_DEL': 'suffix_264944_DEL',
- 'prefix_240251_DUP': 'suffix_151050_DUP',
- 'prefix_240270_DEL': 'suffix_39288_DEL',
- 'prefix_240483_DEL': 'suffix_39678_DEL',
- 'prefix_241524_DEL': 'suffix_121573_DEL',
- 'prefix_241665_DUP': 'suffix_274132_DUP',
- 'prefix_241760_DEL': 'suffix_274237_DEL',
- 'prefix_242143_DEL': 'suffix_178407_DEL',
- 'prefix_242192_DUP': 'suffix_317053_DUP',
- 'prefix_242247_DUP': 'suffix_345761_DUP',
- 'prefix_242331_DEL': 'suffix_169106_DEL',
- 'prefix_24255_DEL': 'suffix_212769_DEL',
- 'prefix_242646_DEL': 'suffix_212305_DEL',
- 'prefix_242678_DEL': 'suffix_259663_DEL',
- 'prefix_242679_DEL': 'suffix_259723_DEL',
- 'prefix_242681_DEL': 'suffix_259864_DEL',
- 'prefix_242685_DEL': 'suffix_260161_DEL',
- 'prefix_244399_DEL': 'suffix_277206_DEL',
- 'prefix_244573_DUP': 'suffix_277418_DUP',
- 'prefix_24479_DUP': 'suffix_119594_DUP',
- 'prefix_245216_DEL': 'suffix_270267_DEL',
- 'prefix_245217_DEL': 'suffix_89519_DEL',
- 'prefix_245531_DEL': 'suffix_21740_DEL',
- 'prefix_245561_DEL': 'suffix_56540_DEL',
- 'prefix_245634_DUP': 'suffix_70271_DUP',
- 'prefix_245655_DEL': 'suffix_38769_DEL',
- 'prefix_245659_DUP': 'suffix_175204_DUP',
- 'prefix_245721_DUP': 'suffix_199804_DUP',
- 'prefix_245813_DUP': 'suffix_40862_DUP',
- 'prefix_24581_DUP': 'suffix_141949_DUP',
- 'prefix_24583_DEL': 'suffix_217839_DEL',
- 'prefix_245903_DUP': 'suffix_127338_DUP',
- 'prefix_24592_DEL': 'suffix_46695_DEL',
- 'prefix_245933_DUP': 'suffix_279006_DUP',
- 'prefix_24593_DEL': 'suffix_46762_DEL',
- 'prefix_245959_DUP': 'suffix_121405_DUP',
- 'prefix_24595_DEL': 'suffix_46839_DEL',
- 'prefix_245986_DUP': 'suffix_239962_DUP',
- 'prefix_246310_DEL': 'suffix_139843_DEL',
- 'prefix_24637_DEL': 'suffix_185437_DEL',
- 'prefix_24662_DEL': 'suffix_302153_DEL',
- 'prefix_247373_DEL': 'suffix_280718_DEL',
- 'prefix_247836_DUP': 'suffix_309310_DUP',
- 'prefix_247847_DUP': 'suffix_311619_DUP',
- 'prefix_247848_DUP': 'suffix_311683_DUP',
- 'prefix_248168_DEL': 'suffix_132850_DEL',
- 'prefix_24841_DUP': 'suffix_2568_DUP',
- 'prefix_248446_DUP': 'suffix_90044_DUP',
- 'prefix_24870_DEL': 'suffix_337649_DEL',
- 'prefix_248998_DUP': 'suffix_282433_DUP',
- 'prefix_24904_DEL': 'suffix_298012_DEL',
- 'prefix_249320_DUP': 'suffix_227500_DUP',
- 'prefix_249361_DEL': 'suffix_220755_DEL',
- 'prefix_24943_DEL': 'suffix_157892_DEL',
- 'prefix_249451_DUP': 'suffix_6339_DUP',
- 'prefix_250214_DEL': 'suffix_293400_DEL',
- 'prefix_250459_DEL': 'suffix_284644_DEL',
- 'prefix_250460_DEL': 'suffix_69510_DEL',
- 'prefix_250461_DEL': 'suffix_134254_DEL',
- 'prefix_250483_DUP': 'suffix_277241_DUP',
- 'prefix_250509_DEL': 'suffix_218771_DEL',
- 'prefix_250600_DEL': 'suffix_52226_DEL',
- 'prefix_250616_DEL': 'suffix_322013_DEL',
- 'prefix_250674_DUP': 'suffix_304173_DUP',
- 'prefix_250694_DUP': 'suffix_313136_DUP',
- 'prefix_250818_DEL': 'suffix_27205_DEL',
- 'prefix_250819_DEL': 'suffix_27210_DEL',
- 'prefix_250889_DUP': 'suffix_111556_DUP',
- 'prefix_250932_DUP': 'suffix_133045_DUP',
- 'prefix_250981_DEL': 'suffix_91074_DEL',
- 'prefix_251102_DEL': 'suffix_146078_DEL',
- 'prefix_251291_DUP': 'suffix_284913_DUP',
- 'prefix_251847_DEL': 'suffix_224352_DEL',
- 'prefix_251893_DEL': 'suffix_212162_DEL',
- 'prefix_252103_DEL': 'suffix_306862_DEL',
- 'prefix_252152_DEL': 'suffix_252976_DEL',
- 'prefix_252262_DEL': 'suffix_171030_DEL',
- 'prefix_252266_DUP': 'suffix_253194_DUP',
- 'prefix_252268_DEL': 'suffix_127157_DEL',
- 'prefix_25237_DEL': 'suffix_344078_DEL',
- 'prefix_252465_DEL': 'suffix_255499_DEL',
- 'prefix_252468_DUP': 'suffix_61453_DUP',
- 'prefix_252588_DEL': 'suffix_179491_DEL',
- 'prefix_252589_DEL': 'suffix_158777_DEL',
- 'prefix_252657_DEL': 'suffix_332312_DEL',
- 'prefix_252672_DUP': 'suffix_8208_DUP',
- 'prefix_252677_DEL': 'suffix_297350_DEL',
- 'prefix_252719_DEL': 'suffix_118143_DEL',
- 'prefix_253047_DUP': 'suffix_211078_DUP',
- 'prefix_254257_DEL': 'suffix_258839_DEL',
- 'prefix_254295_DEL': 'suffix_37577_DEL',
- 'prefix_254321_DUP': 'suffix_102639_DUP',
- 'prefix_254471_DUP': 'suffix_210560_DUP',
- 'prefix_254486_DEL': 'suffix_61825_DEL',
- 'prefix_254504_DEL': 'suffix_89660_DEL',
- 'prefix_254514_DEL': 'suffix_100226_DEL',
- 'prefix_254902_DUP': 'suffix_165508_DUP',
- 'prefix_254973_DEL': 'suffix_166161_DEL',
- 'prefix_255104_DEL': 'suffix_4167_DEL',
- 'prefix_255420_DUP': 'suffix_72092_DUP',
- 'prefix_255620_DEL': 'suffix_258860_DEL',
- 'prefix_255624_DEL': 'suffix_260164_DEL',
- 'prefix_256004_DEL': 'suffix_211645_DEL',
- 'prefix_257014_DUP': 'suffix_170805_DUP',
- 'prefix_257015_DUP': 'suffix_170872_DUP',
- 'prefix_257016_DUP': 'suffix_170906_DUP',
- 'prefix_257671_DUP': 'suffix_291855_DUP',
- 'prefix_258103_DEL': 'suffix_292360_DEL',
- 'prefix_25895_DEL': 'suffix_105875_DEL',
- 'prefix_259285_DEL': 'suffix_176497_DEL',
- 'prefix_260110_DEL': 'suffix_8146_DEL',
- 'prefix_260118_DEL': 'suffix_106667_DEL',
- 'prefix_260165_DUP': 'suffix_241615_DUP',
- 'prefix_260287_DUP': 'suffix_130663_DUP',
- 'prefix_260388_DUP': 'suffix_68171_DUP',
- 'prefix_260444_DUP': 'suffix_295009_DUP',
- 'prefix_260683_DEL': 'suffix_169109_DEL',
- 'prefix_260715_DEL': 'suffix_100707_DEL',
- 'prefix_260762_DEL': 'suffix_117930_DEL',
- 'prefix_260790_DUP': 'suffix_172476_DUP',
- 'prefix_260802_DEL': 'suffix_323217_DEL',
- 'prefix_260879_DEL': 'suffix_27705_DEL',
- 'prefix_260902_DEL': 'suffix_67106_DEL',
- 'prefix_260908_DEL': 'suffix_129735_DEL',
- 'prefix_260_DEL': 'suffix_55007_DEL',
- 'prefix_261548_DEL': 'suffix_16881_DEL',
- 'prefix_26254_DEL': 'suffix_29316_DEL',
- 'prefix_262735_DEL': 'suffix_297605_DEL',
- 'prefix_263583_DUP': 'suffix_318930_DUP',
- 'prefix_263614_DEL': 'suffix_309554_DEL',
- 'prefix_263688_DUP': 'suffix_287109_DUP',
- 'prefix_263713_DEL': 'suffix_315234_DEL',
- 'prefix_263717_DEL': 'suffix_206974_DEL',
- 'prefix_263844_DEL': 'suffix_93604_DEL',
- 'prefix_264229_DEL': 'suffix_299315_DEL',
- 'prefix_264544_DEL': 'suffix_159878_DEL',
- 'prefix_26468_DEL': 'suffix_29565_DEL',
- 'prefix_264705_DUP': 'suffix_122589_DUP',
- 'prefix_264818_DEL': 'suffix_165079_DEL',
- 'prefix_264891_DUP': 'suffix_113524_DUP',
- 'prefix_265301_DEL': 'suffix_300475_DEL',
- 'prefix_265379_DEL': 'suffix_246254_DEL',
- 'prefix_266533_DEL': 'suffix_301845_DEL',
- 'prefix_266544_DEL': 'suffix_301853_DEL',
- 'prefix_266545_DEL': 'suffix_301855_DEL',
- 'prefix_267_DEL': 'suffix_122539_DEL',
- 'prefix_269805_DEL': 'suffix_305404_DEL',
- 'prefix_2705_DEL': 'suffix_139305_DEL',
- 'prefix_270760_DEL': 'suffix_306509_DEL',
- 'prefix_27114_DEL': 'suffix_150454_DEL',
- 'prefix_271645_DEL': 'suffix_30695_DEL',
- 'prefix_272591_DEL': 'suffix_308628_DEL',
- 'prefix_272819_DEL': 'suffix_308867_DEL',
- 'prefix_276988_DEL': 'suffix_174302_DEL',
- 'prefix_277163_DUP': 'suffix_29468_DUP',
- 'prefix_277765_DEL': 'suffix_190995_DEL',
- 'prefix_277834_DEL': 'suffix_237788_DEL',
- 'prefix_277849_DUP': 'suffix_187676_DUP',
- 'prefix_278532_DEL': 'suffix_224674_DEL',
- 'prefix_278576_DEL': 'suffix_225982_DEL',
- 'prefix_27857_DEL': 'suffix_31144_DEL',
- 'prefix_278620_DEL': 'suffix_61479_DEL',
- 'prefix_278624_DEL': 'suffix_62018_DEL',
- 'prefix_278768_DEL': 'suffix_260377_DEL',
- 'prefix_280607_DUP': 'suffix_49717_DUP',
- 'prefix_280679_DEL': 'suffix_88375_DEL',
- 'prefix_280785_DUP': 'suffix_317588_DUP',
- 'prefix_282546_DUP': 'suffix_188434_DUP',
- 'prefix_282581_DUP': 'suffix_232495_DUP',
- 'prefix_283354_DEL': 'suffix_183799_DEL',
- 'prefix_283359_DEL': 'suffix_139093_DEL',
- 'prefix_283381_DUP': 'suffix_271584_DUP',
- 'prefix_283498_DUP': 'suffix_184079_DUP',
- 'prefix_284364_DEL': 'suffix_16928_DEL',
- 'prefix_2843_DUP': 'suffix_3431_DUP',
- 'prefix_284546_DUP': 'suffix_234318_DUP',
- 'prefix_284653_DEL': 'suffix_29356_DEL',
- 'prefix_284718_DEL': 'suffix_233901_DEL',
- 'prefix_284720_DEL': 'suffix_233993_DEL',
- 'prefix_285131_DEL': 'suffix_322576_DEL',
- 'prefix_285518_DUP': 'suffix_219653_DUP',
- 'prefix_285910_DUP': 'suffix_131091_DUP',
- 'prefix_285938_DEL': 'suffix_114389_DEL',
- 'prefix_285942_DUP': 'suffix_261140_DUP',
- 'prefix_285998_DUP': 'suffix_191729_DUP',
- 'prefix_286350_DUP': 'suffix_151353_DUP',
- 'prefix_286450_DUP': 'suffix_77791_DUP',
- 'prefix_287805_DEL': 'suffix_325622_DEL',
- 'prefix_295024_DUP': 'suffix_333861_DUP',
- 'prefix_295802_DEL': 'suffix_334964_DEL',
- 'prefix_296006_DUP': 'suffix_335221_DUP',
- 'prefix_296764_DUP': 'suffix_336317_DUP',
- 'prefix_297208_DEL': 'suffix_336953_DEL',
- 'prefix_297213_DEL': 'suffix_336958_DEL',
- 'prefix_297223_DEL': 'suffix_336975_DEL',
- 'prefix_191716_DEL': 'suffix_107536_DEL',
- 'prefix_297224_DEL': 'suffix_336978_DEL',
- 'prefix_297279_DEL': 'suffix_337045_DEL',
- 'prefix_297283_DEL': 'suffix_337053_DEL',
- 'prefix_297309_DUP': 'suffix_337088_DUP',
- 'prefix_29997_DEL': 'suffix_223825_DEL',
- 'prefix_301818_DEL': 'suffix_343282_DEL',
- 'prefix_30205_DEL': 'suffix_253777_DEL',
- 'prefix_30217_DEL': 'suffix_264896_DEL',
- 'prefix_302357_DEL': 'suffix_344078_DEL',
- 'prefix_303466_DUP': 'suffix_345668_DUP',
- 'prefix_32087_DEL': 'suffix_9607_DEL',
- 'prefix_32332_DUP': 'suffix_88460_DUP',
- 'prefix_32367_DEL': 'suffix_119716_DEL',
- 'prefix_32706_DEL': 'suffix_36588_DEL',
- 'prefix_33082_DUP': 'suffix_248523_DUP',
- 'prefix_3322_DEL': 'suffix_336956_DEL',
- 'prefix_33597_DUP': 'suffix_37540_DUP',
- 'prefix_33614_DUP': 'suffix_37570_DUP',
- 'prefix_34068_DEL': 'suffix_38077_DEL',
- 'prefix_34213_DUP': 'suffix_92153_DUP',
- 'prefix_34545_DEL': 'suffix_38571_DEL',
- 'prefix_34597_DEL': 'suffix_109532_DEL',
- 'prefix_34869_DEL': 'suffix_87992_DEL',
- 'prefix_35040_DEL': 'suffix_27009_DEL',
- 'prefix_35106_DEL': 'suffix_237735_DEL',
- 'prefix_35180_DUP': 'suffix_190057_DUP',
- 'prefix_35775_DEL': 'suffix_39951_DEL',
- 'prefix_36042_DUP': 'suffix_42_DUP',
- 'prefix_36108_DUP': 'suffix_339711_DUP',
- 'prefix_36111_DEL': 'suffix_297767_DEL',
- 'prefix_3617_DEL': 'suffix_183767_DEL',
- 'prefix_36425_DEL': 'suffix_109536_DEL',
- 'prefix_36528_DEL': 'suffix_338271_DEL',
- 'prefix_36589_DUP': 'suffix_225864_DUP',
- 'prefix_36614_DEL': 'suffix_36479_DEL',
- 'prefix_36746_DUP': 'suffix_72227_DUP',
- 'prefix_36846_DUP': 'suffix_107654_DUP',
- 'prefix_37079_DUP': 'suffix_245784_DUP',
- 'prefix_37080_DUP': 'suffix_245791_DUP',
- 'prefix_37081_DUP': 'suffix_245798_DUP',
- 'prefix_37082_DUP': 'suffix_246118_DUP',
- 'prefix_37094_DUP': 'suffix_247108_DUP',
- 'prefix_37095_DUP': 'suffix_247270_DUP',
- 'prefix_37096_DUP': 'suffix_247352_DUP',
- 'prefix_37098_DUP': 'suffix_247400_DUP',
- 'prefix_37100_DUP': 'suffix_247574_DUP',
- 'prefix_37108_DUP': 'suffix_248111_DUP',
- 'prefix_37112_DUP': 'suffix_248141_DUP',
- 'prefix_37282_DEL': 'suffix_32945_DEL',
- 'prefix_37834_DEL': 'suffix_42257_DEL',
- 'prefix_3816_DEL': 'suffix_265546_DEL',
- 'prefix_3852_DEL': 'suffix_282714_DEL',
- 'prefix_39151_DEL': 'suffix_336958_DEL',
- 'prefix_39601_DEL': 'suffix_337053_DEL',
- 'prefix_39656_DEL': 'suffix_221906_DEL',
- 'prefix_3991_DEL': 'suffix_317434_DEL',
- 'prefix_40087_DUP': 'suffix_198027_DUP',
- 'prefix_41276_DUP': 'suffix_46123_DUP',
- 'prefix_41357_DEL': 'suffix_121015_DEL',
- 'prefix_42493_DUP': 'suffix_118615_DUP',
- 'prefix_42547_DUP': 'suffix_47670_DUP',
- 'prefix_42549_DEL': 'suffix_47672_DEL',
- 'prefix_43054_DEL': 'suffix_253999_DEL',
- 'prefix_43130_DEL': 'suffix_48422_DEL',
- 'prefix_43183_DEL': 'suffix_48494_DEL',
- 'prefix_43482_DEL': 'suffix_29927_DEL',
- 'prefix_43657_DEL': 'suffix_49127_DEL',
- 'prefix_44112_DEL': 'suffix_12769_DEL',
- 'prefix_44349_DEL': 'suffix_234536_DEL',
- 'prefix_44363_DEL': 'suffix_235902_DEL',
- 'prefix_45800_DEL': 'suffix_182593_DEL',
- 'prefix_45862_DEL': 'suffix_51654_DEL',
- 'prefix_45965_DUP': 'suffix_140827_DUP',
- 'prefix_46144_DEL': 'suffix_299340_DEL',
- 'prefix_46154_DUP': 'suffix_308320_DUP',
- 'prefix_46234_DEL': 'suffix_329984_DEL',
- 'prefix_46275_DEL': 'suffix_292777_DEL',
- 'prefix_46406_DUP': 'suffix_52251_DUP',
- 'prefix_46447_DEL': 'suffix_22508_DEL',
- 'prefix_4670_DEL': 'suffix_52153_DEL',
- 'prefix_46723_DEL': 'suffix_297314_DEL',
- 'prefix_4687_DEL': 'suffix_337096_DEL',
- 'prefix_46883_DEL': 'suffix_322347_DEL',
- 'prefix_46902_DEL': 'suffix_16522_DEL',
- 'prefix_46949_DEL': 'suffix_221643_DEL',
- 'prefix_47442_DEL': 'suffix_32709_DEL',
- 'prefix_47443_DEL': 'suffix_32942_DEL',
- 'prefix_47446_DEL': 'suffix_33262_DEL',
- 'prefix_47486_DUP': 'suffix_341592_DUP',
- 'prefix_47629_DUP': 'suffix_309398_DUP',
- 'prefix_47726_DEL': 'suffix_290347_DEL',
- 'prefix_47796_DEL': 'suffix_1449_DEL',
- 'prefix_477_DUP': 'suffix_59131_DUP',
- 'prefix_4783_DUP': 'suffix_335171_DUP',
- 'prefix_47920_DUP': 'suffix_69500_DUP',
- 'prefix_48061_DUP': 'suffix_297897_DUP',
- 'prefix_48220_DEL': 'suffix_234836_DEL',
- 'prefix_48353_DUP': 'suffix_84392_DUP',
- 'prefix_49235_DEL': 'suffix_39714_DEL',
- 'prefix_4948_DEL': 'suffix_5834_DEL',
- 'prefix_49781_DEL': 'suffix_56043_DEL',
- 'prefix_49980_DUP': 'suffix_56297_DUP',
- 'prefix_50167_DEL': 'suffix_56527_DEL',
- 'prefix_51403_DEL': 'suffix_13373_DEL',
- 'prefix_51897_DEL': 'suffix_336964_DEL',
- 'prefix_52447_DEL': 'suffix_179521_DEL',
- 'prefix_53019_DEL': 'suffix_249573_DEL',
- 'prefix_53360_DEL': 'suffix_28019_DEL',
- 'prefix_53373_DEL': 'suffix_169978_DEL',
- 'prefix_54115_DEL': 'suffix_124254_DEL',
- 'prefix_5415_DEL': 'suffix_274987_DEL',
- 'prefix_55334_DEL': 'suffix_182678_DEL',
- 'prefix_55463_DUP': 'suffix_62593_DUP',
- 'prefix_5580_DEL': 'suffix_106522_DEL',
- 'prefix_56296_DUP': 'suffix_20205_DUP',
- 'prefix_56375_DEL': 'suffix_122348_DEL',
- 'prefix_56758_DEL': 'suffix_133934_DEL',
- 'prefix_5719_DUP': 'suffix_46123_DUP',
- 'prefix_57494_DEL': 'suffix_64715_DEL',
- 'prefix_5756_DEL': 'suffix_195683_DEL',
- 'prefix_58444_DUP': 'suffix_65759_DUP',
- 'prefix_59270_DEL': 'suffix_88382_DEL',
- 'prefix_598_DEL': 'suffix_729_DEL',
- 'prefix_59944_DEL': 'suffix_105066_DEL',
- 'prefix_60675_DEL': 'suffix_215556_DEL',
- 'prefix_61714_DUP': 'suffix_69571_DUP',
- 'prefix_62048_DEL': 'suffix_220763_DEL',
- 'prefix_63074_DEL': 'suffix_262731_DEL',
- 'prefix_63258_DEL': 'suffix_200662_DEL',
- 'prefix_6325_DEL': 'suffix_282673_DEL',
- 'prefix_63634_DEL': 'suffix_192720_DEL',
- 'prefix_6643_DEL': 'suffix_131071_DEL',
- 'prefix_67724_DEL': 'suffix_75807_DEL',
- 'prefix_67761_DEL': 'suffix_300474_DEL',
- 'prefix_68258_DEL': 'suffix_183179_DEL',
- 'prefix_68433_DEL': 'suffix_241887_DEL',
- 'prefix_69473_DEL': 'suffix_213975_DEL',
- 'prefix_70643_DEL': 'suffix_173247_DEL',
- 'prefix_70766_DEL': 'suffix_255510_DEL',
- 'prefix_70923_DUP': 'suffix_64371_DUP',
- 'prefix_71010_DEL': 'suffix_298994_DEL',
- 'prefix_71023_DEL': 'suffix_102982_DEL',
- 'prefix_71045_DEL': 'suffix_262680_DEL',
- 'prefix_71163_DEL': 'suffix_79602_DEL',
- 'prefix_7128_DEL': 'suffix_340564_DEL',
- 'prefix_71349_DUP': 'suffix_310205_DUP',
- 'prefix_71400_DUP': 'suffix_88019_DUP',
- 'prefix_71606_DUP': 'suffix_220541_DUP',
- 'prefix_71676_DUP': 'suffix_327952_DUP',
- 'prefix_71691_DEL': 'suffix_131549_DEL',
- 'prefix_718_DEL': 'suffix_263531_DEL',
- 'prefix_71936_DUP': 'suffix_116525_DUP',
- 'prefix_71961_DEL': 'suffix_140125_DEL',
- 'prefix_71969_DUP': 'suffix_157089_DUP',
- 'prefix_72062_DUP': 'suffix_253345_DUP',
- 'prefix_72143_DEL': 'suffix_172606_DEL',
- 'prefix_72208_DEL': 'suffix_197482_DEL',
- 'prefix_72369_DEL': 'suffix_115813_DEL',
- 'prefix_73088_DEL': 'suffix_81641_DEL',
- 'prefix_73094_DUP': 'suffix_185884_DUP',
- 'prefix_73451_DEL': 'suffix_307441_DEL',
- 'prefix_7358_DEL': 'suffix_337112_DEL',
- 'prefix_74031_DUP': 'suffix_134917_DUP',
- 'prefix_74343_DUP': 'suffix_83006_DUP',
- 'prefix_74628_DEL': 'suffix_208625_DEL',
- 'prefix_74629_DUP': 'suffix_5338_DUP',
- 'prefix_74896_DEL': 'suffix_226397_DEL',
- 'prefix_75369_DEL': 'suffix_136965_DEL',
- 'prefix_75422_DEL': 'suffix_84245_DEL',
- 'prefix_75547_DEL': 'suffix_126315_DEL',
- 'prefix_75551_DUP': 'suffix_155780_DUP',
- 'prefix_75625_DUP': 'suffix_310853_DUP',
- 'prefix_7613_DEL': 'suffix_247991_DEL',
- 'prefix_7837_DUP': 'suffix_9141_DUP',
- 'prefix_78409_DUP': 'suffix_159950_DUP',
- 'prefix_78419_DEL': 'suffix_87809_DEL',
- 'prefix_78493_DUP': 'suffix_87885_DUP',
- 'prefix_78575_DEL': 'suffix_310958_DEL',
- 'prefix_78580_DEL': 'suffix_312313_DEL',
- 'prefix_78692_DEL': 'suffix_332978_DEL',
- 'prefix_78775_DUP': 'suffix_178999_DUP',
- 'prefix_78808_DUP': 'suffix_88248_DUP',
- 'prefix_78890_DEL': 'suffix_203497_DEL',
- 'prefix_78923_DUP': 'suffix_305735_DUP',
- 'prefix_78942_DUP': 'suffix_307917_DUP',
- 'prefix_78949_DEL': 'suffix_308466_DEL',
- 'prefix_78964_DUP': 'suffix_244587_DUP',
- 'prefix_79085_DUP': 'suffix_319372_DUP',
- 'prefix_79632_DUP': 'suffix_260100_DUP',
- 'prefix_80034_DUP': 'suffix_298803_DUP',
- 'prefix_8058_DUP': 'suffix_177412_DUP',
- 'prefix_80933_DEL': 'suffix_90592_DEL',
- 'prefix_8121_DUP': 'suffix_9447_DUP',
- 'prefix_816_DUP': 'suffix_986_DUP',
- 'prefix_81889_DEL': 'suffix_91604_DEL',
- 'prefix_8206_DUP': 'suffix_170741_DUP',
- 'prefix_7357_DEL': 'suffix_337078_DEL',
- 'prefix_82314_DUP': 'suffix_89153_DUP',
- 'prefix_82745_DEL': 'suffix_95957_DEL',
- 'prefix_83209_DUP': 'suffix_256121_DUP',
- 'prefix_83230_DUP': 'suffix_57589_DUP',
- 'prefix_83245_DUP': 'suffix_49612_DUP',
- 'prefix_83313_DEL': 'suffix_31491_DEL',
- 'prefix_83315_DEL': 'suffix_93129_DEL',
- 'prefix_83649_DEL': 'suffix_105765_DEL',
- 'prefix_83650_DEL': 'suffix_105792_DEL',
- 'prefix_83651_DEL': 'suffix_105804_DEL',
- 'prefix_83935_DEL': 'suffix_336309_DEL',
- 'prefix_83975_DUP': 'suffix_134029_DUP',
- 'prefix_83980_DEL': 'suffix_242921_DEL',
- 'prefix_83984_DUP': 'suffix_32384_DUP',
- 'prefix_84015_DUP': 'suffix_271757_DUP',
- 'prefix_84044_DEL': 'suffix_263835_DEL',
- 'prefix_8420_DEL': 'suffix_75588_DEL',
- 'prefix_84288_DUP': 'suffix_338248_DUP',
- 'prefix_84403_DUP': 'suffix_345428_DUP',
- 'prefix_84717_DEL': 'suffix_94529_DEL',
- 'prefix_85642_DEL': 'suffix_236439_DEL',
- 'prefix_86089_DEL': 'suffix_109334_DEL',
- 'prefix_86353_DEL': 'suffix_12713_DEL',
- 'prefix_86387_DEL': 'suffix_24837_DEL',
- 'prefix_87733_DEL': 'suffix_147400_DEL',
- 'prefix_87760_DEL': 'suffix_337119_DEL',
- 'prefix_88833_DUP': 'suffix_37585_DUP',
- 'prefix_88886_DEL': 'suffix_98841_DEL',
- 'prefix_889_DUP': 'suffix_1064_DUP',
- 'prefix_8950_DEL': 'suffix_314461_DEL',
- 'prefix_9017_DEL': 'suffix_344075_DEL',
- 'prefix_90242_DUP': 'suffix_340596_DUP',
- 'prefix_9027_DEL': 'suffix_10504_DEL',
- 'prefix_90299_DEL': 'suffix_170093_DEL',
- 'prefix_92044_DUP': 'suffix_229729_DUP',
- 'prefix_92416_DEL': 'suffix_41456_DEL',
- 'prefix_92442_DUP': 'suffix_140124_DUP',
- 'prefix_92506_DUP': 'suffix_244044_DUP',
- 'prefix_92527_DUP': 'suffix_308955_DUP',
- 'prefix_92550_DEL': 'suffix_336957_DEL',
- 'prefix_92558_DUP': 'suffix_309329_DUP',
- 'prefix_92559_DUP': 'suffix_309345_DUP',
- 'prefix_92627_DUP': 'suffix_342164_DUP',
- 'prefix_92640_DEL': 'suffix_256704_DEL',
- 'prefix_92708_DUP': 'suffix_259795_DUP',
- 'prefix_92724_DEL': 'suffix_252290_DEL',
- 'prefix_99990_DUP': 'suffix_306887_DUP',
- 'prefix_99998_DUP': 'suffix_308100_DUP',
- 'suffix_102137_DEL_2': 'suffix_105232_DEL',
- 'suffix_102188_DEL_2': 'suffix_105284_DEL',
- 'suffix_102279_DEL_2': 'suffix_105380_DEL',
- 'suffix_10229_DUP_2': 'suffix_10602_DUP',
- 'suffix_102741_DUP_2': 'suffix_105859_DUP',
- 'suffix_103521_DUP_2': 'suffix_106670_DUP',
- 'suffix_103609_DEL_2': 'suffix_106761_DEL',
- 'suffix_106849_DUP_2': 'suffix_110077_DUP',
- 'suffix_107254_DUP_2': 'suffix_110487_DUP',
- 'suffix_107284_DEL_2': 'suffix_110517_DEL',
- 'suffix_108892_DUP_2': 'suffix_112158_DUP',
- 'suffix_112876_DUP_2': 'suffix_116262_DUP',
- 'suffix_115899_DUP_2': 'suffix_119375_DUP',
- 'suffix_116678_DUP_2': 'suffix_120179_DUP',
- 'suffix_116680_DEL_2': 'suffix_120181_DEL',
- 'suffix_1179_DEL_2': 'suffix_1216_DEL',
- 'suffix_118089_DEL_2': 'suffix_121646_DEL',
- 'suffix_118753_DUP_2': 'suffix_122349_DUP',
- 'suffix_118957_DUP_2': 'suffix_122563_DUP',
- 'suffix_121705_DUP_2': 'suffix_125377_DUP',
- 'suffix_121728_DUP_2': 'suffix_125401_DUP',
- 'suffix_123193_DEL_2': 'suffix_126890_DEL',
- 'suffix_12347_DEL_2': 'suffix_12771_DEL',
- 'suffix_123507_DEL_2': 'suffix_127211_DEL',
- 'suffix_123859_DUP_2': 'suffix_127567_DUP',
- 'suffix_126255_DEL_2': 'suffix_130031_DEL',
- 'suffix_126378_DEL_2': 'suffix_130158_DEL',
- 'suffix_126871_DEL_2': 'suffix_130662_DEL',
- 'suffix_127257_DEL_2': 'suffix_131070_DEL',
- 'suffix_127603_DUP_2': 'suffix_131424_DUP',
- 'suffix_127635_DUP_2': 'suffix_131456_DUP',
- 'suffix_128475_DEL_2': 'suffix_132338_DEL',
- 'suffix_130874_DUP_2': 'suffix_134810_DUP',
- 'suffix_133323_DUP_2': 'suffix_137335_DUP',
- 'suffix_137071_DUP_2': 'suffix_141198_DUP',
- 'suffix_1371_DUP_2': 'suffix_1425_DUP',
- 'suffix_137756_DUP_2': 'suffix_141889_DUP',
- 'suffix_138407_DUP_2': 'suffix_142558_DUP',
- 'suffix_139177_DUP_2': 'suffix_143347_DUP',
- 'suffix_139287_DEL_2': 'suffix_143460_DEL',
- 'suffix_140067_DUP_2': 'suffix_144262_DUP',
- 'suffix_140211_DEL_2': 'suffix_144411_DEL',
- 'suffix_140216_DUP_2': 'suffix_144417_DUP',
- 'suffix_140998_DUP_2': 'suffix_145233_DUP',
- 'suffix_143072_DUP_2': 'suffix_147367_DUP',
- 'suffix_144703_DUP_2': 'suffix_149060_DUP',
- 'suffix_146079_DEL_2': 'suffix_150480_DEL',
- 'suffix_146082_DEL_2': 'suffix_150483_DEL',
- 'suffix_146309_DEL_2': 'suffix_150716_DEL',
- 'suffix_146679_DEL_2': 'suffix_151094_DEL',
- 'suffix_147023_DUP_2': 'suffix_151455_DUP',
- 'suffix_147062_DUP_2': 'suffix_151496_DUP',
- 'suffix_147065_DEL_2': 'suffix_151498_DEL',
- 'suffix_147755_DUP_2': 'suffix_152214_DUP',
- 'suffix_148894_DUP_2': 'suffix_153392_DUP',
- 'suffix_150694_DUP_2': 'suffix_155261_DUP',
- 'suffix_152328_DUP_2': 'suffix_156958_DUP',
- 'suffix_152423_DUP_2': 'suffix_157059_DUP',
- 'suffix_152437_DUP_2': 'suffix_157073_DUP',
- 'suffix_152439_DEL_2': 'suffix_157075_DEL',
- 'suffix_153155_DUP_2': 'suffix_157821_DUP',
- 'suffix_153923_DEL_2': 'suffix_158633_DEL',
- 'suffix_15410_DEL_2': 'suffix_15899_DEL',
- 'suffix_15414_DEL_2': 'suffix_15903_DEL',
- 'suffix_154873_DEL_2': 'suffix_159630_DEL',
- 'suffix_156350_DUP_2': 'suffix_161149_DUP',
- 'suffix_156426_DUP_2': 'suffix_161225_DUP',
- 'suffix_157651_DUP_2': 'suffix_162486_DUP',
- 'suffix_161505_DUP_2': 'suffix_166461_DUP',
- 'suffix_162350_DUP_2': 'suffix_167337_DUP',
- 'suffix_16449_DEL_2': 'suffix_16965_DEL',
- 'suffix_16476_DEL_2': 'suffix_16992_DEL',
- 'suffix_164891_DUP_2': 'suffix_169964_DUP',
- 'suffix_165535_DEL_2': 'suffix_170615_DEL',
- 'suffix_167366_DUP_2': 'suffix_172492_DUP',
- 'suffix_167382_DUP_2': 'suffix_172508_DUP',
- 'suffix_169697_DUP_2': 'suffix_174892_DUP',
- 'suffix_170359_DEL_2': 'suffix_175579_DEL',
- 'suffix_171232_DEL_2': 'suffix_176479_DEL',
- 'suffix_171251_DEL_2': 'suffix_176499_DEL',
- 'suffix_171281_DEL_2': 'suffix_176529_DEL',
- 'suffix_172345_DUP_2': 'suffix_177610_DUP',
- 'suffix_173330_DUP_2': 'suffix_178634_DUP',
- 'suffix_173374_DUP_2': 'suffix_178679_DUP',
- 'suffix_173898_DEL_2': 'suffix_179214_DEL',
- 'suffix_176174_DUP_2': 'suffix_181566_DUP',
- 'suffix_176223_DUP_2': 'suffix_181617_DUP',
- 'suffix_177584_DEL_2': 'suffix_183019_DEL',
- 'suffix_177830_DUP_2': 'suffix_183274_DUP',
- 'suffix_178098_DUP_2': 'suffix_183558_DUP',
- 'suffix_178276_DEL_2': 'suffix_183739_DEL',
- 'suffix_178301_DEL_2': 'suffix_183764_DEL',
- 'suffix_179460_DEL_2': 'suffix_184960_DEL',
- 'suffix_180373_DEL_2': 'suffix_185895_DEL',
- 'suffix_180638_DEL_2': 'suffix_186170_DEL',
- 'suffix_180645_DUP_2': 'suffix_186177_DUP',
- 'suffix_180670_DUP_2': 'suffix_186202_DUP',
- 'suffix_182096_DEL_2': 'suffix_187682_DEL',
- 'suffix_182350_DUP_2': 'suffix_187944_DUP',
- 'suffix_182381_DEL_2': 'suffix_187975_DEL',
- 'suffix_183117_DUP_2': 'suffix_188748_DUP',
- 'suffix_183487_DUP_2': 'suffix_189139_DUP',
- 'suffix_183693_DUP_2': 'suffix_189354_DUP',
- 'suffix_184465_DUP_2': 'suffix_190174_DUP',
- 'suffix_186206_DUP_2': 'suffix_191986_DUP',
- 'suffix_186235_DEL_2': 'suffix_192019_DEL',
- 'suffix_186644_DUP_2': 'suffix_192445_DUP',
- 'suffix_186658_DEL_2': 'suffix_192460_DEL',
- 'suffix_187547_DEL_2': 'suffix_193377_DEL',
- 'suffix_188120_DEL_2': 'suffix_193959_DEL',
- 'suffix_188699_DEL_2': 'suffix_194556_DEL',
- 'suffix_189196_DUP_2': 'suffix_195072_DUP',
- 'suffix_189361_DEL_2': 'suffix_195245_DEL',
- 'suffix_189787_DEL_2': 'suffix_195683_DEL',
- 'suffix_189988_DUP_2': 'suffix_195887_DUP',
- 'suffix_192255_DEL_2': 'suffix_198235_DEL',
- 'suffix_194657_DUP_2': 'suffix_200737_DUP',
- 'suffix_195491_DUP_2': 'suffix_201616_DUP',
- 'suffix_196079_DUP_2': 'suffix_202227_DUP',
- 'suffix_197227_DUP_2': 'suffix_203421_DUP',
- 'suffix_197905_DUP_2': 'suffix_204137_DUP',
- 'suffix_199236_DUP_2': 'suffix_205495_DUP',
- 'suffix_199351_DUP_2': 'suffix_205610_DUP',
- 'suffix_199613_DUP_2': 'suffix_205874_DUP',
- 'suffix_199853_DEL_2': 'suffix_206119_DEL',
- 'suffix_200568_DEL_2': 'suffix_206849_DEL',
- 'suffix_200934_DUP_2': 'suffix_207220_DUP',
- 'suffix_2015_DUP_2': 'suffix_2089_DUP',
- 'suffix_201955_DUP_2': 'suffix_208285_DUP',
- 'suffix_202641_DUP_2': 'suffix_208994_DUP',
- 'suffix_205028_DUP_2': 'suffix_211453_DUP',
- 'suffix_205123_DEL_2': 'suffix_211548_DEL',
- 'suffix_205138_DEL_2': 'suffix_211564_DEL',
- 'suffix_206613_DEL_2': 'suffix_213095_DEL',
- 'suffix_206780_DEL_2': 'suffix_213264_DEL',
- 'suffix_207068_DEL_2': 'suffix_213553_DEL',
- 'suffix_207138_DEL_2': 'suffix_213623_DEL',
- 'suffix_207672_DEL_2': 'suffix_214167_DEL',
- 'suffix_207715_DEL_2': 'suffix_214213_DEL',
- 'suffix_208370_DEL_2': 'suffix_214895_DEL',
- 'suffix_208573_DEL_2': 'suffix_215104_DEL',
- 'suffix_208972_DEL_2': 'suffix_215523_DEL',
- 'suffix_209068_DEL_2': 'suffix_215620_DEL',
- 'suffix_209199_DUP_2': 'suffix_215756_DUP',
- 'suffix_210259_DUP_2': 'suffix_216845_DUP',
- 'suffix_210283_DUP_2': 'suffix_216871_DUP',
- 'suffix_210613_DEL_2': 'suffix_217209_DEL',
- 'suffix_212296_DEL_2': 'suffix_218955_DEL',
- 'suffix_212578_DEL_2': 'suffix_219252_DEL',
- 'suffix_212637_DEL_2': 'suffix_219317_DEL',
- 'suffix_212756_DEL_2': 'suffix_219444_DEL',
- 'suffix_213708_DEL_2': 'suffix_220434_DEL',
- 'suffix_213875_DEL_2': 'suffix_220605_DEL',
- 'suffix_214152_DEL_2': 'suffix_220894_DEL',
- 'suffix_214429_DEL_2': 'suffix_221177_DEL',
- 'suffix_214857_DEL_2': 'suffix_221627_DEL',
- 'suffix_216188_DEL_2': 'suffix_223018_DEL',
- 'suffix_216198_DEL_2': 'suffix_223028_DEL',
- 'suffix_216958_DUP_2': 'suffix_223818_DUP',
- 'suffix_217583_DEL_2': 'suffix_224470_DEL',
- 'suffix_217868_DEL_2': 'suffix_224762_DEL',
- 'suffix_217871_DEL_2': 'suffix_224765_DEL',
- 'suffix_218073_DUP_2': 'suffix_224973_DUP',
- 'suffix_218439_DUP_2': 'suffix_225366_DUP',
- 'suffix_218904_DUP_2': 'suffix_225864_DUP',
- 'suffix_219658_DUP_2': 'suffix_226657_DUP',
- 'suffix_220370_DEL_2': 'suffix_227413_DEL',
- 'suffix_223567_DUP_2': 'suffix_230759_DUP',
- 'suffix_223841_DUP_2': 'suffix_231044_DUP',
- 'suffix_22451_DEL_2': 'suffix_23110_DEL',
- 'suffix_224608_DEL_2': 'suffix_231845_DEL',
- 'suffix_226755_DEL_2': 'suffix_234078_DEL',
- 'suffix_227182_DEL_2': 'suffix_234514_DEL',
- 'suffix_227203_DEL_2': 'suffix_234535_DEL',
- 'suffix_227204_DEL_2': 'suffix_234536_DEL',
- 'suffix_228485_DEL_2': 'suffix_235865_DEL',
- 'suffix_2306_DEL_2': 'suffix_2387_DEL',
- 'suffix_230714_DUP_2': 'suffix_238195_DUP',
- 'suffix_230932_DEL_2': 'suffix_238422_DEL',
- 'suffix_231136_DEL_2': 'suffix_238630_DEL',
- 'suffix_234299_DUP_2': 'suffix_241888_DUP',
- 'suffix_236708_DEL_2': 'suffix_244400_DEL',
- 'suffix_236887_DEL_2': 'suffix_244582_DEL',
- 'suffix_237084_DEL_2': 'suffix_244791_DEL',
- 'suffix_237399_DEL_2': 'suffix_245122_DEL',
- 'suffix_238815_DEL_2': 'suffix_246599_DEL',
- 'suffix_238984_DUP_2': 'suffix_246775_DUP',
- 'suffix_239194_DEL_2': 'suffix_246989_DEL',
- 'suffix_239311_DUP_2': 'suffix_247109_DUP',
- 'suffix_240516_DUP_2': 'suffix_248341_DUP',
- 'suffix_24067_DEL_2': 'suffix_24774_DEL',
- 'suffix_24112_DEL_2': 'suffix_24819_DEL',
- 'suffix_245147_DUP_2': 'suffix_253116_DUP',
- 'suffix_245631_DUP_2': 'suffix_253615_DUP',
- 'suffix_246007_DEL_2': 'suffix_254014_DEL',
- 'suffix_246832_DUP_2': 'suffix_254855_DUP',
- 'suffix_249752_DEL_2': 'suffix_257879_DEL',
- 'suffix_249760_DUP_2': 'suffix_257887_DUP',
- 'suffix_250396_DUP_2': 'suffix_258549_DUP',
- 'suffix_251212_DEL_2': 'suffix_259389_DEL',
- 'suffix_251719_DUP_2': 'suffix_259912_DUP',
- 'suffix_252163_DEL_2': 'suffix_260375_DEL',
- 'suffix_254429_DEL_2': 'suffix_262732_DEL',
- 'suffix_254575_DEL_2': 'suffix_262881_DEL',
- 'suffix_255204_DEL_2': 'suffix_263531_DEL',
- 'suffix_25533_DUP_2': 'suffix_26296_DUP',
- 'suffix_256655_DEL_2': 'suffix_265034_DEL',
- 'suffix_256744_DEL_2': 'suffix_265124_DEL',
- 'suffix_256775_DEL_2': 'suffix_265157_DEL',
- 'suffix_25949_DEL_2': 'suffix_26718_DEL',
- 'suffix_260357_DEL_2': 'suffix_268858_DEL',
- 'suffix_260853_DUP_2': 'suffix_269376_DUP',
- 'suffix_260855_DEL_2': 'suffix_269379_DEL',
- 'suffix_261034_DEL_2': 'suffix_269563_DEL',
- 'suffix_262533_DEL_2': 'suffix_271118_DEL',
- 'suffix_262539_DEL_2': 'suffix_271125_DEL',
- 'suffix_26417_DUP_2': 'suffix_27197_DUP',
- 'suffix_265005_DEL_2': 'suffix_273689_DEL',
- 'suffix_266230_DEL_2': 'suffix_274984_DEL',
- 'suffix_268706_DUP_2': 'suffix_277538_DUP',
- 'suffix_269299_DEL_2': 'suffix_278150_DEL',
- 'suffix_27000_DUP_2': 'suffix_27795_DUP',
- 'suffix_27116_DEL_2': 'suffix_27917_DEL',
- 'suffix_272983_DEL_2': 'suffix_281956_DEL',
- 'suffix_273392_DEL_2': 'suffix_282368_DEL',
- 'suffix_273424_DUP_2': 'suffix_282400_DUP',
- 'suffix_273728_DEL_2': 'suffix_282706_DEL',
- 'suffix_273736_DEL_2': 'suffix_282714_DEL',
- 'suffix_274940_DEL_2': 'suffix_283943_DEL',
- 'suffix_276413_DEL_2': 'suffix_285456_DEL',
- 'suffix_27644_DEL_2': 'suffix_28486_DEL',
- 'suffix_277451_DEL_2': 'suffix_286527_DEL',
- 'suffix_278695_DEL_2': 'suffix_287788_DEL',
- 'suffix_279450_DEL_2': 'suffix_288554_DEL',
- 'suffix_279452_DEL_2': 'suffix_288556_DEL',
- 'suffix_280776_DEL_2': 'suffix_289942_DEL',
- 'suffix_280836_DUP_2': 'suffix_290002_DUP',
- 'suffix_281014_DEL_2': 'suffix_290187_DEL',
- 'suffix_281363_DUP_2': 'suffix_290550_DUP',
- 'suffix_281846_DEL_2': 'suffix_291053_DEL',
- 'suffix_282575_DUP_2': 'suffix_291822_DUP',
- 'suffix_283188_DEL_2': 'suffix_292459_DEL',
- 'suffix_283756_DEL_2': 'suffix_293051_DEL',
- 'suffix_284360_DEL_2': 'suffix_293684_DEL',
- 'suffix_284797_DEL_2': 'suffix_294141_DEL',
- 'suffix_286080_DUP_2': 'suffix_295482_DUP',
- 'suffix_286258_DUP_2': 'suffix_295667_DUP',
- 'suffix_286295_DEL_2': 'suffix_295706_DEL',
- 'suffix_286314_DUP_2': 'suffix_295727_DUP',
- 'suffix_286315_DEL_2': 'suffix_295726_DEL',
- 'suffix_288518_DEL_2': 'suffix_298011_DEL',
- 'suffix_288856_DUP_2': 'suffix_298358_DUP',
- 'suffix_288968_DEL_2': 'suffix_298482_DEL',
- 'suffix_28909_DEL_2': 'suffix_29798_DEL',
- 'suffix_289586_DEL_2': 'suffix_299115_DEL',
- 'suffix_290193_DEL_2': 'suffix_299736_DEL',
- 'suffix_29032_DEL_2': 'suffix_29921_DEL',
- 'suffix_292554_DEL_2': 'suffix_302162_DEL',
- 'suffix_293764_DEL_2': 'suffix_303412_DEL',
- 'suffix_295098_DUP_2': 'suffix_304770_DUP',
- 'suffix_295112_DUP_2': 'suffix_304785_DUP',
- 'suffix_297809_DUP_2': 'suffix_307588_DUP',
- 'suffix_297979_DEL_2': 'suffix_307770_DEL',
- 'suffix_298560_DUP_2': 'suffix_308385_DUP',
- 'suffix_299334_DEL_2': 'suffix_309189_DEL',
- 'suffix_299424_DUP_2': 'suffix_309280_DUP',
- 'suffix_299426_DUP_2': 'suffix_309282_DUP',
- 'suffix_299461_DUP_2': 'suffix_309319_DUP',
- 'suffix_299537_DUP_2': 'suffix_309398_DUP',
- 'suffix_299973_DEL_2': 'suffix_309849_DEL',
- 'suffix_300029_DEL_2': 'suffix_309906_DEL',
- 'suffix_300096_DUP_2': 'suffix_309975_DUP',
- 'suffix_300178_DEL_2': 'suffix_310060_DEL',
- 'suffix_300705_DUP_2': 'suffix_310598_DUP',
- 'suffix_300835_DUP_2': 'suffix_310730_DUP',
- 'suffix_300947_DUP_2': 'suffix_310843_DUP',
- 'suffix_301053_DUP_2': 'suffix_310954_DUP',
- 'suffix_301313_DEL_2': 'suffix_311215_DEL',
- 'suffix_302352_DEL_2': 'suffix_312295_DEL',
- 'suffix_302614_DUP_2': 'suffix_312565_DUP',
- 'suffix_305489_DEL_2': 'suffix_315530_DEL',
- 'suffix_305497_DEL_2': 'suffix_315539_DEL',
- 'suffix_305995_DEL_2': 'suffix_316050_DEL',
- 'suffix_30805_DEL_2': 'suffix_31751_DEL',
- 'suffix_310213_DUP_2': 'suffix_320398_DUP',
- 'suffix_310801_DUP_2': 'suffix_321018_DUP',
- 'suffix_312831_DUP_2': 'suffix_323123_DUP',
- 'suffix_313834_DUP_2': 'suffix_324164_DUP',
- 'suffix_314680_DUP_2': 'suffix_325048_DUP',
- 'suffix_318209_DUP_2': 'suffix_328707_DUP',
- 'suffix_319248_DUP_2': 'suffix_329772_DUP',
- 'suffix_319459_DEL_2': 'suffix_329991_DEL',
- 'suffix_320939_DUP_2': 'suffix_331503_DUP',
- 'suffix_321448_DEL_2': 'suffix_332033_DEL',
- 'suffix_321475_DEL_2': 'suffix_332061_DEL',
- 'suffix_323409_DUP_2': 'suffix_334076_DUP',
- 'suffix_323934_DUP_2': 'suffix_334633_DUP',
- 'suffix_324816_DUP_2': 'suffix_335549_DUP',
- 'suffix_32507_DUP_2': 'suffix_33497_DUP',
- 'suffix_325590_DUP_2': 'suffix_336339_DUP',
- 'suffix_325871_DUP_2': 'suffix_336630_DUP',
- 'suffix_326049_DUP_2': 'suffix_336809_DUP',
- 'suffix_326068_DUP_2': 'suffix_336828_DUP',
- 'suffix_326213_DUP_2': 'suffix_336974_DUP',
- 'suffix_326219_DEL_2': 'suffix_336980_DEL',
- 'suffix_326226_DEL_2': 'suffix_336987_DEL',
- 'suffix_326275_DUP_2': 'suffix_337036_DUP',
- 'suffix_326360_DEL_2': 'suffix_337122_DEL',
- 'suffix_326662_DEL_2': 'suffix_337431_DEL',
- 'suffix_326846_DEL_2': 'suffix_337620_DEL',
- 'suffix_327283_DEL_2': 'suffix_338071_DEL',
- 'suffix_327813_DEL_2': 'suffix_338612_DEL',
- 'suffix_328030_DUP_2': 'suffix_338831_DUP',
- 'suffix_328354_DUP_2': 'suffix_339162_DUP',
- 'suffix_328867_DUP_2': 'suffix_339705_DUP',
- 'suffix_32906_DUP_2': 'suffix_33904_DUP',
- 'suffix_32996_DUP_2': 'suffix_34000_DUP',
- 'suffix_332611_DUP_2': 'suffix_343530_DUP',
- 'suffix_332820_DEL_2': 'suffix_343748_DEL',
- 'suffix_332823_DEL_2': 'suffix_343751_DEL',
- 'suffix_333026_DEL_2': 'suffix_343960_DEL',
- 'suffix_333055_DUP_2': 'suffix_343990_DUP',
- 'suffix_333717_DUP_2': 'suffix_344664_DUP',
- 'suffix_334188_DEL_2': 'suffix_345154_DEL',
- 'suffix_334626_DEL_2': 'suffix_345601_DEL',
- 'suffix_34446_DEL_2': 'suffix_35494_DEL',
- 'suffix_35726_DEL_2': 'suffix_36817_DEL',
- 'suffix_35799_DEL_2': 'suffix_36894_DEL',
- 'suffix_3651_DEL_2': 'suffix_3783_DEL',
- 'suffix_3691_DEL_2': 'suffix_3824_DEL',
- 'suffix_37781_DUP_2': 'suffix_38929_DUP',
- 'suffix_39085_DUP_2': 'suffix_40276_DUP',
- 'suffix_42000_DUP_2': 'suffix_43285_DUP',
- 'suffix_44539_DUP_2': 'suffix_45913_DUP',
- 'suffix_44736_DUP_2': 'suffix_46123_DUP',
- 'suffix_4505_DEL_2': 'suffix_4670_DEL',
- 'suffix_46580_DUP_2': 'suffix_48024_DUP',
- 'suffix_46882_DEL_2': 'suffix_48334_DEL',
- 'suffix_4995_DUP_2': 'suffix_5177_DUP',
- 'suffix_50873_DEL_2': 'suffix_52450_DEL',
- 'suffix_51969_DUP_2': 'suffix_53573_DUP',
- 'suffix_5276_DUP_2': 'suffix_5474_DUP',
- 'suffix_57692_DUP_2': 'suffix_59439_DUP',
- 'suffix_58774_DUP_2': 'suffix_60544_DUP',
- 'suffix_58893_DEL_2': 'suffix_60667_DEL',
- 'suffix_59352_DEL_2': 'suffix_61133_DEL',
- 'suffix_59736_DUP_2': 'suffix_61529_DUP',
- 'suffix_59942_DEL_2': 'suffix_61742_DEL',
- 'suffix_6172_DUP_2': 'suffix_6414_DUP',
- 'suffix_64006_DUP_2': 'suffix_65935_DUP',
- 'suffix_65035_DUP_2': 'suffix_67010_DUP',
- 'suffix_65938_DUP_2': 'suffix_67937_DUP',
- 'suffix_6612_DEL_2': 'suffix_6868_DEL',
- 'suffix_66192_DUP_2': 'suffix_68205_DUP',
- 'suffix_66563_DEL_2': 'suffix_68588_DEL',
- 'suffix_66964_DUP_2': 'suffix_68997_DUP',
- 'suffix_70658_DEL_2': 'suffix_72772_DEL',
- 'suffix_71031_DEL_2': 'suffix_73155_DEL',
- 'suffix_72679_DUP_2': 'suffix_74850_DUP',
- 'suffix_72691_DUP_2': 'suffix_74863_DUP',
- 'suffix_72959_DUP_2': 'suffix_75144_DUP',
- 'suffix_73683_DUP_2': 'suffix_75884_DUP',
- 'suffix_76491_DEL_2': 'suffix_78773_DEL',
- 'suffix_7806_DUP_2': 'suffix_8099_DUP',
- 'suffix_81587_DUP_2': 'suffix_83995_DUP',
- 'suffix_8201_DUP_2': 'suffix_8510_DUP',
- 'suffix_82039_DUP_2': 'suffix_84458_DUP',
- 'suffix_82642_DUP_2': 'suffix_85071_DUP',
- 'suffix_837_DUP_2': 'suffix_862_DUP',
- 'suffix_84346_DEL_2': 'suffix_86829_DEL',
- 'suffix_84718_DUP_2': 'suffix_87225_DUP',
- 'suffix_85814_DUP_2': 'suffix_88367_DUP',
- 'suffix_86689_DUP_2': 'suffix_89280_DUP',
- 'suffix_87664_DUP_2': 'suffix_90289_DUP',
- 'suffix_88351_DEL_2': 'suffix_91000_DEL',
- 'suffix_89655_DEL_2': 'suffix_92358_DEL',
- 'suffix_90187_DEL_2': 'suffix_92909_DEL',
- 'suffix_90707_DEL_2': 'suffix_93436_DEL',
- 'suffix_90995_DEL_2': 'suffix_93733_DEL',
- 'suffix_91277_DEL_2': 'suffix_94018_DEL',
- 'suffix_91760_DUP_2': 'suffix_94518_DUP',
- 'suffix_91769_DEL_2': 'suffix_94529_DEL',
- 'suffix_94202_DEL_2': 'suffix_97018_DEL',
- 'prefix_109408_DUP': 'suffix_220966_DUP',
- 'prefix_109696_DUP': 'suffix_9533_DUP',
- 'prefix_118254_DUP': 'suffix_132626_DUP',
- 'prefix_120819_DUP': 'suffix_135755_DUP',
- 'prefix_135048_DUP': 'suffix_151788_DUP',
- 'prefix_136173_DEL': 'suffix_54386_DEL',
- 'prefix_148570_DEL': 'suffix_107482_DEL',
- 'prefix_150603_DUP': 'suffix_169662_DUP',
- 'prefix_150646_DUP': 'suffix_343955_DUP',
- 'prefix_175273_DUP': 'suffix_322926_DUP',
- 'prefix_178590_DUP': 'suffix_201523_DUP',
- 'prefix_18102_DUP': 'suffix_47262_DUP',
- 'prefix_184122_DUP': 'suffix_118450_DUP',
- 'prefix_191470_DUP': 'suffix_117959_DUP',
- 'prefix_19177_DUP': 'suffix_21314_DUP',
- 'prefix_193273_DUP': 'suffix_218537_DUP',
- 'prefix_195055_DUP': 'suffix_277123_DUP',
- 'prefix_195635_DUP': 'suffix_337960_DUP',
- 'prefix_196043_DUP': 'suffix_223818_DUP',
- 'prefix_196756_DUP': 'suffix_95953_DUP',
- 'prefix_200666_DEL': 'suffix_4258_DEL',
- 'prefix_212037_DEL': 'suffix_130018_DEL',
- 'prefix_2145_DUP': 'suffix_2610_DUP',
- 'prefix_22446_DUP': 'suffix_312822_DUP',
- 'prefix_229750_DUP': 'suffix_190694_DUP',
- 'prefix_237049_DUP': 'suffix_268708_DUP',
- 'prefix_237750_DUP': 'suffix_42133_DUP',
- 'prefix_239745_DEL': 'suffix_308552_DEL',
- 'prefix_240072_DUP': 'suffix_108801_DUP',
- 'prefix_240150_DEL': 'suffix_5244_DEL',
- 'prefix_242658_DUP': 'suffix_84380_DUP',
- 'prefix_246643_DEL': 'suffix_93326_DEL',
- 'prefix_24840_DUP': 'suffix_882_DUP',
- 'prefix_252587_DUP': 'suffix_158624_DUP',
- 'prefix_254761_DUP': 'suffix_57665_DUP',
- 'prefix_255040_DUP': 'suffix_106670_DUP',
- 'prefix_261005_DEL': 'suffix_178167_DEL',
- 'prefix_263591_DUP': 'suffix_319425_DUP',
- 'prefix_263818_DEL': 'suffix_201219_DEL',
- 'prefix_285913_DEL': 'suffix_74_DEL',
- 'prefix_297312_DUP': 'suffix_337092_DUP',
- 'prefix_30992_DEL': 'suffix_331860_DEL',
- 'prefix_3358_DEL': 'suffix_71259_DEL',
- 'prefix_33658_DUP': 'suffix_268605_DUP',
- 'prefix_37118_DUP': 'suffix_248738_DUP',
- 'prefix_39859_DUP': 'suffix_122375_DUP',
- 'prefix_42735_DUP': 'suffix_155097_DUP',
- 'prefix_48349_DUP': 'suffix_150163_DUP',
- 'prefix_52511_DUP': 'suffix_234367_DUP',
- 'prefix_54966_DEL': 'suffix_62104_DEL',
- 'prefix_7639_DUP': 'suffix_307081_DUP',
- 'prefix_74657_DUP': 'suffix_83373_DUP',
- 'prefix_77210_DUP': 'suffix_241886_DUP',
- 'prefix_7935_DUP': 'suffix_124466_DUP',
- 'prefix_80037_DUP': 'suffix_332875_DUP',
- 'prefix_82121_DUP': 'suffix_269588_DUP',
- 'prefix_83942_DUP': 'suffix_322084_DUP',
- 'prefix_84486_DUP': 'suffix_94510_DUP',
- 'prefix_92979_DEL': 'suffix_103592_DEL',
- 'prefix_92985_DEL': 'suffix_148088_DEL',
- 'prefix_93168_DEL': 'suffix_184994_DEL',
- 'prefix_932_DEL': 'suffix_334231_DEL',
- 'prefix_9387_DUP': 'suffix_10911_DUP',
- 'prefix_94035_DEL': 'suffix_262718_DEL',
- 'prefix_94077_DEL': 'suffix_103402_DEL',
- 'prefix_94079_DUP': 'suffix_115968_DUP',
- 'prefix_94158_DEL': 'suffix_8045_DEL',
- 'prefix_9428_DEL': 'suffix_10965_DEL',
- 'prefix_94589_DEL': 'suffix_232242_DEL',
- 'prefix_94688_DEL': 'suffix_329994_DEL',
- 'prefix_94693_DUP': 'suffix_133009_DUP',
- 'prefix_94714_DEL': 'suffix_86407_DEL',
- 'prefix_94949_DEL': 'suffix_3640_DEL',
- 'prefix_95467_DEL': 'suffix_106269_DEL',
- 'prefix_95753_DEL': 'suffix_106601_DEL',
- 'prefix_96081_DEL': 'suffix_39335_DEL',
- 'prefix_96409_DEL': 'suffix_107290_DEL',
- 'prefix_96417_DEL': 'suffix_107299_DEL',
- 'prefix_96497_DUP': 'suffix_107390_DUP',
- 'prefix_96678_DEL': 'suffix_107578_DEL',
- 'prefix_96701_DUP': 'suffix_107599_DUP',
- 'prefix_97034_DUP': 'suffix_107976_DUP',
- 'prefix_97265_DUP': 'suffix_296214_DUP',
- 'prefix_97383_DEL': 'suffix_269445_DEL',
- 'prefix_97410_DEL': 'suffix_21697_DEL',
- 'prefix_97624_DEL': 'suffix_19282_DEL',
- 'prefix_98159_DUP': 'suffix_339672_DUP',
- 'prefix_98246_DEL': 'suffix_343159_DEL',
- 'prefix_98359_DUP': 'suffix_253054_DUP',
- 'prefix_98374_DEL': 'suffix_303813_DEL',
- 'prefix_98492_DEL': 'suffix_256422_DEL',
- 'prefix_99210_DUP': 'suffix_110452_DUP',
- 'prefix_99385_DEL': 'suffix_130483_DEL',
- 'prefix_99429_DEL': 'suffix_316828_DEL',
- 'prefix_99941_DEL': 'suffix_245189_DEL',
- 'prefix_99962_DEL': 'suffix_11652_DEL',
- 'prefix_99964_DUP': 'suffix_184974_DUP',
- 'prefix_99980_DEL': 'suffix_291827_DEL',
- 'prefix_99987_DUP': 'suffix_306785_DUP',
- 'suffix_115271_DUP_2': 'suffix_118725_DUP',
- 'suffix_147347_DUP_2': 'suffix_151788_DUP',
- 'suffix_152903_DUP_2': 'suffix_157559_DUP',
- 'suffix_16434_DEL_2': 'suffix_16950_DEL',
- 'suffix_17973_DEL_2': 'suffix_18512_DEL',
- 'suffix_199564_DUP_2': 'suffix_205825_DUP',
- 'suffix_208055_DUP_2': 'suffix_214569_DUP',
- 'suffix_208666_DEL_2': 'suffix_215204_DEL',
- 'suffix_215449_DUP_2': 'suffix_222247_DUP',
- 'suffix_24241_DUP_2': 'suffix_24952_DUP',
- 'suffix_282215_DUP_2': 'suffix_291438_DUP',
- 'suffix_78257_DEL_2': 'suffix_80567_DEL',
- 'prefix_105041_DUP': 'suffix_157907_DUP',
- 'prefix_105835_DUP': 'suffix_139575_DUP',
- 'prefix_108273_DEL': 'suffix_302962_DEL',
- 'prefix_110759_DUP': 'suffix_121735_DUP',
- 'prefix_112991_DEL': 'suffix_19457_DEL',
- 'prefix_116608_DUP': 'suffix_132952_DUP',
- 'prefix_123515_DEL': 'suffix_138768_DEL',
- 'prefix_133566_DUP': 'suffix_150246_DUP',
- 'prefix_141809_DEL': 'suffix_300943_DEL',
- 'prefix_142858_DEL': 'suffix_128588_DEL',
- 'prefix_146871_DUP': 'suffix_133106_DUP',
- 'prefix_149568_DEL': 'suffix_118635_DEL',
- 'prefix_167368_DEL': 'suffix_189133_DEL',
- 'prefix_176845_DUP': 'suffix_211875_DUP',
- 'prefix_177271_DUP': 'suffix_312822_DUP',
- 'prefix_181179_DUP': 'suffix_234798_DUP',
- 'prefix_185619_DEL': 'suffix_217950_DEL',
- 'prefix_185910_DEL': 'suffix_291085_DEL',
- 'prefix_192960_DEL': 'suffix_290358_DEL',
- 'prefix_219036_DUP': 'suffix_341481_DUP',
- 'prefix_21518_DUP': 'suffix_218342_DUP',
- 'prefix_22434_DEL': 'suffix_189133_DEL',
- 'prefix_22509_DUP': 'suffix_261218_DUP',
- 'prefix_230654_DEL': 'suffix_106308_DEL',
- 'prefix_237348_DUP': 'suffix_155009_DUP',
- 'prefix_239011_DUP': 'suffix_107426_DUP',
- 'prefix_240074_DUP': 'suffix_109378_DUP',
- 'prefix_248113_DUP': 'suffix_291388_DUP',
- 'prefix_252320_DUP': 'suffix_286050_DUP',
- 'prefix_252636_DUP': 'suffix_337479_DUP',
- 'prefix_29923_DEL': 'suffix_157806_DEL',
- 'prefix_30391_DEL': 'suffix_230937_DEL',
- 'prefix_30532_DUP': 'suffix_86319_DUP',
- 'prefix_32634_DUP': 'suffix_95910_DUP',
- 'prefix_32918_DEL': 'suffix_36811_DEL',
- 'prefix_37021_DUP': 'suffix_123374_DUP',
- 'prefix_39925_DEL': 'suffix_107137_DEL',
- 'prefix_45770_DUP': 'suffix_329746_DUP',
- 'prefix_45776_DEL': 'suffix_167381_DEL',
- 'prefix_46296_DUP': 'suffix_255601_DUP',
- 'prefix_47514_DUP': 'suffix_53429_DUP',
- 'prefix_47557_DUP': 'suffix_247215_DUP',
- 'prefix_47647_DUP': 'suffix_311519_DUP',
- 'prefix_49053_DEL': 'suffix_33001_DEL',
- 'prefix_53399_DUP': 'suffix_335231_DUP',
- 'prefix_72156_DUP': 'suffix_122944_DUP',
- 'prefix_7322_DUP': 'suffix_189132_DUP',
- 'prefix_75522_DUP': 'suffix_84376_DUP',
- 'prefix_78549_DEL': 'suffix_50090_DEL',
- 'prefix_80929_DEL': 'suffix_90588_DEL',
- 'prefix_81289_DUP': 'suffix_90965_DUP',
- 'prefix_8811_DUP': 'suffix_148957_DUP',
- 'prefix_91635_DEL': 'suffix_293519_DEL',
- 'prefix_98234_DEL': 'suffix_109388_DEL',
- 'prefix_99986_DUP': 'suffix_306740_DUP',
- 'suffix_10162_DEL_2': 'suffix_10535_DEL',
- 'suffix_119700_DUP_2': 'suffix_123318_DUP',
- 'suffix_146970_DUP_2': 'suffix_151397_DUP',
- 'suffix_189219_DEL_2': 'suffix_195097_DEL',
- 'suffix_200704_DUP_2': 'suffix_206987_DUP',
- 'suffix_208083_DEL_2': 'suffix_214598_DEL',
- 'suffix_208097_DEL_2': 'suffix_214612_DEL',
- 'suffix_212079_DUP_2': 'suffix_218732_DUP',
- 'suffix_24668_DUP_2': 'suffix_25398_DUP',
- 'suffix_247327_DUP_2': 'suffix_255366_DUP',
- 'suffix_248140_DUP_2': 'suffix_256203_DUP',
- 'suffix_26120_DUP_2': 'suffix_26896_DUP',
- 'suffix_26153_DUP_2': 'suffix_26929_DUP',
- 'suffix_28933_DEL_2': 'suffix_29823_DEL',
- 'suffix_83854_DUP_2': 'suffix_86319_DUP',
- 'prefix_113219_DUP': 'suffix_345434_DUP',
- 'prefix_173077_DEL': 'suffix_195402_DEL',
- 'prefix_11978_DEL': 'suffix_283553_DEL',
- 'prefix_284589_DUP': 'suffix_289594_DUP',
- 'prefix_4812_DUP': 'suffix_5671_DUP',
- 'suffix_326634_DEL_2': 'suffix_337403_DEL',
- 'suffix_333975_DUP_2': 'suffix_344929_DUP',
- 'prefix_102605_DEL': 'suffix_265004_DEL',
- 'prefix_103754_DEL': 'suffix_308671_DEL',
- 'prefix_105176_DEL': 'suffix_617_DEL',
- 'prefix_105255_DEL': 'suffix_212161_DEL',
- 'prefix_105413_DUP': 'suffix_16872_DUP',
- 'prefix_108429_DEL': 'suffix_276230_DEL',
- 'prefix_11302_DEL': 'suffix_211819_DEL',
- 'prefix_115700_DUP': 'suffix_201547_DUP',
- 'prefix_117580_DUP': 'suffix_181916_DUP',
- 'prefix_119226_DEL': 'suffix_223666_DEL',
- 'prefix_119993_DUP': 'suffix_62592_DUP',
- 'prefix_121412_DEL': 'suffix_136441_DEL',
- 'prefix_12154_DEL': 'suffix_16787_DEL',
- 'prefix_12156_DEL': 'suffix_16888_DEL',
- 'prefix_12407_DEL': 'suffix_193951_DEL',
- 'prefix_126083_DUP': 'suffix_93868_DUP',
- 'prefix_132933_DUP': 'suffix_178504_DUP',
- 'prefix_132951_DUP': 'suffix_201246_DUP',
- 'prefix_138735_DUP': 'suffix_155567_DUP',
- 'prefix_140549_DUP': 'suffix_157451_DUP',
- 'prefix_141948_DEL': 'suffix_159338_DEL',
- 'prefix_146744_DEL': 'suffix_31511_DEL',
- 'prefix_147468_DUP': 'suffix_166040_DUP',
- 'prefix_152051_DUP': 'suffix_136974_DUP',
- 'prefix_15323_DUP': 'suffix_62805_DUP',
- 'prefix_164074_DEL': 'suffix_2290_DEL',
- 'prefix_167_DEL': 'suffix_46122_DEL',
- 'prefix_168590_DUP': 'suffix_288536_DUP',
- 'prefix_168606_DUP': 'suffix_157956_DUP',
- 'prefix_171981_DUP': 'suffix_35868_DUP',
- 'prefix_173688_DUP': 'suffix_54837_DUP',
- 'prefix_174135_DEL': 'suffix_16791_DEL',
- 'prefix_175272_DUP': 'suffix_322830_DUP',
- 'prefix_175274_DUP': 'suffix_322963_DUP',
- 'prefix_183_DEL': 'suffix_106660_DEL',
- 'prefix_184814_DEL': 'suffix_195404_DEL',
- 'prefix_18485_DEL': 'suffix_29789_DEL',
- 'prefix_185069_DEL': 'suffix_132891_DEL',
- 'prefix_185206_DEL': 'suffix_1868_DEL',
- 'prefix_185338_DEL': 'suffix_290358_DEL',
- 'prefix_191249_DEL': 'suffix_150483_DEL',
- 'prefix_191318_DEL': 'suffix_20510_DEL',
- 'prefix_191655_DUP': 'suffix_327391_DUP',
- 'prefix_191946_DEL': 'suffix_216855_DEL',
- 'prefix_194528_DUP': 'suffix_220210_DUP',
- 'prefix_195246_DUP': 'suffix_293034_DUP',
- 'prefix_198967_DUP': 'suffix_244057_DUP',
- 'prefix_198991_DUP': 'suffix_244754_DUP',
- 'prefix_206418_DUP': 'suffix_234368_DUP',
- 'prefix_20_DUP': 'suffix_268896_DUP',
- 'prefix_216861_DEL': 'suffix_246030_DEL',
- 'prefix_22176_DUP': 'suffix_47721_DUP',
- 'prefix_22336_DUP': 'suffix_159422_DUP',
- 'prefix_22604_DEL': 'suffix_106759_DEL',
- 'prefix_2317_DEL': 'suffix_176841_DEL',
- 'prefix_233367_DUP': 'suffix_100639_DUP',
- 'prefix_23494_DEL': 'suffix_333136_DEL',
- 'prefix_236689_DUP': 'suffix_268271_DUP',
- 'prefix_236881_DUP': 'suffix_251030_DUP',
- 'prefix_23854_DEL': 'suffix_120387_DEL',
- 'prefix_240210_DUP': 'suffix_184198_DUP',
- 'prefix_241766_DEL': 'suffix_274245_DEL',
- 'prefix_242193_DUP': 'suffix_317066_DUP',
- 'prefix_242258_DUP': 'suffix_257305_DUP',
- 'prefix_24253_DEL': 'suffix_212318_DEL',
- 'prefix_243424_DUP': 'suffix_276101_DUP',
- 'prefix_243699_DUP': 'suffix_276379_DUP',
- 'prefix_24375_DUP': 'suffix_258696_DUP',
- 'prefix_24557_DUP': 'suffix_47694_DUP',
- 'prefix_250938_DUP': 'suffix_105404_DUP',
- 'prefix_250978_DEL': 'suffix_114000_DEL',
- 'prefix_251024_DEL': 'suffix_292979_DEL',
- 'prefix_252358_DUP': 'suffix_69555_DUP',
- 'prefix_252448_DUP': 'suffix_210958_DUP',
- 'prefix_253050_DUP': 'suffix_11042_DUP',
- 'prefix_255274_DEL': 'suffix_74443_DEL',
- 'prefix_255883_DEL': 'suffix_105498_DEL',
- 'prefix_257310_DUP': 'suffix_291433_DUP',
- 'prefix_260697_DUP': 'suffix_101357_DUP',
- 'prefix_262045_DUP': 'suffix_125836_DUP',
- 'prefix_266484_DUP': 'suffix_301782_DUP',
- 'prefix_26953_DEL': 'suffix_307435_DEL',
- 'prefix_272536_DUP': 'suffix_308571_DUP',
- 'prefix_273661_DEL': 'suffix_309764_DEL',
- 'prefix_277674_DEL': 'suffix_126886_DEL',
- 'prefix_27955_DEL': 'suffix_118689_DEL',
- 'prefix_280210_DEL': 'suffix_63525_DEL',
- 'prefix_23608_DEL': 'suffix_141141_DEL',
- 'prefix_284014_DUP': 'suffix_217043_DUP',
- 'prefix_297304_DEL': 'suffix_337079_DEL',
- 'prefix_298624_DUP': 'suffix_338928_DUP',
- 'prefix_29886_DEL': 'suffix_122545_DEL',
- 'prefix_30066_DEL': 'suffix_105693_DEL',
- 'prefix_30121_DUP': 'suffix_82107_DUP',
- 'prefix_31125_DUP': 'suffix_34864_DUP',
- 'prefix_31175_DEL': 'suffix_334478_DEL',
- 'prefix_32814_DEL': 'suffix_290329_DEL',
- 'prefix_3353_DUP': 'suffix_4034_DUP',
- 'prefix_3417_DUP': 'suffix_106659_DUP',
- 'prefix_34842_DUP': 'suffix_110095_DUP',
- 'prefix_352_DEL': 'suffix_195699_DEL',
- 'prefix_36954_DUP': 'suffix_322053_DUP',
- 'prefix_36966_DUP': 'suffix_137770_DUP',
- 'prefix_37088_DUP': 'suffix_246551_DUP',
- 'prefix_37312_DEL': 'suffix_269442_DEL',
- 'prefix_37395_DEL': 'suffix_268115_DEL',
- 'prefix_37462_DUP': 'suffix_155684_DUP',
- 'prefix_37511_DUP': 'suffix_88367_DUP',
- 'prefix_38100_DUP': 'suffix_280153_DUP',
- 'prefix_41266_DEL': 'suffix_46106_DEL',
- 'prefix_44220_DEL': 'suffix_29831_DEL',
- 'prefix_45074_DEL': 'suffix_102910_DEL',
- 'prefix_4522_DEL': 'suffix_246030_DEL',
- 'prefix_463_DUP': 'suffix_291433_DUP',
- 'prefix_46468_DUP': 'suffix_273617_DUP',
- 'prefix_46530_DEL': 'suffix_258864_DEL',
- 'prefix_47637_DUP': 'suffix_309976_DUP',
- 'prefix_51641_DEL': 'suffix_58293_DEL',
- 'prefix_53677_DUP': 'suffix_34864_DUP',
- 'prefix_54008_DUP': 'suffix_285632_DUP',
- 'prefix_54421_DUP': 'suffix_47721_DUP',
- 'prefix_55677_DUP': 'suffix_62795_DUP',
- 'prefix_65173_DUP': 'suffix_290375_DUP',
- 'prefix_56142_DUP': 'suffix_147501_DUP',
- 'prefix_58202_DUP': 'suffix_116683_DUP',
- 'prefix_5881_DUP': 'suffix_230728_DUP',
- 'prefix_62733_DUP': 'suffix_228856_DUP',
- 'prefix_628_DEL': 'suffix_195672_DEL',
- 'prefix_6327_DEL': 'suffix_283625_DEL',
- 'prefix_63755_DUP': 'suffix_258929_DUP',
- 'prefix_6487_DEL': 'suffix_337048_DEL',
- 'prefix_6596_DEL': 'suffix_35678_DEL',
- 'prefix_66180_DEL': 'suffix_42022_DEL',
- 'prefix_6683_DUP': 'suffix_189168_DUP',
- 'prefix_69844_DEL': 'suffix_139225_DEL',
- 'prefix_71567_DUP': 'suffix_297455_DUP',
- 'prefix_71575_DUP': 'suffix_292697_DUP',
- 'prefix_71627_DUP': 'suffix_101357_DUP',
- 'prefix_72141_DUP': 'suffix_334049_DUP',
- 'prefix_74496_DUP': 'suffix_37213_DUP',
- 'prefix_74947_DEL': 'suffix_1698_DEL',
- 'prefix_77626_DEL': 'suffix_35767_DEL',
- 'prefix_778_DUP': 'suffix_166040_DUP',
- 'prefix_78301_DEL': 'suffix_336999_DEL',
- 'prefix_78948_DUP': 'suffix_308282_DUP',
- 'prefix_79561_DUP': 'suffix_216297_DUP',
- 'prefix_79966_DUP': 'suffix_202096_DUP',
- 'prefix_8308_DUP': 'suffix_62795_DUP',
- 'prefix_83433_DEL': 'suffix_307445_DEL',
- 'prefix_83930_DUP': 'suffix_32721_DUP',
- 'prefix_86900_DUP': 'suffix_47150_DUP',
- 'prefix_89846_DEL': 'suffix_234997_DEL',
- 'prefix_92729_DEL': 'suffix_287108_DEL',
- 'prefix_93407_DEL': 'suffix_336228_DEL',
- 'prefix_93530_DUP': 'suffix_324867_DUP',
- 'prefix_93992_DUP': 'suffix_181474_DUP',
- 'prefix_95800_DUP': 'suffix_106659_DUP',
- 'prefix_96646_DUP': 'suffix_212303_DUP',
- 'prefix_97428_DEL': 'suffix_335378_DEL',
- 'prefix_9785_DEL': 'suffix_336447_DEL',
- 'prefix_9871_DEL': 'suffix_13825_DEL',
- 'suffix_106521_DEL_2': 'suffix_109744_DEL',
- 'suffix_111044_DUP_2': 'suffix_114370_DUP',
- 'suffix_115236_DEL_2': 'suffix_118689_DEL',
- 'suffix_118984_DUP_2': 'suffix_122590_DUP',
- 'suffix_123189_DEL_2': 'suffix_126886_DEL',
- 'suffix_128305_DUP_2': 'suffix_132156_DUP',
- 'suffix_128994_DEL_2': 'suffix_132877_DEL',
- 'suffix_143611_DEL_2': 'suffix_147926_DEL',
- 'suffix_145008_DUP_2': 'suffix_149379_DUP',
- 'suffix_145792_DUP_2': 'suffix_150187_DUP',
- 'suffix_146879_DUP_2': 'suffix_151303_DUP',
- 'suffix_152658_DUP_2': 'suffix_157304_DUP',
- 'suffix_15417_DEL_2': 'suffix_15906_DEL',
- 'suffix_1560_DEL_2': 'suffix_1619_DEL',
- 'suffix_159007_DUP_2': 'suffix_163879_DUP',
- 'suffix_16278_DUP_2': 'suffix_16792_DUP',
- 'suffix_16287_DUP_2': 'suffix_16801_DUP',
- 'suffix_16320_DUP_2': 'suffix_16834_DUP',
- 'suffix_179450_DEL_2': 'suffix_184949_DEL',
- 'suffix_188093_DEL_2': 'suffix_193931_DEL',
- 'suffix_189764_DUP_2': 'suffix_195660_DUP',
- 'suffix_189797_DEL_2': 'suffix_195693_DEL',
- 'suffix_189963_DEL_2': 'suffix_195862_DEL',
- 'suffix_190204_DEL_2': 'suffix_196111_DEL',
- 'suffix_190596_DEL_2': 'suffix_196510_DEL',
- 'suffix_198910_DUP_2': 'suffix_205165_DUP',
- 'suffix_198945_DUP_2': 'suffix_205200_DUP',
- 'suffix_205378_DUP_2': 'suffix_211816_DUP',
- 'suffix_205382_DEL_2': 'suffix_211819_DEL',
- 'suffix_206291_DEL_2': 'suffix_212760_DEL',
- 'suffix_208232_DEL_2': 'suffix_214749_DEL',
- 'suffix_208718_DUP_2': 'suffix_215260_DUP',
- 'suffix_210692_DEL_2': 'suffix_217291_DEL',
- 'suffix_210883_DUP_2': 'suffix_217485_DUP',
- 'suffix_211544_DUP_2': 'suffix_218173_DUP',
- 'suffix_216501_DEL_2': 'suffix_223347_DEL',
- 'suffix_216580_DEL_2': 'suffix_223427_DEL',
- 'suffix_233162_DEL_2': 'suffix_240724_DEL',
- 'suffix_238770_DUP_2': 'suffix_246551_DUP',
- 'suffix_239364_DEL_2': 'suffix_247162_DEL',
- 'suffix_253024_DUP_2': 'suffix_261270_DUP',
- 'suffix_26252_DEL_2': 'suffix_27028_DEL',
- 'suffix_26422_DEL_2': 'suffix_27202_DEL',
- 'suffix_273614_DEL_2': 'suffix_282591_DEL',
- 'suffix_274216_DEL_2': 'suffix_283203_DEL',
- 'suffix_285251_DEL_2': 'suffix_294612_DEL',
- 'suffix_285526_DUP_2': 'suffix_294904_DUP',
- 'suffix_286293_DEL_2': 'suffix_295704_DEL',
- 'suffix_28918_DEL_2': 'suffix_29808_DEL',
- 'suffix_290688_DEL_2': 'suffix_300251_DEL',
- 'suffix_291524_DUP_2': 'suffix_301110_DUP',
- 'suffix_291969_DUP_2': 'suffix_301562_DUP',
- 'suffix_299400_DUP_2': 'suffix_309256_DUP',
- 'suffix_303591_DEL_2': 'suffix_313583_DEL',
- 'suffix_30447_DUP_2': 'suffix_31375_DUP',
- 'suffix_304499_DEL_2': 'suffix_314515_DEL',
- 'suffix_3148_DEL_2': 'suffix_3268_DEL',
- 'suffix_324233_DUP_2': 'suffix_334946_DUP',
- 'suffix_32624_DUP_2': 'suffix_33615_DUP',
- 'suffix_35346_DEL_2': 'suffix_36419_DEL',
- 'suffix_35506_DEL_2': 'suffix_36591_DEL',
- 'suffix_38424_DUP_2': 'suffix_39596_DUP',
- 'suffix_38845_DEL_2': 'suffix_40030_DEL',
- 'suffix_39353_DUP_2': 'suffix_40557_DUP',
- 'suffix_41150_DUP_2': 'suffix_42404_DUP',
- 'suffix_47113_DUP_2': 'suffix_48575_DUP',
- 'suffix_48026_DUP_2': 'suffix_49547_DUP',
- 'suffix_49445_DUP_2': 'suffix_50997_DUP',
- 'suffix_61720_DUP_2': 'suffix_63567_DUP',
- 'suffix_62316_DUP_2': 'suffix_64183_DUP',
- 'suffix_9094_DUP_2': 'suffix_9431_DUP',
- 'suffix_914_DEL_2': 'suffix_925_DEL',
- 'suffix_92273_DUP_2': 'suffix_95045_DUP',
- 'suffix_93367_DUP_2': 'suffix_96167_DUP',
- 'prefix_156333_DUP': 'suffix_176635_DUP',
- 'prefix_1723_DUP': 'suffix_176578_DUP',
- 'prefix_172949_DEL': 'suffix_195223_DUP',
- 'prefix_18_DUP': 'suffix_20_DUP',
- 'prefix_192882_DUP': 'suffix_225192_DUP',
- 'prefix_230001_DEL': 'suffix_106669_DEL',
- 'prefix_241273_DUP': 'suffix_273630_DUP',
- 'prefix_260458_DUP': 'suffix_295039_DUP',
- 'prefix_26552_DEL': 'suffix_118629_DEL',
- 'prefix_268993_DUP': 'suffix_304512_DUP',
- 'prefix_34289_DUP': 'suffix_122919_DUP',
- 'prefix_42135_DUP': 'suffix_47150_DUP',
- 'prefix_447_DUP': 'suffix_195223_DUP',
- 'prefix_5926_DUP': 'suffix_277846_DUP',
- 'prefix_6582_DEL': 'suffix_212509_DEL',
- 'prefix_65866_DEL': 'suffix_282591_DEL',
- 'prefix_92476_DUP': 'suffix_110991_DUP',
- 'prefix_10078_DEL': 'suffix_195635_DEL',
- 'prefix_106132_DEL': 'suffix_195692_DEL',
- 'prefix_120338_DUP': 'suffix_135104_DUP',
- 'prefix_143062_DUP': 'suffix_160819_DUP',
- 'prefix_167275_DUP': 'suffix_308395_DUP',
- 'prefix_191895_DEL': 'suffix_216785_DEL',
- 'prefix_191896_DEL': 'suffix_216785_DEL',
- 'prefix_23369_DEL': 'suffix_296578_DEL',
- 'prefix_244929_DUP': 'suffix_277868_DUP',
- 'prefix_42634_DUP': 'suffix_166401_DUP',
- 'prefix_91448_DUP': 'suffix_148093_DUP',
- },
-}
SV_DROPPED_IDS = {
'cluster_6_last_call_cnv_17479_DUP', 'cluster_1_last_call_cnv_30127_DEL', 'cluster_19_COHORT_cnv_23176_DEL',
'phase2_DEL_chrX_1149', 'prefix_121357_DUP', 'prefix_73945_DEL', 'phase2_DUP_chr9_1663', 'phase2_CPX_chr20_4',
@@ -2249,7 +214,7 @@ def handle(self, *args, **options):
logger.info('Done')
@staticmethod
- def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_GRCh38, variant_id_updates=None):
+ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_GRCh38):
if not variants_ids:
return set()
logger.info(f'Finding keys for {len(variants_ids)} {dataset_type} (GRCh{genome_version}) variant ids')
@@ -2259,13 +224,8 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_
return set(variants_ids)
mapped_variant_ids = list(variant_key_map.keys())
- if variant_id_updates:
- reverse_lookup = {v: k for k, v in variant_id_updates.items()}
- mapped_variant_ids = [reverse_lookup[vid] for vid in mapped_variant_ids]
update_fields = ['key', 'dataset_type']
- if variant_id_updates:
- update_fields.append('variant_id')
total_num_updated = 0
for i in range(0, len(mapped_variant_ids), BATCH_SIZE):
batch_ids = mapped_variant_ids[i:i + BATCH_SIZE]
@@ -2273,8 +233,6 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_
family__project__genome_version=genome_version, variant_id__in=batch_ids,
)
for variant in saved_variants:
- if variant_id_updates:
- variant.variant_id = variant_id_updates[variant.variant_id]
variant.key = variant_key_map[variant.variant_id]
variant.dataset_type = dataset_type
num_updated = SavedVariant.objects.bulk_update(saved_variants, update_fields)
@@ -2328,47 +286,11 @@ def _resolve_reloaded_svs(cls, variant_ids):
if not missing_with_search_data:
return
- gcnv_id_map = cls._load_gcnv_id_map()
missing_by_sample_type = defaultdict(list)
- update_variants_by_sample_type = defaultdict(dict)
for variant in missing_with_search_data:
variant_id = variant['variant_id']
sample_type = variant['family__individual__sample__sample_type']
- update_id = SV_ID_UPDATE_MAP[sample_type].get(variant_id)
- if not update_id and sample_type == 'WES' :
- suffix = next((suff for suff in ['_DEL', '_DUP'] if variant_id.endswith(suff)), None)
- if suffix:
- base_id = gcnv_id_map.get(variant_id.rsplit(suffix)[0])
- if base_id:
- update_id = f'{base_id}{suffix}'
- if update_id:
- update_variants_by_sample_type[sample_type][variant_id] = update_id
- elif re.match(r'.*_(DEL|DUP)_\d+', variant_id):
- update_variants_by_sample_type[sample_type][variant_id] = variant_id.rsplit('_', 1)[0]
- else:
- missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" )
-
- for sample_type, variant_id_updates in sorted(update_variants_by_sample_type.items()):
- logger.info(f'Mapping reloaded SV_{sample_type} IDs to latest version')
- failed_mapping = cls._set_variant_keys(
- list(variant_id_updates.values()), f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}',
- genome_version=GENOME_VERSION_GRCh38, variant_id_updates=variant_id_updates,
- )
- if failed_mapping:
- logger.info(f'{len(failed_mapping)} variants failed ID mapping: {list(failed_mapping)}')
+ missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" )
for sample_type, variants in missing_by_sample_type.items():
logger.info(f'{len(variants)} remaining SV {sample_type} variants {", ".join(variants)}')
-
- @staticmethod
- def _load_gcnv_id_map():
- file_content = file_iter(GCNV_CALLSET_PATH)
- header = next(file_content).split('\t')
- variant_name_idx = header.index('variant_name')
- old_id_idx = header.index('any_ovl')
- id_map = {}
- for raw_row in file_content:
- row = raw_row.split('\t')
- for old_id in row[old_id_idx].split(';'):
- id_map[old_id.strip()] = row[variant_name_idx].strip()
- return id_map
diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py
index be47bf5bb5..9084105a3d 100644
--- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py
+++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py
@@ -26,6 +26,7 @@ def setUpTestData(cls):
@mock.patch('seqr.utils.file_utils.subprocess.Popen')
def test_command(self, mock_subprocess):
mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA
+ mock_subprocess.return_value.wait.return_value = 0
call_command('set_saved_variant_key')
self.assert_json_logs(user=None, expected=[
@@ -39,21 +40,14 @@ def test_command(self, mock_subprocess):
('Updated batch of 2', None),
('Updated keys for 2 SNV_INDEL (GRCh38) variants', None),
('Finding keys for 2 SV_WGS (GRCh38) variant ids', None),
- ('Found 0 keys', None),
- ('Finding keys for 2 SV_WES (GRCh38) variant ids', None),
- ('Found 0 keys', None),
- ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None),
- ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None),
- ('Mapping reloaded SV_WES IDs to latest version', None),
- ('Finding keys for 1 SV_WES (GRCh38) variant ids', None),
('Found 1 keys', None),
('Updated batch of 1', None),
- ('Updated keys for 1 SV_WES (GRCh38) variants', None),
- ('Mapping reloaded SV_WGS IDs to latest version', None),
- ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None),
+ ('Updated keys for 1 SV_WGS (GRCh38) variants', None),
+ ('No key found for 1 variants', None),
+ ('Finding keys for 1 SV_WES (GRCh38) variant ids', None),
('Found 1 keys', None),
('Updated batch of 1', None),
- ('Updated keys for 1 SV_WGS (GRCh38) variants', None),
+ ('Updated keys for 1 SV_WES (GRCh38) variants', None),
('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None),
('Found 1 keys', None),
('Updated batch of 1', None),
@@ -70,9 +64,9 @@ def test_command(self, mock_subprocess):
{'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, 'saved_variant_json': {}},
{'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000002_na19675': mock.ANY, 'I000017_na20889': mock.ANY}, 'saved_variant_json': {}},
{'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000018_na21234': {'sampleId': 'NA20885', 'ab': 0.0, 'gq': 99.0, 'numAlt': 1}}, 'saved_variant_json': {}},
- {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_70191_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}},
+ {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1, 'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}},
{'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'dataset_type': 'MITO', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
- {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
+ {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
{'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
{'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
{'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
@@ -106,6 +100,7 @@ class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest):
@mock.patch('seqr.utils.file_utils.subprocess.Popen')
def test_command(self, mock_subprocess):
mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA
+ mock_subprocess.return_value.wait.return_value = 0
call_command('set_saved_variant_key')
self.assert_json_logs(user=None, expected=[
@@ -121,12 +116,8 @@ def test_command(self, mock_subprocess):
('Finding keys for 2 SV_WES (GRCh38) variant ids', None),
('Found 0 keys', None),
('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None),
- ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None),
- ('Mapping reloaded SV_WGS IDs to latest version', None),
- ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None),
- ('Found 0 keys', None),
- ("1 variants failed ID mapping: ['suffix_19107_DEL']", None),
('1 remaining SV WES variants prefix_19107_DEL - 12', None),
+ ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None),
('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None),
('Found 0 keys', None),
('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None),
diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py
index 8f78c87766..5153c0ec1c 100644
--- a/clickhouse_search/managers.py
+++ b/clickhouse_search/managers.py
@@ -14,7 +14,8 @@
from seqr.utils.search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \
X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \
EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \
- EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, PATH_FREQ_OVERRIDE_CUTOFF, \
+ EXTENDED_SPLICE_REGION_CONSEQUENCE, CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, CLINVAR_LIKELY_PATH_FILTER, \
+ CLINVAR_CONFLICTING_P_LP, PATH_FREQ_OVERRIDE_CUTOFF, \
HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS
from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS
@@ -42,6 +43,36 @@ def _clinvar_tuple(self):
output_field=NamedTupleField(list(self.clinvar_fields.values()), null_if_empty=True, null_empty_arrays=True),
)
+ @classmethod
+ def _clinvar_path_q(cls, pathogenicity):
+ clinvar_path_filters = [
+ f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES
+ ]
+ return cls._clinvar_filter_q(clinvar_path_filters) if clinvar_path_filters else None
+
+ @classmethod
+ def _clinvar_filter_q(cls, clinvar_filters):
+ ranges = [[None, None]]
+ for path_filter, start, end in CLINVAR_PATH_RANGES:
+ if path_filter in clinvar_filters:
+ ranges[-1][1] = end
+ if ranges[-1][0] is None:
+ ranges[-1][0] = start
+ elif ranges[-1] != [None, None]:
+ ranges.append([None, None])
+ ranges = [r for r in ranges if r[0] is not None]
+
+ clinvar_qs = [cls._clinvar_range_q(path_range) for path_range in ranges]
+
+ if CLINVAR_CONFLICTING_P_LP in clinvar_filters:
+ max_path = next(end for path_filter, _, end in CLINVAR_PATH_RANGES if path_filter == CLINVAR_LIKELY_PATH_FILTER)
+ clinvar_qs.append(cls._clinvar_conflicting_path_filter({1: (max_path, "{field} <= '{value}'")}))
+
+ clinvar_q = clinvar_qs[0]
+ for q in clinvar_qs[1:]:
+ clinvar_q |= q
+ return clinvar_q
+
def _seqr_pop_fields(self, seqr_populations):
sample_types = [
sample_type.lower() for sample_type in
@@ -393,15 +424,21 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs):
return results
def _filter_in_silico(self, results, in_silico=None, **kwargs):
+ in_silico = in_silico or {}
+ require_score = in_silico.get('requireScore', False)
+
in_silico_q = None
- for score, value in (in_silico or {}).items():
+ if in_silico.get('alphamissense'):
+ in_silico_q = self._alphamissense_q(in_silico['alphamissense'], require_score)
+
+ for score, value in in_silico.items():
score_q = self._get_in_silico_score_q(score, value)
if in_silico_q is None:
in_silico_q = score_q
elif score_q:
in_silico_q |= score_q
- if in_silico_q and not in_silico.get('requireScore', False):
+ if in_silico_q and not require_score:
in_silico_q |= Q(**{f'predictions__{score}__isnull': True for score in in_silico.keys() if score in self.prediction_fields})
if in_silico_q:
@@ -421,6 +458,14 @@ def _get_in_silico_score_q(self, score, value):
except ValueError:
return Q(**{score_column: value})
+ def _alphamissense_q(self, value, require_score):
+ if not ('alphamissensePathogenicity' in self.transcript_fields and value):
+ return None
+ q = Q(**{f'{self.transcript_field}__array_exists': {'alphamissensePathogenicity': (value, '{value} <= {field}')}})
+ if not require_score:
+ q |= Q(**{f'{self.transcript_field}__array_all': {'alphamissensePathogenicity': (None, 'isNull({field})')}})
+ return q
+
def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, genes=None, intervals=None, exclude_locations=False, padded_interval_end=None, **kwargs):
transcript_field = self.transcript_field
if self.model.GENOTYPE_OVERRIDE_FIELDS:
@@ -591,34 +636,13 @@ def _hgmd_filter(hgmd):
return ('{field}__classification__range', (min_class, max_class))
return ('{field}__classification__gt', min_class)
- @classmethod
- def _clinvar_filter_q(cls, clinvar_filters, _get_range_q=None):
- ranges = [[None, None]]
- for path_filter, start, end in CLINVAR_PATH_RANGES:
- if path_filter in clinvar_filters:
- ranges[-1][1] = end
- if ranges[-1][0] is None:
- ranges[-1][0] = start
- elif ranges[-1] != [None, None]:
- ranges.append([None, None])
- ranges = [r for r in ranges if r[0] is not None]
-
- clinvar_qs = [(_get_range_q or cls._clinvar_range_q)(path_range) for path_range in ranges]
- clinvar_q = clinvar_qs[0]
- for q in clinvar_qs[1:]:
- clinvar_q |= q
- return clinvar_q
-
- @classmethod
- def _clinvar_range_q(cls, path_range):
+ @staticmethod
+ def _clinvar_range_q(path_range):
return Q(clinvar__0__range=path_range, clinvar_key__isnull=False)
- @classmethod
- def _clinvar_path_q(cls, pathogenicity, _get_range_q=None):
- clinvar_path_filters = [
- f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES
- ]
- return cls._clinvar_filter_q(clinvar_path_filters, _get_range_q=_get_range_q) if clinvar_path_filters else None
+ @staticmethod
+ def _clinvar_conflicting_path_filter(conflicting_filter):
+ return Q(clinvar__5__array_exists=conflicting_filter, clinvar_key__isnull=False)
def explode_gene_id(self, gene_id_key):
consequence_field = self.GENE_CONSEQUENCE_FIELD if self.has_annotation(self.GENE_CONSEQUENCE_FIELD) else self.transcript_field
@@ -702,9 +726,12 @@ def clinvar_model(self):
def genome_version(self):
return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion']
- def search(self, sample_data, freqs=None, annotations=None, **kwargs):
+ def search(self, sample_data, freqs=None, annotations=None, exclude_keys=None, **kwargs):
entries = self.filter_locus(**kwargs)
+ if exclude_keys:
+ entries = entries.exclude(key__in=exclude_keys)
+
entries = self._join_annotations(entries)
is_sv_class = 'cn' in self.call_fields
@@ -752,6 +779,14 @@ def result_values(self, sample_data=None):
def _has_clinvar(self):
return hasattr(self.model, 'clinvar_join')
+ @staticmethod
+ def _clinvar_range_q(path_range):
+ return Q(clinvar_join__pathogenicity__range=path_range)
+
+ @staticmethod
+ def _clinvar_conflicting_path_filter(conflicting_filter):
+ return Q(clinvar_join__conflicting_pathogenicities__array_exists=conflicting_filter)
+
def _search_call_data(self, entries, sample_data, inheritance_mode=None, inheritance_filter=None, qualityFilter=None, pathogenicity=None, annotate_carriers=False, annotate_hom_alts=False, skip_individual_guid=False, **kwargs):
project_guids = sample_data['project_guids']
project_filter = Q(project_guid__in=project_guids) if len(project_guids) > 1 else Q(project_guid=project_guids[0])
@@ -783,12 +818,10 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
quality_filter = qualityFilter or {}
individual_genotype_filter = (inheritance_filter or {}).get('genotype')
if inheritance_mode or individual_genotype_filter or quality_filter:
- clinvar_override_q = AnnotationsQuerySet._clinvar_path_q(
- pathogenicity, _get_range_q=lambda path_range: Q(clinvar_join__pathogenicity__range=path_range),
- ) if self._has_clinvar() else None
+ clinvar_override_q = self._clinvar_path_q(pathogenicity) if self._has_clinvar() else None
inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples = self._get_inheritance_quality_qs(
sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q,
- annotate_carriers, custom_affected=(inheritance_filter or {}).get('affected') or {},
+ annotate_carriers, inheritance_filter=inheritance_filter or {},
)
if quality_filter.get('vcf_filter'):
q = Q(filters__len=0)
@@ -799,20 +832,9 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
entries = entries.annotate(carriers=self._carriers_expression(unaffected_samples))
if multi_sample_type_families:
- if gt_filter:
- inheritance_q |= multi_sample_type_family_q
- entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples)
- elif inheritance_q is not None:
- entries = entries.annotate(passes_inheritance=inheritance_q)
- inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q
- else:
- entries = entries.annotate(passes_inheritance=Value(True))
-
- if quality_q is None:
- entries = entries.annotate(passes_quality=Value(True))
- else:
- entries = entries.annotate(passes_quality=quality_q)
- quality_q = Q(passes_quality=True) | multi_sample_type_family_q
+ entries, inheritance_q, quality_q = self._get_multi_sample_type_family_call_qs(
+ entries, multi_sample_type_family_q, inheritance_q, quality_q, gt_filter, family_missing_type_samples,
+ )
if inheritance_q is not None:
entries = entries.filter(inheritance_q)
@@ -821,11 +843,13 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families)
- def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected):
+ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, inheritance_filter):
samples_by_genotype = defaultdict(list)
affected_samples = []
unaffected_samples = []
family_missing_type_samples = defaultdict(lambda: defaultdict(list))
+ custom_affected = inheritance_filter.get('affected') or {}
+ allow_no_call = inheritance_filter.get('allowNoCall')
for sample in sample_data['samples']:
affected = custom_affected.get(sample['individual_guid']) or sample['affected']
genotype = self._sample_genotype(sample, affected, inheritance_mode, individual_genotype_filter)
@@ -849,22 +873,30 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i
'sampleId': (affected_samples, 'has({value}, {field})'),
})
elif samples_by_genotype:
- if all(len(self.genotype_lookup[genotype]) == 1 for genotype in samples_by_genotype.keys()):
- samples_by_gt = {self.genotype_lookup[genotype][0]: samples for genotype, samples in samples_by_genotype.items()}
+ genotype_lookup = self.genotype_lookup
+ if allow_no_call and inheritance_mode:
+ unaffected_genotype = self.INHERITANCE_FILTERS.get(inheritance_mode, {}).get(UNAFFECTED)
+ if unaffected_genotype and -1 not in genotype_lookup[unaffected_genotype]:
+ genotype_lookup = {**genotype_lookup, unaffected_genotype: [-1] + genotype_lookup[unaffected_genotype]}
+ if inheritance_mode == X_LINKED_RECESSIVE and -1 not in genotype_lookup[REF_REF]:
+ genotype_lookup = {**genotype_lookup, REF_REF: [-1] + genotype_lookup[REF_REF]}
+
+ if all(len(genotype_lookup[genotype]) == 1 for genotype in samples_by_genotype.keys()):
+ samples_by_gt = {genotype_lookup[genotype][0]: samples for genotype, samples in samples_by_genotype.items()}
gt_filter_map = ', '.join([f"{gt}, {samples_by_gt.get(gt, [])}" for gt in [-1, 0, 1, 2]])
gt_filter = (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')
else:
genotype_sample_map = ', '.join([f"'{genotype or 'any'}', {samples}" for genotype, samples in samples_by_genotype.items()])
gt_genotypes = defaultdict(list)
for genotype in samples_by_genotype.keys():
- for gt in self.genotype_lookup[genotype]:
+ for gt in genotype_lookup[genotype]:
gt_genotypes[gt].append(genotype or 'any')
gt_genotype_map = ', '.join([f"{gt}, {gt_genotypes[gt]}" for gt in [-1, 0, 1, 2]])
genotype_maps = f'genotype -> map({genotype_sample_map})[genotype], map({gt_genotype_map})'
gt_filter = (genotype_maps, 'has(arrayFlatten(arrayMap({value}[ifNull({field}, -1)])), x.sampleId)')
inheritance_q = Q(calls__array_all={'gt': gt_filter})
- quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q)
+ quality_q = self._quality_q(quality_filter, allow_no_call, affected_samples, clinvar_override_q)
return inheritance_q, quality_q, gt_filter, family_missing_type_samples, unaffected_samples
@@ -878,7 +910,7 @@ def _sample_genotype(self, sample, affected, inheritance_mode, individual_genoty
genotype = REF_REF
return genotype
- def _quality_q(self, quality_filter, affected_samples, clinvar_override_q):
+ def _quality_q(self, quality_filter, allow_no_call, affected_samples, clinvar_override_q):
quality_filter_conditions = {}
for field, scale, *filters in self.quality_filters:
@@ -888,6 +920,8 @@ def _quality_q(self, quality_filter, affected_samples, clinvar_override_q):
value = quality_filter.get(filter_key)
if value:
or_filters = ['isNull({field})', '{field} >= {value}'] + filters
+ if allow_no_call:
+ or_filters.append('isNull(x.gt)')
quality_filter_conditions[field] = (value / scale, f'or({", ".join(or_filters)})')
if not quality_filter_conditions:
@@ -906,6 +940,24 @@ def _quality_q(self, quality_filter, affected_samples, clinvar_override_q):
return quality_q
+ def _get_multi_sample_type_family_call_qs(self, entries, multi_sample_type_family_q, inheritance_q, quality_q, gt_filter, family_missing_type_samples):
+ if gt_filter:
+ inheritance_q |= multi_sample_type_family_q
+ entries = self._annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples)
+ elif inheritance_q is not None:
+ entries = entries.annotate(passes_inheritance=inheritance_q)
+ inheritance_q = Q(passes_inheritance=True) | multi_sample_type_family_q
+ else:
+ entries = entries.annotate(passes_inheritance=Value(True))
+
+ if quality_q is None:
+ entries = entries.annotate(passes_quality=Value(True))
+ else:
+ entries = entries.annotate(passes_quality=quality_q)
+ quality_q = Q(passes_quality=True) | multi_sample_type_family_q
+
+ return entries, inheritance_q, quality_q
+
@staticmethod
def _annotate_failed_family_samples(entries, gt_filter, family_missing_type_samples):
entries = entries.annotate(failed_family_samples= ArrayMap(
diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py
index 061d2c63b0..bef7024541 100644
--- a/clickhouse_search/search.py
+++ b/clickhouse_search/search.py
@@ -9,7 +9,7 @@
from clickhouse_search.backend.fields import NamedTupleField
from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \
- ArrayDistinct, ArrayMap
+ ArrayMap
from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \
BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv
from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP
@@ -18,7 +18,6 @@
from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \
PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE
from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets
-from settings import CLICKHOUSE_SERVICE_HOSTNAME
logger = SeqrLogger(__name__)
@@ -35,6 +34,8 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno
family_guid = None
inheritance_mode = search.get('inheritance_mode')
has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET}
+ exclude_keys = search.pop('exclude_keys', None) or {}
+ exclude_key_pairs = search.pop('exclude_key_pairs', None) or {}
for dataset_type, sample_data in sample_data_by_dataset_type.items():
logger.info(f'Loading {dataset_type} data for {len(set(sample_data["family_guids"]))} families', user)
@@ -45,13 +46,13 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno
dataset_results = []
if inheritance_mode != COMPOUND_HET:
- result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=is_multi_project, **search)
+ result_q = _get_search_results_queryset(entry_cls, annotations_cls, sample_data, skip_individual_guid=is_multi_project, exclude_keys=exclude_keys.get(dataset_type), **search)
dataset_results += _evaluate_results(result_q)
if has_comp_het:
comp_het_sample_data = sample_data
if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(genome_version, **search):
comp_het_sample_data = _no_affected_male_families(sample_data, user)
- result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, skip_individual_guid=is_multi_project, **search)
+ result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), skip_individual_guid=is_multi_project, **search)
dataset_results += _evaluate_results(result_q, is_comp_het=True)
if is_multi_project:
@@ -61,7 +62,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno
if has_comp_het and Sample.DATASET_TYPE_VARIANT_CALLS in sample_data_by_dataset_type and any(
dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) for dataset_type in sample_data_by_dataset_type
):
- results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, **search)
+ results += _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, **search)
cache_results = get_clickhouse_cache_results(results, sort, family_guid)
previous_search_results.update(cache_results)
@@ -84,7 +85,7 @@ def _evaluate_results(result_q, is_comp_het=False):
raise InvalidSearchException('This search returned too many results')
return results
-def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs):
+def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_by_dataset_type, user, exclude_key_pairs, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs):
if annotations_secondary:
annotations = {
**annotations,
@@ -125,7 +126,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b
sv_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][sv_dataset_type]
sv_q = sv_annotations_cls.objects.subquery_join(sv_entries).search(**search_kwargs, annotations=annotations)
- result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families))
+ result_q = _get_comp_het_results_queryset(annotations_cls, snv_indel_q, sv_q, len(families), exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'))
dataset_results = _evaluate_results(result_q, is_comp_het=True)
if skip_individual_guid:
_add_individual_guids(dataset_results, sv_sample_data)
@@ -134,7 +135,7 @@ def _get_multi_data_type_comp_het_results_queryset(genome_version, sample_data_b
return results
-def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs):
+def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_data, annotations=None, annotations_secondary=None, inheritance_mode=None, exclude_key_pairs=None, **search_kwargs):
entries = entry_cls.objects.search(
sample_data, **search_kwargs, inheritance_mode=COMPOUND_HET, annotations=annotations, annotate_carriers=True,
)
@@ -144,10 +145,10 @@ def _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, sample_
annotations=annotations_secondary or annotations, **search_kwargs,
)
- return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(set(sample_data['family_guids'])))
+ return _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, len(set(sample_data['family_guids'])), exclude_key_pairs)
-def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_families):
+def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_families, exclude_key_pairs):
results = annotations_cls.objects.search_compound_hets(primary_q, secondary_q)
if results.has_annotation('primary_carriers') and results.has_annotation('secondary_carriers'):
@@ -207,9 +208,12 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_
**genotype_expressions,
)
- return results.annotate(
+ pair_results = results.annotate(
pair_key=ArraySort(Array('primary_key', 'secondary_key')),
- ).distinct('pair_key').values_list(
+ ).distinct('pair_key')
+ if exclude_key_pairs:
+ pair_results = pair_results.exclude(pair_key__in=exclude_key_pairs)
+ return pair_results.values_list(
'pair_key',
_result_as_tuple(results, 'primary_'),
_result_as_tuple(results, 'secondary_'),
@@ -639,28 +643,6 @@ def get_clickhouse_annotations(genome_version, dataset_type, keys):
return format_clickhouse_results(results, genome_version)
-def get_clickhouse_genes(genome_version, dataset_type, keys):
- results = get_annotations_queryset(genome_version, dataset_type, keys)
- return results.aggregate(
- gene_ids=ArrayDistinct(GroupArrayArray(ArrayMap(results.transcript_field, mapped_expression='x.geneId')), output_field=ArrayField(StringField())),
- )['gene_ids']
-
-
-def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys):
- results = get_annotations_queryset(genome_version, dataset_type, keys)
- return list(results.filter(
- **{f'{results.transcript_field}__array_exists': {'geneId': (f"'{gene_id}'",)}},
- ).values_list('key', flat=True))
-
-
-def get_all_clickhouse_keys_for_gene(gene_model, genome_version):
- entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS]
- return list(entry_cls.objects.filter_locus(require_gene_filter=True, genes={gene_model.gene_id: {
- 'id': gene_model.id,
- **{f'{field}Grch{genome_version}': getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}},
- }).values_list('key', flat=True).distinct())
-
-
def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, reverse=False):
key_lookup_class = KEY_LOOKUP_CLASS_MAP[genome_version][dataset_type]
lookup = {}
diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py
index 5869c0924d..02bb5e101e 100644
--- a/clickhouse_search/search_tests.py
+++ b/clickhouse_search/search_tests.py
@@ -24,7 +24,6 @@
from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin
-@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'localhost')
class ClickhouseSearchTests(DifferentDbTransactionSupportMixin, SearchTestHelper, TestCase):
databases = '__all__'
fixtures = ['users', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts']
@@ -498,6 +497,40 @@ def test_inheritance_filter(self):
],
)
+ self._set_grch37_search()
+ self._assert_expected_search([], inheritance_mode=inheritance_mode)
+ self._assert_expected_search(
+ [GRCH37_VARIANT], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True},
+ )
+
+ def test_exclude_previous_search_results(self):
+ VariantSearchResults.objects.create(variant_search_id=79516, search_hash='abc1234')
+ self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [
+ VARIANT1, VARIANT2, [VARIANT3, VARIANT2], [GCNV_VARIANT4, GCNV_VARIANT3],
+ ]})]
+ self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']]
+
+ self._assert_expected_search(
+ [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3],
+ inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS,
+ exclude={'previousSearch': True, 'previousSearchHash': 'abc1234'}, cached_variant_fields=[
+ [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}],
+ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}],
+ {}, {},
+ ],
+ )
+
+ self.mock_redis.get.side_effect = [None, json.dumps({'all_results': [
+ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3,
+ ]})]
+ self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']]
+ self._assert_expected_search(
+ [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]],
+ inheritance_mode='recessive', cached_variant_fields=[
+ {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}],
+ ],
+ )
+
def test_quality_filter(self):
quality_filter = {'vcf_filter': 'pass'}
self._assert_expected_search(
@@ -571,16 +604,31 @@ def test_quality_filter(self):
{},
]
+ quality_filter['min_gq'] = 50
+ self._assert_expected_search(
+ [VARIANT1, selected_family_3_variant, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter,
+ annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']},
+ cached_variant_fields=cached_variant_fields[:1] + cached_variant_fields[2:],
+ )
+
self._assert_expected_search(
[VARIANT1, VARIANT2, selected_family_3_variant, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter,
- annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']}, cached_variant_fields=cached_variant_fields,
+ annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'conflicting_p_lp', 'vus_or_conflicting']},
+ cached_variant_fields=cached_variant_fields,
)
self._assert_expected_search(
[VARIANT2, selected_family_3_variant, MITO_VARIANT1], quality_filter=quality_filter,
- annotations=annotations, pathogenicity={'clinvar': ['pathogenic']}, cached_variant_fields=cached_variant_fields[1:],
+ annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp']}, cached_variant_fields=cached_variant_fields[1:],
+ )
+
+ self._set_grch37_search()
+ quality_filter = {'min_gq': 1, 'min_ab': 10}
+ self._assert_expected_search([], quality_filter=quality_filter, annotations=None, pathogenicity=None)
+ self._assert_expected_search(
+ [GRCH37_VARIANT], quality_filter=quality_filter, inheritance_filter={'allowNoCall': True},
)
-#
+
def test_location_search(self):
self._assert_expected_search(
[MULTI_FAMILY_VARIANT, VARIANT4], **LOCATION_SEARCH, cached_variant_fields=[
@@ -790,14 +838,14 @@ def test_frequency_filter(self):
annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters
self._assert_expected_search(
- [VARIANT1, VARIANT2, VARIANT4, MITO_VARIANT1],
+ [VARIANT1, VARIANT4, MITO_VARIANT1],
freqs={'gnomad_genomes': {'af': 0.01, 'hh': 10}, 'gnomad_mito': {'af': 0.01}},
annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting']},
)
self._assert_expected_search(
[VARIANT2, VARIANT4, MITO_VARIANT1], freqs={'gnomad_genomes': {'af': 0.01}, 'gnomad_mito': {'af': 0.01}},
- annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'vus_or_conflicting']},
+ annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'conflicting_p_lp', 'vus_or_conflicting']},
)
def test_annotations_filter(self):
@@ -809,6 +857,8 @@ def test_annotations_filter(self):
[VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity,
)
+ self._assert_expected_search([VARIANT2], pathogenicity={'clinvar': ['conflicting_p_lp']})
+
exclude = {'clinvar': pathogenicity['clinvar'][1:]}
pathogenicity['clinvar'] = pathogenicity['clinvar'][:1]
snv_38_only_annotations = {'SCREEN': ['CTCF-only', 'DNase-only'], 'UTRAnnotator': ['5_prime_UTR_stop_codon_loss_variant']}
@@ -1188,6 +1238,12 @@ def test_in_silico_filter(self):
[VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'requireScore': True},
)
+ self._assert_expected_search(
+ [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3],
+ in_silico={'alphamissense': 0.5},
+ )
+ self._assert_expected_search([VARIANT2], in_silico={'alphamissense': 0.5, 'requireScore': True})
+
sv_in_silico = {'strvctvre': 0.1, 'requireScore': True}
self._assert_expected_search(
[GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], in_silico=sv_in_silico,
@@ -1238,7 +1294,7 @@ def test_sort(self):
)
self._assert_expected_search(
- [VARIANT2, MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT1, MITO_VARIANT3, VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4],
+ [MITO_VARIANT1, MITO_VARIANT2, VARIANT4, VARIANT1, VARIANT2, MITO_VARIANT3, VARIANT3, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4],
sort='gnomad',
)
@@ -1267,7 +1323,7 @@ def test_sort(self):
)
self._assert_expected_search(
- [VARIANT2, VARIANT1, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='alphamissense',
+ [VARIANT2, VARIANT4, VARIANT1, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sort='alphamissense',
)
sort = 'in_omim'
diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py
index 91c001d888..0c4d2d95bc 100644
--- a/clickhouse_search/test_utils.py
+++ b/clickhouse_search/test_utils.py
@@ -128,7 +128,7 @@
'topmed': {'af': 0.246152, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604},
'exac': {'af': 0.29499999, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.41530353},
'gnomad_exomes': {'af': 0.28899795, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.4116475},
- 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0},
+ 'gnomad_genomes': {'af': 0.03444932, 'ac': 927, 'an': 26912, 'hom': 0, 'hemi': 0, 'filter_af': 0.04027665},
},
'predictions': {
'cadd': 20.9,
@@ -298,7 +298,7 @@
'transcripts': {
'ENSG00000097046': [
{'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000466716.5:c.-264+1G>A', 'hgvsp': None, 'transcriptId': 'ENST00000466716', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': ['5UTR_SPLICE']}, 'spliceregion': {'extended_intronic_splice_region_variant': True}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'splice_donor_variant', 'consequenceTerms': ['splice_donor_variant'], 'transcriptRank': 0},
- {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1},
+ {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': 0.1}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1},
{'aminoAcids': 'T/I', 'canonical': None, 'codons': 'aCc/aTc', 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'transcriptId': 'ENST00000257261', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 12}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001281501.1', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 2},
],
},
@@ -388,12 +388,12 @@
'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1, 'filters': ['VQSRTrancheSNP99.90to99.95'],
}, 'I000006_hg00733': {
'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733',
- 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306,
+ 'familyGuid': 'F000002_2', 'numAlt': None, 'dp': None, 'gq': 0, 'ab': None,
'filters': ['VQSRTrancheSNP99.90to99.95'],
},
},
'populations': {
- 'seqr': {'ac': 3, 'hom': 1},
+ 'seqr': {'ac': 2, 'hom': 1},
'topmed': {'af': 0.52131897, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149},
'exac': {'af': 0.63, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.81987739},
'gnomad_exomes': {'af': 0.63542193, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.82261163},
@@ -1250,7 +1250,7 @@
'fiveutrConsequence': None,
'geneId': 'ENSG00000097046',
}, {
- 'alphamissensePathogenicity': None,
+ 'alphamissensePathogenicity': 0.1,
'canonical': 1,
'consequenceTerms': ['missense_variant'],
'extendedIntronicSpliceRegionVariant': False,
diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py
index f3a7dfffe4..601546a59f 100644
--- a/matchmaker/views/matchmaker_api.py
+++ b/matchmaker/views/matchmaker_api.py
@@ -2,10 +2,9 @@
import requests
from datetime import datetime
from django.core.mail.message import EmailMessage
-from django.db.models import prefetch_related_objects, Q, F, Value, CharField
+from django.db.models import prefetch_related_objects, Q, Value, CharField
from django.db.models.functions import Coalesce
-from clickhouse_search.search import get_transcripts_by_key
from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \
MatchmakerIncomingQuery
from matchmaker.matchmaker_utils import get_mme_genes_phenotypes_for_results, parse_mme_patient, \
@@ -15,12 +14,11 @@
from seqr.utils.communication_utils import safe_post_to_slack
from seqr.utils.logging_utils import SeqrLogger
from seqr.utils.middleware import ErrorsWarningsException
-from seqr.utils.search.utils import backend_specific_call
from seqr.utils.xpos_utils import get_chrom_pos
from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \
create_model_from_json
from seqr.views.utils.json_utils import create_json_response
-from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants_with_tags, \
+from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants, \
get_json_for_matchmaker_submission, get_json_for_matchmaker_submissions
from seqr.views.utils.permissions_utils import check_mme_permissions, check_project_permissions, analyst_required, \
has_project_permissions, login_and_policies_required, get_project_and_check_permissions
@@ -50,44 +48,30 @@ def get_individual_mme_matches(request, submission_guid):
results = MatchmakerResult.objects.filter(submission=submission)
- response_json = get_json_for_saved_variants_with_tags(
- SavedVariant.objects.filter(family=submission.individual.family), genome_version=project.genome_version, additional_values={
- 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()),
- 'transcripts': F('saved_variant_json__transcripts'),
- }, additional_model_fields=backend_specific_call([], ['key', 'genotypes']),
- )
+ hpo_terms_by_id = get_hpo_terms_by_id(
+ {feature['id'] for feature in (submission.features or []) if feature.get('id')})
+ phenotypes = parse_mme_features(submission.features, hpo_terms_by_id)
+ response_json = _get_submission_detail_response(submission, phenotypes)
- variants = response_json['savedVariantsByGuid'].values()
- backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)(
- variants, project.genome_version, request.user,
- )
+ gene_variants = response_json['mmeSubmissionsByGuid'][submission.guid]['geneVariants']
+ gene_ids = {gv['geneId'] for gv in gene_variants}
+ variant_guids = {gv['variantGuid'] for gv in gene_variants}
- gene_ids = set()
+ variants = get_json_for_saved_variants(
+ SavedVariant.objects.filter(guid__in=variant_guids), genome_version=project.genome_version, additional_values={
+ 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()),
+ },
+ )
+ response_json['savedVariantsByGuid'] = {}
for variant in variants:
chrom, pos = get_chrom_pos(variant['xpos'])
variant.update({'chrom': chrom, 'pos': pos})
- gene_ids.update(list(variant.get('transcripts', {}).keys()))
-
- hpo_terms_by_id = get_hpo_terms_by_id(
- {feature['id'] for feature in (submission.features or []) if feature.get('id')})
- phenotypes = parse_mme_features(submission.features, hpo_terms_by_id)
- response_json.update(_get_submission_detail_response(submission, phenotypes))
+ response_json['savedVariantsByGuid'][variant['variantGuid']] = variant
return _parse_mme_results(
submission, results, request.user, additional_genes=gene_ids, response_json=response_json)
-def _add_clickhouse_transcripts(variants, genome_version, user):
- keys = {variant['key'] for variant in variants}
- try:
- transcripts_by_key = get_transcripts_by_key(genome_version, keys)
- except Exception as e:
- logger.error(f'Error loading transcripts from clickhouse: {e}', user)
- transcripts_by_key = {}
- for variant in variants:
- variant['transcripts'] = transcripts_by_key.get(variant['key'], {})
-
-
@login_and_policies_required
def get_mme_nodes(request):
return create_json_response({'mmeNodes': list(MME_NODES_BY_NAME.keys())})
diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py
index bea9a7694c..43ef6a8c8c 100644
--- a/matchmaker/views/matchmaker_api_tests.py
+++ b/matchmaker/views/matchmaker_api_tests.py
@@ -190,9 +190,7 @@ def test_get_individual_mme_matches(self):
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(set(response_json.keys()), {
- 'mmeResultsByGuid', 'genesById', 'savedVariantsByGuid', 'variantTagsByGuid',
- 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'mmeContactNotes',
- 'mmeSubmissionsByGuid',
+ 'mmeResultsByGuid', 'genesById', 'savedVariantsByGuid', 'mmeContactNotes', 'mmeSubmissionsByGuid',
})
self.assertSetEqual(
@@ -228,9 +226,9 @@ def test_get_individual_mme_matches(self):
)
self.assertSetEqual(
set(response_json['savedVariantsByGuid'].keys()),
- {'SV0000001_2103343353_r0390_100', 'SV0059957_11562437_f019313_1', 'SV0059956_11560662_f019313_1'}
+ {'SV0000001_2103343353_r0390_100'}
)
- saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'transcripts', 'genomeVersion', 'noteGuids', 'functionalDataGuids', 'tagGuids'}
+ saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'genomeVersion'}
self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields)
self.assertDictEqual(response_json['mmeContactNotes'], {})
diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json
index 4a9e17b50b..16d334cba4 100644
--- a/seqr/fixtures/1kg_project.json
+++ b/seqr/fixtures/1kg_project.json
@@ -1844,6 +1844,7 @@
"ref": "GAGA",
"alt": "G",
"variant_id": "21-3343353-GAGA-G",
+ "gene_ids": ["ENSG00000135953"],
"saved_variant_json": {
"variantId": "21-3343353-GAGA-G",
"clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null},
@@ -1895,6 +1896,7 @@
"numAlt": 1
}
},
+ "gene_ids": ["ENSG00000240361", "ENSG00000135953"],
"saved_variant_json": {
"clinvar": {
"pathogenicity": "Uncertain_significance",
@@ -2006,6 +2008,7 @@
"ref": "G",
"alt": "A",
"variant_id": "1-46859832-G-A",
+ "gene_ids": ["ENSG00000197530"],
"saved_variant_json": {
"clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null},
"liftedOverGenomeVersion": "38",
@@ -2046,6 +2049,7 @@
"ref": "G",
"alt": "CA",
"variant_id": "1-1562437-G-CA",
+ "gene_ids": ["ENSG00000197530"],
"saved_variant_json": {
"clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null},
"liftedOverGenomeVersion": "38",
@@ -2136,6 +2140,7 @@
"I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1},
"I000017_na20889": {"sampleId": "NA20885", "ab": 0.0, "ad": "71,0", "gq": 99.0, "dp": "71", "pl": "0,213,1918", "numAlt": 1}
},
+ "gene_ids": ["ENSG00000240361", "ENSG00000135953"],
"saved_variant_json": {
"clinvar": {
"pathogenicity": "Uncertain_significance",
@@ -2190,6 +2195,7 @@
"ref": null,
"alt": null,
"variant_id": "prefix_19107_DEL",
+ "gene_ids": ["ENSG00000135953", "ENSG00000223972", "ENSG00000240361"],
"saved_variant_json": {
"liftedOverGenomeVersion": null,
"pos": 249045487,
diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json
index 9ff4fdd18d..0d64249d1c 100644
--- a/seqr/fixtures/report_variants.json
+++ b/seqr/fixtures/report_variants.json
@@ -12,6 +12,7 @@
"ref": "C",
"alt": "T",
"variant_id": "19-1912634-C-T",
+ "gene_ids": ["ENSG00000240361"],
"saved_variant_json": {
"liftedOverGenomeVersion": "37",
"pos": 1912634,
@@ -57,6 +58,7 @@
"ref": "G",
"alt": "T",
"variant_id": "19-1912633-G-T",
+ "gene_ids": ["ENSG00000135953"],
"saved_variant_json": {
"liftedOverGenomeVersion": "37",
"pos": 1912633,
@@ -105,6 +107,7 @@
"ref": "G",
"alt": "C",
"variant_id": "19-1912632-G-C",
+ "gene_ids": ["ENSG00000135953"],
"saved_variant_json": {
"pos": 1912632,
"end": 1912632,
@@ -145,6 +148,7 @@
"genotypes": {
"I000018_na21234": {"sampleId": "NA20885", "numAlt": 2, "dp": 3943, "hl": 1.0, "mitoCn": 214, "contamination": 0.0, "filters": ["artifact_prone_site"]}
},
+ "gene_ids": ["ENSG00000198727"],
"saved_variant_json": {
"variantId": "M-14783-T-C", "chrom": "M", "pos": 14783, "ref": "T", "alt": "C", "xpos": 25000014783,
"genomeVersion": "38", "liftedOverGenomeVersion": "37", "liftedOverChrom": "MT", "liftedOverPos": 14783,
@@ -178,6 +182,7 @@
"genotypes": {
"I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2}
},
+ "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"],
"saved_variant_json": {
"liftedOverGenomeVersion": null,
"pos": 249045487,
diff --git a/seqr/fixtures/variant_searches.json b/seqr/fixtures/variant_searches.json
index 53529b97e3..117c0fb012 100644
--- a/seqr/fixtures/variant_searches.json
+++ b/seqr/fixtures/variant_searches.json
@@ -80,7 +80,8 @@
],
"clinvar": [
"pathogenic",
- "likely_pathogenic"
+ "likely_pathogenic",
+ "conflicting_p_lp"
]
},
"qualityFilter": {
@@ -175,7 +176,8 @@
],
"clinvar": [
"pathogenic",
- "likely_pathogenic"
+ "likely_pathogenic",
+ "conflicting_p_lp"
]
},
"qualityFilter": {
@@ -321,6 +323,7 @@
"clinvar": [
"pathogenic",
"likely_pathogenic",
+ "conflicting_p_lp",
"vus_or_conflicting"
]
},
@@ -422,6 +425,7 @@
"clinvar": [
"pathogenic",
"likely_pathogenic",
+ "conflicting_p_lp",
"vus_or_conflicting"
]
},
diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py
index 8d8fd2f417..df48683bea 100644
--- a/seqr/management/commands/deactivate_project_search.py
+++ b/seqr/management/commands/deactivate_project_search.py
@@ -1,8 +1,7 @@
from django.core.management.base import BaseCommand, CommandError
-from clickhouse_search.search import delete_clickhouse_project
from seqr.models import Project, Sample
-from seqr.utils.search.utils import backend_specific_call
+from seqr.utils.search.utils import es_only
import logging
logger = logging.getLogger(__name__)
@@ -12,6 +11,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('project')
+ @es_only
def handle(self, *args, **options):
project = Project.objects.get(guid=options['project'])
@@ -21,15 +21,3 @@ def handle(self, *args, **options):
updated = Sample.bulk_update(user=None, update_json={'is_active': False}, individual__family__project=project)
logger.info(f'Deactivated {len(updated)} samples')
-
- if updated:
- dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct()
- for dataset_type, sample_type in dataset_types:
- backend_specific_call(
- lambda *args, **kwargs: True, self._delete_clickhouse_project,
- )(project, dataset_type, sample_type)
-
- @staticmethod
- def _delete_clickhouse_project(project, dataset_type, sample_type):
- info = delete_clickhouse_project(project, dataset_type, sample_type)
- logger.info(info)
diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py
index ce3e0edbab..7c9f14efc7 100644
--- a/seqr/management/commands/reload_saved_variant_json.py
+++ b/seqr/management/commands/reload_saved_variant_json.py
@@ -1,5 +1,5 @@
import logging
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
from django.db.models.query_utils import Q
from seqr.models import Project
from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json
diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
index c3ba7986c9..f2a7c6b8ce 100644
--- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
+++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
@@ -338,6 +338,7 @@
def mock_opened_file(index):
m = mock.MagicMock()
+ m.wait.return_value = 0
m.stdout = [json.dumps(OPENED_RUN_JSON_FILES[index]).encode()]
return m
diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py
index 2a35dc9a4c..87c74a2f61 100644
--- a/seqr/management/tests/deactivate_project_search_tests.py
+++ b/seqr/management/tests/deactivate_project_search_tests.py
@@ -15,9 +15,8 @@
VARIANT_ID = '21-3343353-GAGA-G'
-class DeactivateProjectSearchTest(object):
-
- DELETE_SUCCESS_LOGS = []
+class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase):
+ fixtures = ['users', '1kg_project']
@responses.activate
@mock.patch('seqr.management.commands.deactivate_project_search.input')
@@ -47,54 +46,21 @@ def test_command(self, mock_input):
'updateType': 'bulk_update'},
}),
('Deactivated 11 samples', None),
- ] + self.DELETE_SUCCESS_LOGS)
+ ])
active_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True)
self.assertEqual(active_samples.count(), 0)
- self._assert_expected_delete()
# Re-running has no effect
self.reset_logs()
call_command('deactivate_project_search', PROJECT_GUID)
self.assert_json_logs(user=None, expected=[('Deactivated 0 samples', None)])
- def _assert_expected_delete(self):
- pass
-
-class ElasticsearchDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest):
- fixtures = ['users', '1kg_project']
-
-class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase, DeactivateProjectSearchTest):
+class ClickhouseDeactivateProjectSearchTest(AnvilAuthenticationTestCase):
fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_search']
- DELETE_SUCCESS_LOGS = [
- ('Deleted all MITO search data for project 1kg project nåme with uniçøde', None),
- ('Deleted all SNV_INDEL search data for project 1kg project nåme with uniçøde', None),
- ('Deleted all GCNV search data for project 1kg project nåme with uniçøde', None),
- ]
-
- def _assert_expected_delete(self):
- self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0)
- self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0)
- self.assertEqual(EntriesMito.objects.filter(project_guid=PROJECT_GUID).count(), 0)
- self.assertEqual(ProjectGtStatsMito.objects.filter(project_guid=PROJECT_GUID).count(), 0)
- self.assertEqual(EntriesGcnv.objects.filter(project_guid=PROJECT_GUID).count(), 0)
-
- updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop'))
- self.assertDictEqual(updated_seqr_pops_by_key, {
- 1: (2, 2, 1, 1),
- 2: (1, 1, 0, 0),
- 3: (0, 0, 0, 0),
- 4: (0, 0, 0, 0),
- 5: (1, 1, 0, 0),
- 6: (0, 0, 0, 0),
- 22: (0, 3, 0, 1),
- })
-
- updated_mito_pops_by_key = dict(AnnotationsMito.objects.all().join_seqr_pop().values_list('key', 'seqrPop'))
- self.assertDictEqual(updated_mito_pops_by_key, {
- 6: (0, 0, 0, 0),
- 7: (0, 0, 0, 0),
- 8: (0, 0, 0, 0),
- })
+ def test_command(self):
+ with self.assertRaises(ValueError) as e:
+ call_command('deactivate_project_search', 'foo')
+ self.assertEqual(str(e.exception), 'handle is disabled without the elasticsearch backend')
diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py
index ae5983fa0f..51b9f3e5c5 100644
--- a/seqr/management/tests/reload_saved_variant_json_tests.py
+++ b/seqr/management/tests/reload_saved_variant_json_tests.py
@@ -1,7 +1,7 @@
#-*- coding: utf-8 -*-
import mock
-from django.core.management import call_command, CommandError
+from django.core.management import call_command
from django.test import TestCase
PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de'
@@ -88,7 +88,6 @@ def test_with_param_command(self, mock_get_variants, mock_logger):
mock_logger.error.assert_called_with('Error reloading variants in 1kg project n\xe5me with uni\xe7\xf8de: Database error.')
-@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost')
class ClickhouseReloadSavedVariantJsonTest(TestCase):
fixtures = ['users', '1kg_project']
diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py
index f696bbc7be..1ea732e2ac 100644
--- a/seqr/management/tests/transfer_families_to_different_project_tests.py
+++ b/seqr/management/tests/transfer_families_to_different_project_tests.py
@@ -1,4 +1,3 @@
-import responses
from django.core.management import call_command
from seqr.models import Family, VariantTagType, VariantTag, Sample
diff --git a/seqr/management/tests/update_individuals_sample_qc_tests.py b/seqr/management/tests/update_individuals_sample_qc_tests.py
index 361b1b5698..4ef0c87b96 100644
--- a/seqr/management/tests/update_individuals_sample_qc_tests.py
+++ b/seqr/management/tests/update_individuals_sample_qc_tests.py
@@ -99,6 +99,7 @@ def test_command(self):
# Test 'sample_qc' not in metadata
self.mock_ls_process.communicate.return_value = b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-02-24/metadata.json\n', b''
self.mock_metadata_file.stdout = [json.dumps({}).encode()]
+ self.mock_metadata_file.wait.return_value = 0
self.mock_subprocess.side_effect = [self.mock_ls_process, self.mock_metadata_file]
with self.assertRaises(CommandError):
diff --git a/seqr/migrations/0081_savedvariant_gene_ids.py b/seqr/migrations/0081_savedvariant_gene_ids.py
new file mode 100644
index 0000000000..2d3092b515
--- /dev/null
+++ b/seqr/migrations/0081_savedvariant_gene_ids.py
@@ -0,0 +1,73 @@
+# Generated by Django 4.2.24 on 2025-09-25 19:26
+
+from collections import defaultdict
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap
+from clickhouse_search.search import get_annotations_queryset
+
+BATCH_SIZE = 10000
+
+class JsonKeys(models.Func):
+ function = 'jsonb_object_keys'
+
+
+def populate_variant_gene_ids(apps, schema_editor):
+ SavedVariant = apps.get_model('seqr', 'SavedVariant')
+ db_alias = schema_editor.connection.alias
+ variants = SavedVariant.objects.using(db_alias).filter(key__isnull=True)
+ for variant in variants:
+ variant.gene_ids = sorted(variant.saved_variant_json.get('transcripts', {}).keys(), key=lambda gene_id: _transcript_sort(gene_id, variant.saved_variant_json))
+ num_updated = SavedVariant.objects.using(db_alias).bulk_update(variants, ['gene_ids'])
+ if num_updated:
+ print(f'Populated gene_ids for {num_updated} variants')
+ for dataset_type in ['SNV_INDEL', 'MITO', 'SV_WGS', 'SV_WES']:
+ for genome_version in ['37', '38']:
+ variants = SavedVariant.objects.using(db_alias).filter(
+ key__isnull=False, family__project__genome_version=genome_version, dataset_type=dataset_type,
+ )
+ if not variants.exists():
+ continue
+ variants_by_key = defaultdict(list)
+ for v in variants:
+ variants_by_key[v.key].append(v)
+ all_keys = sorted(variants_by_key.keys())
+ print(f'Populating gene_ids for {len(all_keys)} {dataset_type} ({genome_version}) clickhouse variants')
+ for i in range(0, len(all_keys), BATCH_SIZE):
+ batch_keys = all_keys[i:i + BATCH_SIZE]
+ clickhouse_qs = get_annotations_queryset(genome_version, dataset_type, batch_keys)
+ gene_ids_expr = ArrayDistinct(ArrayMap(
+ clickhouse_qs.transcript_field, mapped_expression='x.geneId',
+ output_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=20)),
+ ))
+ to_update = []
+ for key, gene_ids in clickhouse_qs.values_list('key', gene_ids_expr):
+ for variant in variants_by_key[key]:
+ variant.gene_ids = gene_ids
+ to_update.append(variant)
+ num_updated = SavedVariant.objects.using(db_alias).bulk_update(to_update, ['gene_ids'])
+ print(f'Updated {num_updated} variants')
+
+
+def _transcript_sort(gene_id, saved_variant_json):
+ gene_transcripts = saved_variant_json['transcripts'][gene_id]
+ main_transcript_id = saved_variant_json.get('mainTranscriptId')
+ is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts)
+ return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('seqr', '0080_savedvariant_dataset_type_savedvariant_genotypes_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='savedvariant',
+ name='gene_ids',
+ field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=20), blank=True, null=True, size=None),
+ ),
+ migrations.RunPython(populate_variant_gene_ids, reverse_code=migrations.RunPython.noop),
+ ]
diff --git a/seqr/models.py b/seqr/models.py
index 4f4951e90d..ea45964737 100644
--- a/seqr/models.py
+++ b/seqr/models.py
@@ -841,6 +841,7 @@ class SavedVariant(ModelWithGUID):
selected_main_transcript_id = models.CharField(max_length=20, null=True)
saved_variant_json = models.JSONField(default=dict)
genotypes = models.JSONField(default=dict)
+ gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True)
dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES, null=True, blank=True)
acmg_classification = models.JSONField(null=True) # ACMG based classification
diff --git a/seqr/utils/file_utils.py b/seqr/utils/file_utils.py
index 23afc08a7a..5120cee0f3 100644
--- a/seqr/utils/file_utils.py
+++ b/seqr/utils/file_utils.py
@@ -82,7 +82,7 @@ def file_iter(file_path, byte_range=None, raw_content=False, user=None, **kwargs
def _google_bucket_file_iter(gs_path, byte_range=None, raw_content=False, user=None, **kwargs):
"""Iterate over lines in the given file"""
range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else ''
- process = _run_gsutil_command(
+ process = run_gsutil_with_wait(
'cat{}'.format(range_arg), gs_path, gunzip=gs_path.endswith("gz") and not raw_content, user=user, **kwargs)
for line in process.stdout:
if not raw_content:
@@ -109,8 +109,8 @@ def _get_gs_file_list(gs_path, user, check_subfolders, allow_missing):
return [line for line in all_lines if is_google_bucket_file_path(line)]
-def run_gsutil_with_wait(command, gs_path, user=None):
- process = _run_gsutil_command(command, gs_path, user=user)
+def run_gsutil_with_wait(command, gs_path, user=None, **kwargs):
+ process = _run_gsutil_command(command, gs_path, user=user, **kwargs)
if process.wait() != 0:
errors = [line.decode('utf-8').strip() for line in process.stdout]
raise Exception('Run command failed: ' + ' '.join(errors))
diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py
index f77ba0a074..da167b8789 100644
--- a/seqr/utils/search/add_data_utils.py
+++ b/seqr/utils/search/add_data_utils.py
@@ -67,7 +67,7 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N
def trigger_data_loading(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str,
genome_version: str, data_path: str, user: User, raise_error: bool = False, skip_expect_tdr_metrics: bool = True,
- skip_validation: bool = False, skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None,
+ skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None,
success_message: str = None, error_message: str = None, success_slack_channel: str = SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL):
variables = {
'projects_to_run': sorted([p.guid for p in projects]) if projects else None,
@@ -77,7 +77,6 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam
'sample_type': sample_type,
}
bool_variables = {
- 'skip_validation': skip_validation,
'skip_check_sex_and_relatedness': skip_check_sex_and_relatedness,
'skip_expect_tdr_metrics': skip_expect_tdr_metrics,
}
@@ -182,7 +181,6 @@ def get_loading_samples_validator(vcf_samples: list[str], loaded_individual_ids:
def validate_expected_samples(record_family_ids, previous_loaded_individuals, sample_type):
errors = []
- nonlocal loaded_sample_types
if loaded_sample_types is not None:
if sample_type:
loaded_sample_types.append(sample_type)
diff --git a/seqr/utils/search/constants.py b/seqr/utils/search/constants.py
index ca4ee9d9a9..c276793263 100644
--- a/seqr/utils/search/constants.py
+++ b/seqr/utils/search/constants.py
@@ -63,7 +63,8 @@
CLINVAR_KEY = 'clinvar'
CLINVAR_PATH_FILTER = 'pathogenic'
CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic'
-CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER}
+CLINVAR_CONFLICTING_P_LP = 'conflicting_p_lp'
+CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER, CLINVAR_CONFLICTING_P_LP}
PATH_FREQ_OVERRIDE_CUTOFF = 0.05
CLINVAR_PATH_RANGES = [
(CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'),
diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py
index ae6b57e4c4..36f9cfaa9a 100644
--- a/seqr/utils/search/search_utils_tests.py
+++ b/seqr/utils/search/search_utils_tests.py
@@ -279,9 +279,9 @@ def test_invalid_search_query_variants(self):
self._test_invalid_search_params(query_variants)
- def _test_expected_search_call(self, mock_get_variants, results_cache, search_fields=None, has_gene_search=False,
+ def _get_expected_search_call(self, results_cache, search_fields=None, has_gene_search=False,
rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo',
- dataset_type=None, secondary_dataset_type=None, omitted_sample_guids=None,
+ dataset_type=None, secondary_dataset_type=None, exclude_keys=None, exclude_key_pairs=None,
exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs):
genes = intervals = None
has_included_gene_search = has_gene_search and not exclude_locations
@@ -315,8 +315,16 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi
expected_search['annotations'] = annotations
if annotations_secondary is not None:
expected_search['annotations_secondary'] = annotations_secondary
+ if exclude_keys is not None:
+ expected_search['exclude_keys'] = exclude_keys
+ if exclude_key_pairs is not None:
+ expected_search['exclude_key_pairs'] = exclude_key_pairs
- mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs)
+ return (mock.ANY, expected_search, self.user, results_cache, '37'), kwargs, genes
+
+ def _test_expected_search_call(self, mock_get_variants, *args, has_gene_search=False, exclude_locations=False, omitted_sample_guids=None, **kwargs):
+ call_args, call_kwargs, genes = self._get_expected_search_call(*args, has_gene_search=has_gene_search, exclude_locations=exclude_locations, **kwargs)
+ mock_get_variants.assert_called_with(*call_args, **call_kwargs)
self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations)
if genes:
@@ -341,16 +349,17 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid
def test_query_variants(self, mock_get_variants):
+ parsed_variants = [{**v, 'key': (i+1) * 1000 } for i, v in enumerate(PARSED_VARIANTS)]
def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs):
- previous_search_results['all_results'] = PARSED_VARIANTS
+ previous_search_results['all_results'] =parsed_variants
previous_search_results['total_results'] = 5
- return PARSED_VARIANTS
+ return parsed_variants
mock_get_variants.side_effect = _mock_get_variants
variants, total = query_variants(self.results_model, user=self.user)
- self.assertListEqual(variants, PARSED_VARIANTS)
+ self.assertListEqual(variants, parsed_variants)
self.assertEqual(total, 5)
- results_cache = {'all_results': PARSED_VARIANTS, 'total_results': 5}
+ results_cache = {'all_results': parsed_variants, 'total_results': 5}
self.assert_cached_results(results_cache)
self._test_expected_search_call(
mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False,
@@ -466,6 +475,25 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v
omitted_sample_guids=['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'],
)
+ self.set_cache(None)
+ mock_get_variants.reset_mock()
+ self.search_model.search = {
+ 'inheritance': {'mode': 'any_affected'},
+ 'exclude': {'previousSearch': True, 'previousSearchHash': 'abc1234', 'clinvar': ['benign']},
+ }
+ previous_search_model = VariantSearch.objects.create(search={'inheritance': {'mode': 'de_novo'}})
+ previous_results_model = VariantSearchResults.objects.create(variant_search=previous_search_model, search_hash='abc1234')
+ previous_results_model.families.set(self.families)
+ query_variants(self.results_model, user=self.user)
+ self._test_exclude_previous_search(
+ mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False,
+ inheritance_mode='any_affected', exclude={'clinvar': ['benign']}, search_fields=['exclude'],
+ )
+
+ def _test_exclude_previous_search(self, mock_get_variants, *args, num_searches=1, **kwargs):
+ self._test_expected_search_call(mock_get_variants, *args, **kwargs)
+ self.assertEqual(mock_get_variants.call_count, num_searches)
+
def test_cached_query_variants(self):
self.set_cache({'total_results': 4, 'all_results': self.CACHED_VARIANTS})
variants, total = query_variants(self.results_model, user=self.user)
@@ -504,7 +532,6 @@ def _mock_get_gene_counts(families, search, user, previous_search_results, genom
)
-@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '')
@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost')
class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests):
databases = ['default', 'reference_data']
@@ -595,7 +622,6 @@ def _test_invalid_no_location_search_params(self):
pass
-@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost')
class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests):
databases = '__all__'
fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_transcripts']
@@ -626,6 +652,28 @@ def _assert_expected_get_single_variant_call(self, mock_call, variant_id, expect
def test_query_variants(self, mock_call):
super().test_query_variants(mock_call)
+ def _test_exclude_previous_search(self, mock_get_variants, *args, **kwargs):
+ super()._test_exclude_previous_search(
+ mock_get_variants, *args, **kwargs,
+ exclude_key_pairs={}, exclude_keys={'MITO': [1000, 2000]}, num_searches=2,
+ )
+ call_args, call_kwargs, _ = self._get_expected_search_call(*args, inheritance_mode='de_novo', sort=None, num_results=100)
+ mock_get_variants.assert_has_calls([mock.call(*call_args, **call_kwargs)])
+
+ # Test when previous results are cached
+ mock_get_variants.reset_mock()
+ self.mock_redis.get.side_effect = [
+ None,
+ json.dumps({'all_results': [VARIANT1, [VARIANT1, VARIANT2], [VARIANT1, SV_VARIANT1], VARIANT2, [VARIANT4, VARIANT3], SV_VARIANT1]}),
+ ]
+ self.mock_redis.keys.side_effect = [[], ['search_results__abc1234__gnomad']]
+ query_variants(self.results_model, user=self.user)
+ super()._test_exclude_previous_search(
+ mock_get_variants, *args, **kwargs, num_searches=1,
+ exclude_key_pairs={'SNV_INDEL': [[1, 2], [3, 4]], 'SNV_INDEL,SV_WGS': [[1, 12]]},
+ exclude_keys={'SNV_INDEL': [1, 2], 'SV_WGS': [12]},
+ )
+
def test_cached_query_variants(self):
Project.objects.filter(id=1).update(genome_version='38')
super().test_cached_query_variants()
diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py
index 02d0c5508f..d7ef84aa5a 100644
--- a/seqr/utils/search/utils.py
+++ b/seqr/utils/search/utils.py
@@ -7,7 +7,7 @@
from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \
get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
-from seqr.models import Sample, Individual, Project
+from seqr.models import Sample, Individual, Project, VariantSearchResults
from seqr.utils.logging_utils import SeqrLogger
from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json
from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \
@@ -17,7 +17,7 @@
get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \
es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS
from seqr.utils.gene_utils import parse_locus_list_items
-from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS
+from seqr.utils.xpos_utils import get_xpos, format_chrom
logger = SeqrLogger(__name__)
@@ -293,6 +293,13 @@ def _query_variants(search_model, user, previous_search_results, genome_version,
if duplicates:
raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded')
+ exclude.pop('previousSearch', None)
+ exclude_previous_hash = exclude.pop('previousSearchHash', None)
+ if exclude_previous_hash:
+ parsed_search.update(backend_specific_call(
+ lambda *args: {}, _get_clickhouse_exclude_keys,
+ )(exclude_previous_hash, user, genome_version))
+
for annotation_key in ['annotations', 'annotations_secondary']:
if parsed_search.get(annotation_key):
parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v}
@@ -323,6 +330,34 @@ def _query_variants(search_model, user, previous_search_results, genome_version,
return variant_results, previous_search_results.get('total_results')
+def _get_clickhouse_exclude_keys(search_hash, user, genome_version):
+ previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash)
+ cached_results = _get_any_sort_cached_results(previous_search_model)
+ if not cached_results:
+ cached_results = {}
+ _query_variants(previous_search_model, user, cached_results, genome_version)
+ results = cached_results['all_results']
+ exclude_keys = defaultdict(list)
+ exclude_key_pairs = defaultdict(list)
+ for variant in results:
+ if isinstance(variant, list):
+ dt1= variant_dataset_type(variant[0])
+ dt2 = variant_dataset_type(variant[1])
+ dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2]))
+ exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']]))
+ else:
+ dataset_type = variant_dataset_type(variant)
+ exclude_keys[dataset_type].append(variant['key'])
+ return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)}
+
+
+def variant_dataset_type(variant):
+ if not parse_variant_id(variant['variantId']):
+ sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES
+ return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}'
+ return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS
+
+
def get_variant_query_gene_counts(search_model, user):
return backend_specific_call(
_get_es_variant_query_gene_counts,
diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py
index 4618e275ea..73cd7381d9 100644
--- a/seqr/views/apis/anvil_workspace_api_tests.py
+++ b/seqr/views/apis/anvil_workspace_api_tests.py
@@ -427,6 +427,7 @@ def test_validate_anvil_vcf(self, mock_subprocess, mock_file_logger, mock_utils_
mock_subprocess.side_effect = [mock_file_exist_or_list_subproc, mock_get_header_subproc]
mock_file_exist_or_list_subproc.communicate.return_value = b'gs://test_bucket/test_path-001.vcf.gz\ngs://test_bucket/test_path-102.vcf.gz\n', None
mock_get_header_subproc.stdout = BASIC_META + INFO_META + FORMAT_META + HEADER_LINE + DATA_LINES
+ mock_get_header_subproc.wait.return_value = 0
response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_SHARDED_DATA_PATH))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {'fullDataPath': 'gs://test_bucket/test_path-*.vcf.gz', 'vcfSamples': ['HG00735', 'NA19675_1', 'NA19679']})
diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py
index 4c7713148c..f4daeb7b92 100644
--- a/seqr/views/apis/data_manager_api.py
+++ b/seqr/views/apis/data_manager_api.py
@@ -353,7 +353,7 @@ def load_data(request):
success = trigger_data_loading(
projects_by_guid.values(), individual_ids, sample_type, dataset_type, request_json['genomeVersion'],
- _callset_path(request_json), user=request.user, skip_validation=request_json.get('skipValidation', False),
+ _callset_path(request_json), user=request.user,
skip_check_sex_and_relatedness=request_json.get('skipSRChecks', False), vcf_sample_id_map=vcf_sample_id_map,
raise_error=is_local, skip_expect_tdr_metrics=is_local, success_message=success_message, error_message=error_message,
)
@@ -389,8 +389,8 @@ def _get_valid_search_individuals(project, airtable_samples, vcf_samples, datase
}
def fetch_missing_vcf_samples(missing_vcf_samples):
samples = _get_dataset_type_samples_for_matched_pdos(
- LOADABLE_PDO_STATUSES + AVAILABLE_PDO_STATUSES, **get_sample_kwargs,
- additional_sample_filters={'VCFIDWithMismatch': sorted(missing_vcf_samples)},
+ LOADABLE_PDO_STATUSES + AVAILABLE_PDO_STATUSES, **get_sample_kwargs, sample_fields=['VCFIDWithMismatch'],
+ additional_sample_filters={'SeqrIDWithMismatch': sorted(missing_vcf_samples)},
)
vcf_sample_id_map.update({
s['sample_id']: s['VCFIDWithMismatch'] for s in samples
@@ -425,7 +425,8 @@ def trigger_delete_project(request):
request_json = json.loads(request.body)
project_guid = request_json.pop('project')
project = Project.objects.get(guid=project_guid)
- return _trigger_data_update(delete_clickhouse_project, request_json, project)
+ samples = Sample.objects.filter(individual__family__project=project)
+ return _trigger_data_update(delete_clickhouse_project, request.user, samples, request_json, project)
@data_manager_required
@@ -434,15 +435,18 @@ def trigger_delete_family(request):
request_json = json.loads(request.body)
family_guid = request_json.pop('family')
project = Project.objects.get(family__guid=family_guid)
- return _trigger_data_update(delete_clickhouse_family, request_json, project, family_guid)
+ samples = Sample.objects.filter(individual__family__guid=family_guid)
+ return _trigger_data_update(delete_clickhouse_family, request.user, samples, request_json, project, family_guid)
-def _trigger_data_update(clickhouse_func, request_json, project, *args):
+def _trigger_data_update(clickhouse_func, user, samples, request_json, project, *args):
dataset_type = request_json.get('datasetType')
- sample_types = Sample.objects.filter(
- individual__family__project=project, dataset_type=dataset_type, is_active=True,
- ).values_list('sample_type', flat=True).distinct() if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None]
- info = []
+ samples = samples.filter(dataset_type=dataset_type, is_active=True)
+ sample_types = list(
+ samples.values_list('sample_type', flat=True).distinct()
+ ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None]
+ updated = Sample.bulk_update(user=user, update_json={'is_active': False}, queryset=samples)
+ info = [f'Deactivated search for {len(updated)} individuals']
for sample_type in sample_types:
info.append(clickhouse_func(project, *args, dataset_type=dataset_type, sample_type=sample_type))
return create_json_response({'info': info})
diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py
index a9e1818bd3..c700050910 100644
--- a/seqr/views/apis/data_manager_api_tests.py
+++ b/seqr/views/apis/data_manager_api_tests.py
@@ -14,7 +14,7 @@
from seqr.views.utils.orm_to_json_utils import _get_json_for_models
from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest
from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses
-from seqr.models import Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization
+from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization
from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL
PROJECT_GUID = 'R0001_1kg'
@@ -823,6 +823,7 @@ def _test_update_rna_seq(self, data_type, mock_open, mock_subprocess, mock_load_
mock_does_file_exist.wait.return_value = 0
mock_file_iter = mock.MagicMock()
def _set_file_iter_stdout(rows):
+ mock_file_iter.wait.return_value = 0
mock_file_iter.stdout = [('\t'.join([str(col) for col in row]) + '\n').encode() for row in rows]
mock_subprocess.side_effect = [mock_does_file_exist, mock_file_iter, mock_does_file_exist]
@@ -1377,7 +1378,7 @@ def test_get_loaded_projects(self):
self._assert_expected_airtable_errors(url)
- def _assert_expected_pm_access(self, get_response, mock_current_gene_version=None):
+ def _assert_expected_pm_access(self, get_response):
response = get_response()
self.assertEqual(response.status_code, 200)
self.login_data_manager_user()
@@ -1403,7 +1404,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m
mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp'
body = {**self.REQUEST_BODY, 'projects': [
json.dumps(option) for option in self.PROJECT_OPTIONS + [{'projectGuid': 'R0005_not_project'}]
- ], 'vcfSamples': self.VCF_SAMPLES, 'skipValidation': True}
+ ], 'vcfSamples': self.VCF_SAMPLES, 'skipSRChecks': True}
response = self.client.post(url, content_type='application/json', data=json.dumps(body))
self.assertEqual(response.status_code, 400)
self.assertDictEqual(response.json(), {'error': 'The following projects are invalid: R0005_not_project'})
@@ -1419,7 +1420,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m
)
self.assertDictEqual(response.json(), {'success': True})
- self._assert_expected_load_data_requests(sample_type='WES', skip_validation=True)
+ self._assert_expected_load_data_requests(sample_type='WES', skip_check_sex_and_relatedness=True)
self._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, 'SNV_INDEL', sample_type='WES', has_remap=bool(self.MOCK_AIRTABLE_KEY))
variables = {
@@ -1431,7 +1432,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m
'reference_genome': 'GRCh38',
'callset_path': f'{self.TRIGGER_CALLSET_DIR}/callset.vcf',
'sample_type': 'WES',
- 'skip_validation': True,
+ 'skip_check_sex_and_relatedness': True,
}
if self.SKIP_TDR:
variables['skip_expect_tdr_metrics'] = True
@@ -1445,8 +1446,8 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m
responses.calls.reset()
self.reset_logs()
- del body['skipValidation']
- del variables['skip_validation']
+ del body['skipSRChecks']
+ del variables['skip_check_sex_and_relatedness']
body.update({'datasetType': 'SV', 'filePath': f'{self.CALLSET_DIR}/sv_callset.vcf'})
self._trigger_error(url, body, variables, mock_open, mock_gzip_open, mock_mkdir)
@@ -1489,7 +1490,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m
})
self.assertEqual(len(responses.calls), 0)
- def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_validation=False):
+ def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_check_sex_and_relatedness=False):
projects = [PROJECT_GUID, NON_ANALYST_PROJECT_GUID]
if skip_project:
projects = projects[1:]
@@ -1502,8 +1503,8 @@ def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_t
}
if self.SKIP_TDR:
body['skip_expect_tdr_metrics'] = True
- if skip_validation:
- body['skip_validation'] = True
+ if skip_check_sex_and_relatedness:
+ body['skip_check_sex_and_relatedness'] = True
self.assertDictEqual(json.loads(responses.calls[-1].request.body), body)
def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_mkdir):
@@ -1757,6 +1758,7 @@ def setUp(self):
self.mock_does_file_exist = mock.MagicMock()
self.mock_file_iter = mock.MagicMock()
self.mock_file_iter.stdout = []
+ self.mock_file_iter.wait.return_value = 0
self.mock_subprocess.side_effect = [self.mock_does_file_exist, self.mock_file_iter]
self.addCleanup(patcher.stop)
patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack')
@@ -1887,8 +1889,8 @@ def _assert_expected_airtable_call(self, required_sample_field, project_guid, ca
def _assert_expected_airtable_vcf_id_call(self, required_sample_field=None, additional_vcf_ids='', **kwargs):
self._assert_expected_airtable_call(
- required_sample_field, 'R0004_non_analyst_project', **kwargs, additional_fields=['VCFIDWithMismatch'],
- additional_filter=f"OR(VCFIDWithMismatch='NA21234'{additional_vcf_ids})",
+ required_sample_field, 'R0004_non_analyst_project', **kwargs, additional_fields=['VCFIDWithMismatch', 'SeqrIDWithMismatch'],
+ additional_filter=f"OR(SeqrIDWithMismatch='NA21234'{additional_vcf_ids})",
additional_pdo_statuses=",SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';'))",
)
@@ -1942,7 +1944,7 @@ def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_m
body['sampleType'] = 'WGS'
self.assertEqual(len(responses.calls), 1)
self._assert_expected_airtable_vcf_id_call(
- required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",VCFIDWithMismatch='NA21987'",
+ required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",SeqrIDWithMismatch='NA21987'",
)
responses.calls.reset()
@@ -2053,7 +2055,10 @@ def _test_validate_dataset_type(self, url):
def _assert_expected_delete_project(self, response):
self.assertEqual(response.status_code, 200)
self.assertDictEqual(response.json(), {
- 'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de'],
+ 'info': [
+ 'Deactivated search for 7 individuals',
+ 'Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de',
+ ],
})
self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0)
self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0)
@@ -2069,12 +2074,23 @@ def _assert_expected_delete_project(self, response):
22: (0, 3, 0, 1),
})
+ project_samples = Sample.objects.filter(individual__family__project__guid=PROJECT_GUID, is_active=True)
+ self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0)
+ self.assertEqual(project_samples.count(), 4)
+
def _assert_expected_delete_family(self, response):
self.assertEqual(response.status_code, 200)
self.assertDictEqual(response.json(), {
- 'info': ['Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg'],
+ 'info': [
+ 'Deactivated search for 3 individuals',
+ 'Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg',
+ ],
})
+ family_samples = Sample.objects.filter(individual__family_id=2, is_active=True)
+ self.assertEqual(family_samples.filter(dataset_type='SV').count(), 0)
+ self.assertEqual(family_samples.count(),4)
+
def _assert_expected_airtable_errors(self, url):
responses.replace(
responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples',
diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py
index e26037621c..8b23536b52 100644
--- a/seqr/views/apis/family_api.py
+++ b/seqr/views/apis/family_api.py
@@ -8,7 +8,6 @@
from django.db.models import Count, Max, Q, F
from django.db.models.fields.files import ImageFieldFile
-from clickhouse_search.search import get_clickhouse_genes
from matchmaker.models import MatchmakerSubmission
from reference_data.models import Omim
from seqr.utils.gene_utils import get_genes_for_variant_display
@@ -58,12 +57,10 @@ def family_page_data(request, family_guid):
additional_fields = backend_specific_call([],['key', 'dataset_type'])
discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values(
- 'xpos', 'xpos_end', *additional_fields,
- svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'),
+ 'xpos', 'xpos_end', 'gene_ids', *additional_fields,
+ svType=F('saved_variant_json__svType')
)
- gene_ids = backend_specific_call(
- _variants_gene_ids, _clickhouse_variants_gene_ids,
- )(discovery_variants, project.genome_version, request.user)
+ gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']}
discovery_variant_intervals = [dict(zip(
['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'],
[*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)]
@@ -106,27 +103,6 @@ def family_page_data(request, family_guid):
return create_json_response(response)
-def _variants_gene_ids(variants, *args, **kwargs):
- return {gene_id for variant in variants for gene_id in (variant['transcripts'] or {}).keys()}
-
-
-def _clickhouse_variants_gene_ids(variants, genome_version, user):
- keys_by_dataset_type = defaultdict(set)
- no_key_variants = []
- for v in variants:
- if v['key']:
- keys_by_dataset_type[v['dataset_type']].add(v['key'])
- else:
- no_key_variants.append(v)
- gene_ids = _variants_gene_ids(no_key_variants)
- for dataset_type, keys in keys_by_dataset_type.items():
- try:
- gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys))
- except Exception as e:
- logger.error(f'Error loading genes from clickhouse: {e}', user)
- return gene_ids
-
-
def _intervals_overlap(interval1, interval2):
return interval1['chrom'] == interval2['chrom'] and (
(interval2['start'] <= interval1['start'] <= interval2['end']) or
diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py
index 19c9b98f24..c75c101669 100644
--- a/seqr/views/apis/family_api_tests.py
+++ b/seqr/views/apis/family_api_tests.py
@@ -199,10 +199,8 @@ def test_family_page_data(self):
'end': 14409,
}]},
}
- self._assert_expected_family2_data(omim_options, url.replace(FAMILY_GUID, FAMILY_GUID2), response_keys, family_fields)
- def _assert_expected_family2_data(self, omim_options, family_2_url, response_keys, family_fields):
- response = self.client.get(family_2_url)
+ response = self.client.get(url.replace(FAMILY_GUID, FAMILY_GUID2))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(set(response_json.keys()), response_keys)
@@ -231,12 +229,8 @@ def test_family_variant_tag_summary(self):
self.assertSetEqual(set(response_json.keys()), response_keys)
family = response_json['familiesByGuid'][FAMILY_GUID]
- self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryTags'})
- self.assertListEqual(family['discoveryTags'], [{
- 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]},
- 'mainTranscriptId': 'ENST00000258436',
- 'selectedMainTranscriptId': None,
- }])
+ self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryGeneIds'})
+ self.assertListEqual(family['discoveryGeneIds'], ['ENSG00000135953'])
project = response_json['projectsByGuid'][PROJECT_GUID]
self.assertSetEqual(set(project.keys()), {'variantTagTypes', 'variantFunctionalTagTypes'})
@@ -708,16 +702,3 @@ class AnvilFamilyAPITest(AnvilAuthenticationTestCase, FamilyAPITest):
EXTERNAL_ANVIL_CAN_DELETE = True
- def _assert_expected_family2_data(self, omim_options, *args, **kwargs):
- super()._assert_expected_family2_data(omim_options, *args, **kwargs)
-
- # test graceful handling when clickhouse is down
- self.reset_logs()
- connections['clickhouse'].close()
- super()._assert_expected_family2_data({}, *args, **kwargs)
- self.assert_json_logs(self.analyst_user, [
- ("Error loading genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", {
- 'severity': 'ERROR',
- '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent',
- }),
- ])
diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py
index 0d97ea3e43..55616ed4f8 100644
--- a/seqr/views/apis/individual_api_tests.py
+++ b/seqr/views/apis/individual_api_tests.py
@@ -1018,7 +1018,7 @@ def test_import_gregor_metadata(self, mock_subprocess):
'OR4G11P', '', '', '', 'Heterozygous', '', 'unknown', 'Broad_NA20889_1_248367227', '', 'Candidate',
'IRIDA syndrome', 'MONDO:0008788', 'Autosomal dominant', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '',
])
- self._set_metadata_file_iter(mock_subprocess, genetic_findings_table)
+ mock_subprocess.return_value.stdout.__iter__.return_value = [b'Bucket is a requester pays bucket but no user project provided']
url = reverse(import_gregor_metadata, args=[PM_REQUIRED_PROJECT_GUID])
self.check_pm_login(url)
@@ -1028,6 +1028,13 @@ def test_import_gregor_metadata(self, mock_subprocess):
'sampleType': 'exome',
}
response = self.client.post(url, content_type='application/json', data=json.dumps(body))
+ self.assertEqual(response.status_code, 500)
+ self.assertDictEqual(response.json(), {'error': 'Run command failed: Bucket is a requester pays bucket but no user project provided'})
+
+ mock_subprocess.reset_mock()
+ self._set_metadata_file_iter(mock_subprocess, genetic_findings_table)
+ mock_subprocess.return_value.wait.return_value = 0
+ response = self.client.post(url, content_type='application/json', data=json.dumps(body))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(set(response_json.keys()), {
@@ -1146,7 +1153,7 @@ def test_import_gregor_metadata(self, mock_subprocess):
).order_by('family_id', 'variant_id').distinct().values(
'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json__genomeVersion',
'saved_variant_json__transcripts', 'saved_variant_json__genotypes', 'saved_variant_json__mainTranscriptId',
- 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes'
+ 'saved_variant_json__hgvsc', 'key', 'dataset_type', 'genotypes', 'gene_ids',
)
self.assertEqual(len(saved_variants), 4)
self.assertDictEqual(saved_variants[0], {
@@ -1162,6 +1169,7 @@ def test_import_gregor_metadata(self, mock_subprocess):
'key': 100,
'dataset_type': 'SNV_INDEL',
'genotypes': mock.ANY,
+ 'gene_ids': ['ENSG00000240361', 'ENSG00000135953'],
})
self.assertEqual(len(saved_variants[0]['genotypes']), 2)
self.assertDictEqual(saved_variants[1], {
@@ -1179,6 +1187,7 @@ def test_import_gregor_metadata(self, mock_subprocess):
'key': None,
'dataset_type': None,
'genotypes': {},
+ 'gene_ids': ['ENSG00000240361'],
})
new_family_genotypes = {new_family_individual_guid: {'numAlt': 2}}
self.assertDictEqual(saved_variants[2], {
@@ -1194,6 +1203,7 @@ def test_import_gregor_metadata(self, mock_subprocess):
'key': 100,
'dataset_type': 'SNV_INDEL',
'genotypes': new_family_genotypes,
+ 'gene_ids': ['ENSG00000135953'],
})
variant_tags = VariantTag.objects.filter(variant_tag_type__name='GREGoR Finding')
@@ -1224,14 +1234,19 @@ def test_import_gregor_metadata(self, mock_subprocess):
mock_subprocess.assert_has_calls([
mock.call('gsutil cat gs://test_bucket/data_tables/experiment_dna_short_read.tsv', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
mock.call('gsutil cat gs://test_bucket/data_tables/experiment.tsv', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
mock.call('gsutil cat gs://test_bucket/data_tables/participant.tsv', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
mock.call('gsutil cat gs://test_bucket/data_tables/phenotype.tsv', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
mock.call('gsutil cat gs://test_bucket/data_tables/genetic_findings.tsv', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
])
diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py
index 81d8c13f87..80ba7b1c14 100644
--- a/seqr/views/apis/project_api_tests.py
+++ b/seqr/views/apis/project_api_tests.py
@@ -41,20 +41,11 @@
{'id': 'recSgwrXNkmlIB5eM', 'fields': {'Status': 'Available in Seqr'}},
]}
-DISCOVERY_TAG = {
- 'mainTranscriptId': None,
- 'selectedMainTranscriptId': 'ENST00000371839',
- 'transcripts': {
- 'ENSG00000135953': mock.ANY,
- 'ENSG00000240361': mock.ANY,
- },
-}
class ProjectAPITest(object):
CREATE_PROJECT_JSON = WORKSPACE_CREATE_PROJECT_JSON
REQUIRED_FIELDS = ['name', 'genomeVersion', 'workspaceNamespace', 'workspaceName']
AIRTABLE_TRACKING_URL = f'{MOCK_AIRTABLE_URL}/appUelDNM3BnWaR7M/AnVIL%20Seqr%20Loading%20Requests%20Tracking'
- DISCOVERY_TAG = DISCOVERY_TAG
@mock.patch('seqr.views.utils.airtable_utils.logger')
@mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_URL', MOCK_AIRTABLE_URL)
@@ -387,8 +378,7 @@ def test_project_families(self):
empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID])
self._check_empty_project(empty_url, response_keys)
- gene_ids = self._assert_expected_project_families(url, response_keys)
- self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'})
+ self._assert_expected_project_families(url, response_keys)
def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False):
response = self.client.get(url)
@@ -401,7 +391,7 @@ def _assert_expected_project_families(self, url, response_keys, no_discovery_tag
family_3 = response_json['familiesByGuid']['F000003_3']
empty_family = response_json['familiesByGuid']['F000013_13']
family_fields = {
- 'individualGuids', 'discoveryTags', 'caseReviewStatuses', 'caseReviewStatusLastModified', 'hasRequiredMetadata',
+ 'individualGuids', 'discoveryGeneIds', 'caseReviewStatuses', 'caseReviewStatusLastModified', 'hasRequiredMetadata',
'parents', 'hasPhenotypePrioritization', 'hasRna', 'externalData',
}
family_fields.update(SUMMARY_FAMILY_FIELDS)
@@ -434,22 +424,17 @@ def _assert_expected_project_families(self, url, response_keys, no_discovery_tag
self.assertListEqual(empty_family['externalData'], [])
- self.assertListEqual(family_3['discoveryTags'], [])
- self.assertListEqual(empty_family['discoveryTags'], [])
- family_1_tags = [] if no_discovery_tags else [{
- 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]},
- 'mainTranscriptId': 'ENST00000258436',
- 'selectedMainTranscriptId': None,
- }]
- self.assertListEqual(family_1['discoveryTags'], family_1_tags)
- family_2_tags = [] if no_discovery_tags else [self.DISCOVERY_TAG]
- self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], family_2_tags)
+ self.assertListEqual(family_3['discoveryGeneIds'], [])
+ self.assertListEqual(empty_family['discoveryGeneIds'], [])
+ self.assertListEqual(family_1['discoveryGeneIds'], ['ENSG00000135953'])
+ family_2_tags = [] if no_discovery_tags else ['ENSG00000135953']
+ self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryGeneIds'], family_2_tags)
no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'}
self.assertSetEqual({
- len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families
+ len(response_json['familiesByGuid'][family_guid]['discoveryGeneIds']) for family_guid in no_discovery_families
}, {0})
- return set(response_json['genesById'].keys())
+ self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953'})
def test_project_individuals(self):
url = reverse(project_individuals, args=[PROJECT_GUID])
@@ -722,7 +707,6 @@ class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest):
PROJECT_COLLABORATORS = ANVIL_COLLABORATORS
PROJECT_COLLABORATOR_GROUPS = None
HAS_EMPTY_PROJECT = False
- DISCOVERY_TAG = {**DISCOVERY_TAG, 'mainTranscriptId': 'ENST00000505820'}
def test_create_and_delete_project(self, *args, **kwargs):
super(AnvilProjectAPITest, self).test_create_and_delete_project(*args, **kwargs)
@@ -798,20 +782,15 @@ def test_project_collaborators(self):
self.assertEqual(self.mock_get_ws_access_level.call_count, 5)
def _assert_expected_project_families(self, *args, **kwargs):
- gene_ids = super()._assert_expected_project_families(*args, **kwargs)
+ super()._assert_expected_project_families(*args, **kwargs)
# Test success when clickhouse is unavailable
self.reset_logs()
connections['clickhouse'].close()
- self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}}
- no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True)
- self.assertSetEqual(no_clickhouse_gene_ids, set())
+ super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True)
self.assert_json_logs(None, [
("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", {
'severity': 'ERROR',
'@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent',
}),
])
-
- return gene_ids
-
diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py
index 8a75fb697b..6fa1fd5b29 100644
--- a/seqr/views/apis/saved_variant_api.py
+++ b/seqr/views/apis/saved_variant_api.py
@@ -5,7 +5,7 @@
from django.db.models import Q, F
from clickhouse_search.backend.functions import ArrayFilter, ArrayMap
-from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset
+from clickhouse_search.search import get_transcripts_queryset
from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\
Family, GeneNote, Project
from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json
@@ -137,26 +137,20 @@ def _create_variant_note(saved_variants, note_json, user, genome_version):
def _variant_gene_id(variant, genome_version):
- main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId')
- if main_transcript_id:
+ if variant.selected_main_transcript_id and len(variant.gene_ids) > 1:
return next(
gene_id for gene_id, transcripts in variant.saved_variant_json['transcripts'].items()
- if any(t['transcriptId'] == main_transcript_id for t in transcripts))
- return next(gene_id for gene_id in sorted(variant.saved_variant_json['transcripts']))
+ if any(t['transcriptId'] == variant.selected_main_transcript_id for t in transcripts))
+ return variant.gene_ids[0]
def _clickhouse_variant_gene_id(variant, genome_version):
- if not variant.key:
+ if not (variant.key and variant.selected_main_transcript_id) or len(variant.gene_ids) == 1:
return _variant_gene_id(variant, genome_version)
- if variant.selected_main_transcript_id:
- qs = get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap(
- ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]),
- mapped_expression='x.geneId', output_field=ArrayField(StringField()),
- )).annotate(gene_id=F('gene_ids__0'))
- else:
- annotations = get_annotations_queryset(genome_version, variant.dataset_type, [variant.key])
- qs = annotations.annotate(gene_id=F(f'{annotations.transcript_field}__0__geneId'))
- return qs.values_list('gene_id', flat=True)[0]
+ return get_transcripts_queryset(genome_version, [variant.key]).annotate(gene_ids=ArrayMap(
+ ArrayFilter('transcripts', conditions=[{'transcriptId': (variant.selected_main_transcript_id, "{field} = '{value}'")}]),
+ mapped_expression='x.geneId', output_field=ArrayField(StringField()),
+ )).values_list(F('gene_ids__0'), flat=True)[0]
@login_and_policies_required
diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py
index 9c13be3624..c4fbb5c38c 100644
--- a/seqr/views/apis/saved_variant_api_tests.py
+++ b/seqr/views/apis/saved_variant_api_tests.py
@@ -95,7 +95,7 @@
'chrom': '2',
'genotypes': {},
'genomeVersion': '37',
- 'mainTranscriptId': None,
+ 'mainTranscriptId': 'ENST00000450625',
'populations': {
'seqr': {'ac': 0, 'hom': 0},
'exac': {'ac': 0, 'af': 0.0, 'an': 0, 'filter_af': 0.0, 'hemi': 0, 'het': 0, 'hom': 0},
@@ -121,7 +121,25 @@
'splice_ai_consequence': None,
'vest': None},
'ref': 'AAAG',
- 'transcripts': {},
+ 'transcripts': {
+ 'ENSG00000177000': [
+ {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0},
+ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': {
+ 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant',
+ 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None},
+ }},
+ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426925.5:c.-677A>C', 'hgvsp': None, 'transcriptId': 'ENST00000426925', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 11}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 2, 'utrannotator': {
+ 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 8,'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant',
+ 'fiveutrAnnotation': {'type': None, 'KozakContext': 'TCAATGC', 'KozakStrength': 'Weak', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 588, 'FrameWithCDS': 'inFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None, 'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None},
+ }},
+ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000412806.1:c.138+1722A>C', 'hgvsp': None, 'transcriptId': 'ENST00000412806', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3},'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 3},
+ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000459627.1:n.298A>C', 'hgvsp': None, 'transcriptId': 'ENST00000459627', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 3}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 4},
+ {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000465894.6:n.33A>C', 'hgvsp': None, 'transcriptId': 'ENST00000465894', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 5},
+ ],
+ 'ENSG00000277258': [
+ {'aminoAcids': 'L/F', 'canonical': None, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000277258', 'hgvsc': 'ENST00000450625.1:c.156A>C', 'hgvsp': 'ENSP00000389484.1:p.Leu52Phe', 'transcriptId': 'ENST00000450625', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9978}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant', 'NMD_transcript_variant'], 'transcriptRank': 6},
+ ]
+ },
'familyGuids': ['F000001_1', 'F000002_2'],
'variantId': '2-61413835-AAAG-A',
'CAID': None,
@@ -385,7 +403,7 @@ def test_create_saved_variant(self):
saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1')
variant_json = {'xpos': 2061413835}
variant_json.update(create_variant_json)
- self._assert_created_variant(saved_variant, variant_json)
+ self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258', 'ENSG00000177000'])
variant_json.update({
'variantGuid': variant_guid,
@@ -394,6 +412,8 @@ def test_create_saved_variant(self):
'selectedMainTranscriptId': None,
'noteGuids': [],
'functionalDataGuids': [],
+ 'transcripts': mock.ANY,
+ 'mainTranscriptId': mock.ANY,
})
response_json = response.json()
response_variant_json = response_json['savedVariantsByGuid'][variant_guid]
@@ -415,9 +435,10 @@ def test_create_saved_variant(self):
def _format_create_variant(self, variant_json, **kwargs):
return variant_json
- def _assert_created_variant(self, saved_variant, variant_json, **kwargs):
+ def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, **kwargs):
for field in ['xpos', 'ref', 'alt']:
self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None))
+ self.assertEqual(saved_variant.gene_ids, gene_ids or [])
def test_create_saved_sv_variant(self):
# SVs are only supported on build 38
@@ -465,7 +486,7 @@ def test_create_saved_sv_variant(self):
saved_variant = SavedVariant.objects.get(guid=variant_guid, family__guid='F000001_1')
variant_json.update({'xpos': 2061413835})
- self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES')
+ self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', gene_ids=['ENSG00000240361'])
self.assertEqual(saved_variant.xpos_end, 2061414175)
variant_json.update({
@@ -1030,7 +1051,7 @@ class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest):
SAVED_VARIANT_DETAIL_FIELDS = SAVED_VARIANT_DETAIL_FIELDS
def _assert_created_variant(self, saved_variant, variant_json, **kwargs):
- super()._assert_created_variant(saved_variant, variant_json)
+ super()._assert_created_variant(saved_variant, variant_json, **kwargs)
self.assertDictEqual(variant_json, saved_variant.saved_variant_json)
@@ -1121,7 +1142,7 @@ def _format_create_variant(self, variant_json, key=123, **kwargs):
return {**variant_json, 'key': key}
def _assert_created_variant(self, saved_variant, variant_json, dataset_type='SNV_INDEL', **kwargs):
- super()._assert_created_variant(saved_variant, variant_json)
+ super()._assert_created_variant(saved_variant, variant_json, **kwargs)
self.assertEqual(variant_json['key'], saved_variant.key)
self.assertEqual(dataset_type, saved_variant.dataset_type)
self.assertDictEqual(variant_json['genotypes'], saved_variant.genotypes)
diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py
index 1b0f7f621b..4a4940fc73 100644
--- a/seqr/views/apis/summary_data_api.py
+++ b/seqr/views/apis/summary_data_api.py
@@ -1,25 +1,22 @@
from collections import defaultdict
from datetime import datetime
-from django.contrib.postgres.aggregates import ArrayAgg
from django.core.exceptions import PermissionDenied
from django.core.mail.message import EmailMessage
from django.contrib.auth.models import User
-from django.db.models import CharField, F, Q, Value
+from django.db.models import CharField, F, Value
from django.db.models.functions import Coalesce, Concat, JSONObject, NullIf
import json
-from clickhouse_search.search import get_clickhouse_keys_for_gene, get_all_clickhouse_keys_for_gene
from matchmaker.matchmaker_utils import get_mme_gene_phenotype_ids_for_submissions, parse_mme_features, \
get_mme_metrics, get_hpo_terms_by_id
from matchmaker.models import MatchmakerSubmission
-from reference_data.models import HumanPhenotypeOntology, GeneInfo, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38
+from reference_data.models import HumanPhenotypeOntology
from seqr.models import Project, Family, Individual, VariantTagType, SavedVariant, FamilyAnalysedBy, Sample
from seqr.views.utils.airtable_utils import AirtableSession
from seqr.views.utils.file_utils import load_uploaded_file
from seqr.utils.communication_utils import safe_post_to_slack, set_email_message_stream
from seqr.utils.gene_utils import get_genes
from seqr.utils.middleware import ErrorsWarningsException
-from seqr.utils.search.utils import backend_specific_call
from seqr.views.utils.json_utils import create_json_response
from seqr.utils.logging_utils import SeqrLogger
from seqr.utils.xpos_utils import get_chrom_pos
@@ -115,11 +112,7 @@ def saved_variants_page(request, tag):
saved_variant_models = saved_variant_models.filter(family__project__guid__in=get_project_guids_user_can_view(request.user))
if gene:
- gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call(
- lambda *args: Q(),
- _saved_variant_with_clickhouse_gene_q,
- )(saved_variant_models, gene, is_all_tags)
- saved_variant_models = saved_variant_models.filter(gene_filter)
+ saved_variant_models = saved_variant_models.filter(gene_ids__overlap=[gene])
elif saved_variant_models.count() > MAX_SAVED_VARIANTS:
return create_json_response({'error': 'Select a gene to filter variants'}, status=400)
@@ -131,35 +124,6 @@ def saved_variants_page(request, tag):
return create_json_response(response_json)
-def _saved_variant_with_clickhouse_gene_q(saved_variant_models, gene_id, is_all_tags):
- key_qs = []
- if is_all_tags:
- gene_model = GeneInfo.objects.get(gene_id=gene_id)
- for genome_version in [GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38]:
- gene_keys = get_all_clickhouse_keys_for_gene(gene_model, genome_version)
- if gene_keys:
- key_qs.append(
- Q(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, family__project__genome_version=genome_version, key__in=gene_keys)
- )
- saved_variant_models = saved_variant_models.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS)
-
- search_type_keys = saved_variant_models.filter(key__isnull=False).values(
- 'dataset_type', genome_version=F('family__project__genome_version'),
- ).annotate(keys=ArrayAgg('key', distinct=True))
- for agg in search_type_keys:
- gene_keys = get_clickhouse_keys_for_gene(gene_id, **agg)
- if gene_keys:
- q = Q(dataset_type=agg['dataset_type'], family__project__genome_version=agg['genome_version'], key__in=gene_keys)
- key_qs.append(q)
-
- if not key_qs:
- return Q()
- has_key_q = key_qs[0]
- for q in key_qs[1:]:
- has_key_q |= q
- return has_key_q
-
-
@login_and_policies_required
def hpo_summary_data(request, hpo_id):
data = Individual.objects.filter(
diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py
index 7df3cad33e..92fecee045 100644
--- a/seqr/views/apis/summary_data_api_tests.py
+++ b/seqr/views/apis/summary_data_api_tests.py
@@ -628,6 +628,7 @@ def _assert_expected_new_saved_variant(self, new_saved_variant):
self.assertEqual(new_saved_variant.family_id, 2)
self.assertEqual(new_saved_variant.ref, 'GAGA')
self.assertEqual(new_saved_variant.alt, 'G')
+ self.assertListEqual(new_saved_variant.gene_ids, ['ENSG00000135953', 'ENSG00000228198'])
def _has_expected_metadata_response(self, response, expected_individuals, has_airtable=False, has_duplicate=False):
self.assertEqual(response.status_code, 200)
diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py
index a52b3cbb1a..01691142e2 100644
--- a/seqr/views/apis/variant_search_api.py
+++ b/seqr/views/apis/variant_search_api.py
@@ -113,6 +113,8 @@ def _get_or_create_results_model(search_hash, search_context, user):
).distinct()
search_dict = search_context.get('search', {})
+ if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'):
+ search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash']
search_model = VariantSearch.objects.filter(search=search_dict).filter(
Q(created_by=user) | Q(name__isnull=False)).first()
if not search_model:
diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py
index c20bb87f08..e1cdaa263e 100644
--- a/seqr/views/apis/variant_search_api_tests.py
+++ b/seqr/views/apis/variant_search_api_tests.py
@@ -528,7 +528,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro
['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
'', '2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)', '', '', '', '', '', '',
'', '', '', '', '', '', '', '', '', '', '', ''],
- ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0',
+ ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0.03444932',
'0.28899795', '0.246152', '20.9', '0.197',
'2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C',
'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', '', '', '', 'HG00731', '2', '', '99', '1.0',
@@ -562,7 +562,7 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro
['12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
'2', 'AIP (None)|Known gene for phenotype (None)|Excluded (None)', 'a later note (None)|test n\xf8te (None)',
'', '', '', '', '', '', '', '', '', '', '', '', '', '', '',],
- ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0',
+ ['1', '38724419', 'T', 'G', 'ENSG00000177000', 'missense_variant', '10', '0.29499999', '0.03444932',
'0.28899795', '0.246152', '20.9', '0.197',
'2.001', '0.0', '0.1', '0.05', '', '', 'rs1801131', 'ENST00000383791.8:c.156A>C',
'ENSP00000373301.3:p.Leu52Phe', 'Conflicting_classifications_of_pathogenicity', '1', '2', '', '', 'HG00731', '2', '', '99', '1.0',
@@ -641,16 +641,20 @@ def test_query_variants(self, mock_get_variants, mock_get_gene_counts, mock_erro
mock_error_logger.assert_not_called()
# Test no results
+ VariantSearchResults.objects.get(search_hash=SEARCH_HASH).delete()
mock_get_variants.side_effect = _get_empty_es_variants
response = self.client.post(url, content_type='application/json', data=json.dumps({
- 'projectFamilies': PROJECT_FAMILIES, 'search': SEARCH
+ 'projectFamilies': PROJECT_FAMILIES, 'search': {**SEARCH, 'exclude': {'previousSearch': True}}, 'previousSearchHash': 'abc1234',
}))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertDictEqual(response_json, {
'searchedVariants': [],
'search': {
- 'search': SEARCH,
+ 'search': {**SEARCH, 'exclude': {
+ 'previousSearch': True,
+ 'previousSearchHash': 'abc1234',
+ }},
'projectFamilies': PROJECT_FAMILIES,
'totalResults': 0,
}
diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py
index 33518adef3..f2cbc40e18 100644
--- a/seqr/views/status_tests.py
+++ b/seqr/views/status_tests.py
@@ -8,7 +8,6 @@
from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses
-@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '')
@mock.patch('seqr.views.status.redis.StrictRedis')
@mock.patch('seqr.views.status.logger')
class ElasticsearchStatusTest(TestCase):
diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py
index 0031ed61b5..c8d458f1be 100644
--- a/seqr/views/utils/airtable_utils.py
+++ b/seqr/views/utils/airtable_utils.py
@@ -171,7 +171,7 @@ def get_samples_for_sample_ids(self, sample_ids, fields):
records_by_id.update(self._get_samples_for_id_field(missing, 'SeqrCollaboratorSampleID', fields))
return records_by_id
- def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, additional_sample_filters=None, project_guid=None, required_sample_fields=None):
+ def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, sample_fields=None, additional_sample_filters=None, project_guid=None, required_sample_fields=None):
pdo_fields = pdo_fields or []
additional_and_filters = [
f'LEN({{{required_sample_field}}})>0' for required_sample_field in required_sample_fields or []
@@ -182,7 +182,7 @@ def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, additional
sample_records = self.fetch_records(
'Samples', fields=[
'CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', *pdo_fields,
- *(additional_sample_filters or {}).keys(),
+ *(sample_fields or []), *(additional_sample_filters or {}).keys(),
],
or_filters={'PDOStatus': pdo_statuses},
and_filters={'SeqrProject': f'{BASE_URL}project/{project_guid}/project_page'} if project_guid else {},
diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py
index ec867c784d..02b4af65a6 100644
--- a/seqr/views/utils/anvil_metadata_utils.py
+++ b/seqr/views/utils/anvil_metadata_utils.py
@@ -1,7 +1,6 @@
from collections import defaultdict
from datetime import datetime
-from clickhouse_backend.models import ArrayField, StringField
from django.db.models import F, Q, Value, CharField, Aggregate
from django.db.models.functions import Replace
from django.contrib.auth.models import User
@@ -9,7 +8,7 @@
import requests
from typing import Callable, Iterable
-from clickhouse_search.backend.functions import ArrayMap, ArrayFilter
+from clickhouse_search.backend.functions import ArrayFilter
from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset
from matchmaker.models import MatchmakerSubmission
from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP
@@ -411,7 +410,7 @@ def _get_variant_json_by_guid(saved_variants, *args, **kwargs):
for v in saved_variants:
main_transcript = _get_variant_main_transcript(v)
gene_id = main_transcript.get('geneId')
- gene_ids = [gene_id] if gene_id else v.saved_variant_json.get('transcripts', {}).keys()
+ gene_ids = [gene_id] if gene_id else v.gene_ids
variant_json_by_guid[v.guid] = {
**v.saved_variant_json, 'main_transcript': main_transcript, 'gene_id': gene_id, 'gene_ids': gene_ids,
}
@@ -426,7 +425,7 @@ def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar):
for v in saved_variants:
if v.key:
end_chrom, end = get_chrom_pos(v.xpos_end)
- variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end}
+ variant_json_by_guid[v.guid] = {'genotypes': v.genotypes, 'endChrom': end_chrom, 'end': end, 'gene_ids': v.gene_ids}
variant_keys_by_search_type[v.genome_version][v.dataset_type][v.key].append((v.guid, v.selected_main_transcript_id))
else:
no_key_variants.append(v)
@@ -442,7 +441,7 @@ def _get_clickhouse_variant_json_by_guid(saved_variants, include_clinvar):
def _set_clickhouse_sv_json(variant_json_by_guid, genome_version, dataset_type, key_map, include_clinvar):
annotations = get_annotations_queryset(genome_version, dataset_type, key_map.keys()).values(
- 'key', svType=F('sv_type'), gene_ids=ArrayMap('sorted_gene_consequences', mapped_expression='x.geneId', output_field=ArrayField(StringField())),
+ 'key', svType=F('sv_type'),
)
defaults = {'CAID': None, 'gene_id': None, 'main_transcript': {}}
if include_clinvar:
@@ -486,8 +485,9 @@ def _set_clickhouse_snv_indel_json(variant_json_by_guid, genome_version, dataset
**anns,
'main_transcript': main_transcript,
'gene_id': gene_id,
- 'gene_ids': [gene_id] if gene_id else [transcript['geneId'] for transcript in all_transcripts],
})
+ if gene_id:
+ variant_json_by_guid[guid]['gene_ids'] = [gene_id]
def _get_main_transcripts_expression(field_name, qs, selected_transcripts):
diff --git a/seqr/views/utils/file_utils_tests.py b/seqr/views/utils/file_utils_tests.py
index 544183691d..1c4c2f15df 100644
--- a/seqr/views/utils/file_utils_tests.py
+++ b/seqr/views/utils/file_utils_tests.py
@@ -154,6 +154,7 @@ def test_temp_file_upload(self, *args, **kwargs):
mock.call(f'gsutil mv {self._temp_file_path()} {gs_file}', stdout=-1, stderr=-2, shell=True), # nosec
mock.call().wait(),
mock.call(f'gsutil cat {gs_file} | gunzip -c -q - ', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
mock.call().stdout.__iter__(),
])
diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py
index 546aa2fb2c..8e96d02f1a 100644
--- a/seqr/views/utils/project_context_utils.py
+++ b/seqr/views/utils/project_context_utils.py
@@ -118,25 +118,29 @@ def add_child_ids(response):
def families_discovery_tags(families, genome_version, project=None):
- families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families}
+ families_by_guid = {f['familyGuid']: dict(discoveryGeneIds=[], **f) for f in families}
family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()}
discovery_variants = SavedVariant.objects.filter(
varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter,
- )
+ ).annotate(family_guid=F('family__guid'))
+
+ family_discovery_genes = list(discovery_variants.filter(
+ Q(gene_ids__len=1) | (Q(selected_main_transcript_id__isnull=True) & ~Q(dataset_type__startswith='SV'))
+ ).values_list('family_guid', F('gene_ids__0')))
try:
- discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)(
- discovery_variants, genome_version=genome_version,
+ backend_specific_call(_get_no_key_selected_transcript_gene_id, _get_clickhouse_selected_transcript_gene_id)(
+ family_discovery_genes,
+ discovery_variants.filter(selected_main_transcript_id__isnull=False, gene_ids__len__gt=1),
+ genome_version=genome_version,
)
except Exception as e:
logger.error(f'Error loading discovery genes from clickhouse: {e}', None)
- discovery_tags = []
gene_ids = set()
- for tag in discovery_tags:
- tag['transcripts'] = tag.get('transcripts') or {}
- gene_ids.update(list(tag['transcripts'].keys()))
- families_by_guid[tag.pop('family_guid')]['discoveryTags'].append(tag)
+ for family_guid, gene_id in family_discovery_genes:
+ gene_ids.add(gene_id)
+ families_by_guid[family_guid]['discoveryGeneIds'].append(gene_id)
return {
'familiesByGuid': families_by_guid,
@@ -144,37 +148,39 @@ def families_discovery_tags(families, genome_version, project=None):
}
-def _get_no_key_tags(discovery_variants, **kwargs):
- return discovery_variants.values(
- family_guid=F('family__guid'), selectedMainTranscriptId=F('selected_main_transcript_id'),
- transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'),
- )
+def _get_transcripts_selected_gene(selected_main_transcript_id, transcripts):
+ return next((
+ gene_id for gene_id, gene_transcripts in (transcripts or {}).items()
+ if any(t.get('transcriptId') == selected_main_transcript_id for t in gene_transcripts)
+ ), None)
+
+
+def _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants, **kwargs):
+ family_discovery_genes += [
+ (family_guid, _get_transcripts_selected_gene(selected_main_transcript_id, transcripts))
+ for family_guid, selected_main_transcript_id, transcripts in discovery_variants.values_list(
+ 'family_guid', 'selected_main_transcript_id', 'saved_variant_json__transcripts',
+ )
+ ]
-def _get_clickhouse_tags(discovery_variants, genome_version):
- discovery_tags = list(_get_no_key_tags(discovery_variants.filter(key__isnull=True)))
+def _get_clickhouse_selected_transcript_gene_id(family_discovery_genes, discovery_variants, genome_version):
+ _get_no_key_selected_transcript_gene_id(family_discovery_genes, discovery_variants.filter(key__isnull=True))
tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate(
keys=ArrayAgg('key', distinct=True),
- tags=ArrayAgg(JSONObject(key='key', family_guid='family__guid', selectedMainTranscriptId='selected_main_transcript_id')),
+ variants=ArrayAgg(JSONObject(key='key', family_guid='family_guid', selected_main_transcript_id='selected_main_transcript_id')),
)
- for dataset_type, keys, tags in tags_by_dataset_type.values_list('dataset_type', 'keys', 'tags'):
+ for dataset_type, keys, variants in tags_by_dataset_type.values_list('dataset_type', 'keys', 'variants'):
if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
transcripts_by_key = get_transcripts_by_key(genome_version, keys)
else:
qs = get_annotations_queryset(genome_version, dataset_type, keys)
transcripts_by_key = dict(qs.values_list('key', qs.transcript_field))
- for tag in tags:
- key = tag.pop('key')
- tag['transcripts'] = transcripts_by_key[key]
- tag['mainTranscriptId'] = next((
- t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts
- if t.get('transcriptRank') == 0
- ), None)
- discovery_tags.append(tag)
-
- return discovery_tags
+ for v in variants:
+ gene_id = _get_transcripts_selected_gene(v['selected_main_transcript_id'], transcripts_by_key[v['key']])
+ family_discovery_genes.append((v['family_guid'], gene_id))
MME_TAG_NAME = 'MME Submission'
diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py
index 58c764391f..80462c1a7a 100644
--- a/seqr/views/utils/test_utils.py
+++ b/seqr/views/utils/test_utils.py
@@ -1,7 +1,6 @@
# Utilities used for unit and integration tests.
from collections import defaultdict
from copy import deepcopy
-from datetime import datetime
from django.contrib.auth.models import User, Group
from django.core.management import call_command
from django.db import connections, transaction
@@ -12,7 +11,6 @@
import logging
import mock
import re
-import requests
import responses
from urllib.parse import quote_plus, urlparse
@@ -47,8 +45,6 @@ class AuthenticationTestCase(TestCase):
no_policy_user = None
def setUp(self):
- patcher = mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', self.CLICKHOUSE_HOSTNAME)
- patcher.start()
patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME)
patcher.start()
self.addCleanup(patcher.stop)
diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py
index 1371b142de..b60e415d8f 100644
--- a/seqr/views/utils/variant_utils.py
+++ b/seqr/views/utils/variant_utils.py
@@ -16,7 +16,7 @@
from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \
RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType
from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids
-from seqr.utils.search.utils import backend_specific_call, parse_variant_id
+from seqr.utils.search.utils import backend_specific_call, variant_dataset_type
from seqr.utils.gene_utils import get_genes_for_variants
from seqr.utils.middleware import ErrorsWarningsException
from seqr.utils.xpos_utils import get_xpos
@@ -94,10 +94,14 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,):
update_json = {
'key': variant_json['key'],
'genotypes': variant_json.get('genotypes', {}),
- 'dataset_type': _dataset_type(variant_id, variant_json),
+ 'dataset_type': variant_dataset_type({'variantId': variant_id, **variant_json}),
}
else:
update_json = {'saved_variant_json': variant_json}
+ if 'transcripts' in variant_json:
+ update_json['gene_ids'] = sorted(variant_json['transcripts'].keys(), key=lambda gene_id: _transcript_sort(gene_id, variant_json))
+ elif 'gene_ids' in variant_json:
+ update_json['gene_ids'] = variant_json['gene_ids']
return {
'xpos': xpos,
'xpos_end': xpos + var_length,
@@ -108,11 +112,11 @@ def parse_saved_variant_json(variant_json, family_id, variant_id=None,):
}, update_json
-def _dataset_type(variant_id, variant):
- if not parse_variant_id(variant_id):
- sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES
- return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}'
- return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS
+def _transcript_sort(gene_id, saved_variant_json):
+ gene_transcripts = saved_variant_json['transcripts'][gene_id]
+ main_transcript_id = saved_variant_json.get('mainTranscriptId')
+ is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts)
+ return (not is_main_gene, min(t.get('transcriptRank', 100) for t in gene_transcripts) if gene_transcripts else 100)
def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, user, project=None, load_new_variant_data=False):
@@ -147,15 +151,11 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use
).annotate(saved_variant_ids=ArrayAgg('saved_variants__id', ordering='id'))
}
- variant_genes_by_id = backend_specific_call(
- _get_saved_variant_genes, _get_clickhouse_saved_variant_genes,
- )(saved_variant_map.values(), genome_version)
-
update_tags = []
num_new = 0
for key, variant in family_variant_data.items():
updated_tag = _set_updated_tags(
- key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user, variant_genes_by_id,
+ key, get_metadata(variant), variant['support_vars'], saved_variant_map, existing_tags, tag_type, user,
)
if updated_tag:
update_tags.append(updated_tag)
@@ -169,7 +169,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use
def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_var_ids: list[str],
saved_variant_map: dict[tuple[int, str], SavedVariant], existing_tags: dict[tuple[int, ...], VariantTag],
- tag_type: VariantTagType, user: User, variant_genes_by_id: dict[str, list[str]]):
+ tag_type: VariantTagType, user: User):
variant = saved_variant_map[key]
existing_tag = existing_tags.get(tuple([variant.id]))
updated_tag = None
@@ -187,11 +187,11 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v
VariantTag, {'variant_tag_type': tag_type, 'metadata': json.dumps(metadata)}, user)
tag.saved_variants.add(variant)
- variant_genes = variant_genes_by_id.get(key[1], set())
+ variant_genes = set(variant.gene_ids or [])
support_vars = []
for support_id in support_var_ids:
support_v = saved_variant_map[(key[0], support_id)]
- if variant_genes.intersection(variant_genes_by_id.get(support_id, set())):
+ if variant_genes.intersection(set(support_v.gene_ids)):
support_vars.append(support_v)
if support_vars:
variants = [variant] + support_vars
@@ -204,27 +204,6 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], support_v
return updated_tag
-def _get_saved_variant_genes(variant_models, *args, **kwargs):
- return {v.variant_id: set(v.saved_variant_json['transcripts'].keys()) for v in variant_models}
-
-
-def _get_clickhouse_saved_variant_genes(variant_models, genome_version):
- variant_genes_by_id = {}
- key_id_map = {}
- for v in variant_models:
- if v.key:
- key_id_map[v.key] = v.variant_id
- else:
- variant_genes_by_id[v.variant_id] = set(v.saved_variant_json['transcripts'].keys())
- qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, key_id_map.keys())
- variant_genes_by_id.update({
- key_id_map[key]: set(gene_ids) for key, gene_ids in qs.values_list(
- 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())),
- )
- })
- return variant_genes_by_id
-
-
def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: User, genome_version: str) -> dict[tuple[int, str], dict]:
family_ids = set()
variant_families = defaultdict(list)
@@ -276,22 +255,33 @@ def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int,
genotype_keys = get_clickhouse_genotypes(
project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples,
)
+ gene_ids_by_key = _get_gene_ids_by_key(genome_version, variant_key_map.keys())
for key, genotypes in genotype_keys.items():
variant_id = variant_key_map[key]
chrom, pos, ref, alt = variant_id.split('-')
variants.append({
'key': key, 'variantId': variant_id, 'chrom': chrom, 'pos': int(pos), 'ref': ref, 'alt': alt,
'genotypes': genotypes, 'familyGuids': sorted({g['familyGuid'] for g in genotypes.values()}),
+ 'gene_ids': gene_ids_by_key.get(key),
})
return variants
+def _get_gene_ids_by_key(genome_version, keys):
+ qs = get_annotations_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys)
+ return dict(qs.values_list(
+ 'key', ArrayDistinct(ArrayMap(qs.transcript_field, mapped_expression='x.geneId'), output_field=ArrayField(StringField())),
+ ))
+
+
def _get_clickhouse_variant_keys(variant_data: dict[tuple[int, str], dict], genome_version: str) -> dict[tuple[int, str], dict]:
variant_ids = {key[1] for key in variant_data}
variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, list(variant_ids))
+ gene_ids_by_key = _get_gene_ids_by_key(genome_version, variant_key_map.values())
for (_, variant_id), variant in variant_data.items():
if variant_id in variant_key_map:
variant['key'] = variant_key_map[variant_id]
+ variant['gene_ids'] = gene_ids_by_key.get(variant['key'])
return variant_data
diff --git a/ui/pages/DataManagement/DataManagement.jsx b/ui/pages/DataManagement/DataManagement.jsx
index 600e9044ac..df0828186f 100644
--- a/ui/pages/DataManagement/DataManagement.jsx
+++ b/ui/pages/DataManagement/DataManagement.jsx
@@ -44,13 +44,9 @@ const ES_DATA_MANAGEMENT_PAGES = [
...DATA_MANAGEMENT_PAGES,
]
-const LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [
+const CLICKHOUSE_DATA_MANAGEMENT_PAGES = [
...DATA_MANAGEMENT_PAGES,
{ path: 'pipeline_status', component: () => },
-]
-
-const AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [
- ...DATA_MANAGEMENT_PAGES,
...TRIGGER_SEARCH_DATA_UPDATE_PAGES,
]
@@ -58,10 +54,7 @@ const dataManagementPages = (user, elasticsearchEnabled) => {
if (!user.isDataManager) {
return PM_DATA_MANAGEMENT_PAGES
}
- if (elasticsearchEnabled) {
- return ES_DATA_MANAGEMENT_PAGES
- }
- return user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES
+ return elasticsearchEnabled ? ES_DATA_MANAGEMENT_PAGES : CLICKHOUSE_DATA_MANAGEMENT_PAGES
}
const DataManagement = ({ match, user, pages }) => (
diff --git a/ui/pages/DataManagement/components/LoadData.jsx b/ui/pages/DataManagement/components/LoadData.jsx
index 2048f9b986..5cbb9ead40 100644
--- a/ui/pages/DataManagement/components/LoadData.jsx
+++ b/ui/pages/DataManagement/components/LoadData.jsx
@@ -54,12 +54,6 @@ const FILE_PATH_FIELD = {
}
const CALLSET_PAGE_FIELDS = [
- {
- name: 'skipValidation',
- label: 'Skip Callset Validation',
- component: InlineToggle,
- asFormInput: true,
- },
{
...GENOME_VERSION_FIELD,
component: ButtonRadioGroup,
diff --git a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx
index 6a0e6fdfaa..9bc97d1237 100644
--- a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx
+++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx
@@ -46,25 +46,25 @@ const FAMILY_FIELDS = [
DATASET_TYPE_FIELD,
]
-const TriggerSearchDataUpdateForm = ({ path, fields }) => (
+const TriggerSearchDataUpdateForm = ({ entity, fields }) => (
)
TriggerSearchDataUpdateForm.propTypes = {
- path: PropTypes.string,
+ entity: PropTypes.string,
fields: PropTypes.arrayOf(PropTypes.object),
}
const TriggerDeleteProjects = () => (
-
+
)
const TriggerDeleteFamilies = () => (
-
+
)
export default [
diff --git a/ui/pages/Project/components/FamilyPage.jsx b/ui/pages/Project/components/FamilyPage.jsx
index 71c584612d..76a35f7757 100644
--- a/ui/pages/Project/components/FamilyPage.jsx
+++ b/ui/pages/Project/components/FamilyPage.jsx
@@ -12,7 +12,7 @@ import {
getGenesById,
getHasActiveSearchSampleByFamily,
} from 'redux/selectors'
-import { FAMILY_DETAIL_FIELDS, getVariantMainGeneId } from 'shared/utils/constants'
+import { FAMILY_DETAIL_FIELDS } from 'shared/utils/constants'
import { Error404 } from 'shared/components/page/Errors'
import Family from 'shared/components/panel/family/Family'
import FamilyReads from 'shared/components/panel/family/FamilyReads'
@@ -40,8 +40,8 @@ SearchLink.propTypes = {
}
const DiscoveryGenes = React.memo(({ family, genesById }) => {
- const discoveryGenes = (family.discoveryTags || []).map(
- tag => (genesById[getVariantMainGeneId(tag)] || {}).geneSymbol,
+ const discoveryGenes = (family.discoveryGeneIds || []).map(
+ geneId => (genesById[geneId] || {}).geneSymbol,
).filter(val => val)
return discoveryGenes.length > 0 ? (
@@ -59,7 +59,7 @@ DiscoveryGenes.propTypes = {
const BaseVariantDetail = (
{ project, family, hasActiveVariantSample, compact, genesById, tagTypeCounts, load, loading },
) => (
-
+
(dispatch, getState) => {
export const loadProjectLocusLists = () => loadCurrentProjectChildEntities('locus lists', REQUEST_LOCUS_LISTS, RECEIVE_LOCUS_LISTS)
export const loadFamilyVariantSummary = familyGuid => loadFamilyData(
- familyGuid, 'discoveryTags', 'variant_tag_summary', REQUEST_FAMILY_VARIANT_SUMMARY,
+ familyGuid, 'discoveryGeneIds', 'variant_tag_summary', REQUEST_FAMILY_VARIANT_SUMMARY,
)
export const loadSavedVariants = ({ familyGuids, variantGuid, tag }) => (dispatch, getState) => {
diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js
index 84ff29ea22..c849ee0526 100644
--- a/ui/pages/Project/selectors.js
+++ b/ui/pages/Project/selectors.js
@@ -212,7 +212,7 @@ export const getTaggedVariantsByFamily = createSelector(
getGenesById,
getVariantTagsByGuid,
(savedVariants, genesById, variantTagsByGuid) => Object.values(savedVariants).filter(
- variant => variant.tagGuids.length,
+ variant => variant.tagGuids?.length,
).reduce((acc, variant) => {
const { familyGuids, ...variantDetail } = variant
variantDetail.tags = variant.tagGuids.map(tagGuid => variantTagsByGuid[tagGuid])
@@ -299,7 +299,7 @@ export const getIndividualTaggedVariants = createSelector(
const { familyGuid } = individualsByGuid[individualGuid]
return Object.values(taggedVariants[familyGuid] || []).reduce((acc, variant) => {
const variantDetail = {
- ...variant.genotypes[individualGuid],
+ ...(variant.genotypes || {})[individualGuid],
...variant,
}
return [...acc, ...variant.genes.filter(gene => gene).map(gene => ({
diff --git a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx
index 9493a7936e..bc349932d3 100644
--- a/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx
+++ b/ui/pages/Search/components/VariantSearchFormPanelConfigs.jsx
@@ -1,4 +1,5 @@
import React from 'react'
+import { FormSpy } from 'react-final-form'
import styled from 'styled-components'
import { Form, Grid, Header, List, Loader, Table } from 'semantic-ui-react'
@@ -529,6 +530,16 @@ export const QUALITY_PANEL = {
fieldProps: { control: LazyLabeledSlider, format: val => val || null },
}
+const SUBSCRIPTION = { values: true }
+
+const ExcludeSearchToggle = props => (
+
+ {({ values }) => (
+
+ )}
+
+)
+
const ES_EXCLUDE_FIELDS = [
{
...BASE_LOCUS_FIELD,
@@ -537,6 +548,14 @@ const ES_EXCLUDE_FIELDS = [
},
]
const EXCLUDE_FIELDS = [
+ {
+ name: 'previousSearch',
+ label: 'Exclude Previous Search Results',
+ labelHelp: 'Exclude any results returned by the current search from the next search results',
+ color: 'grey',
+ component: ExcludeSearchToggle,
+ width: 8,
+ },
{
...CLINVAR_FIELD,
...PATHOGENICITY_FIELD_PROPS,
diff --git a/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx b/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx
index b82e317c49..16fbac3414 100644
--- a/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx
+++ b/ui/pages/Search/components/filters/CustomInheritanceFilter.jsx
@@ -2,12 +2,13 @@ import React from 'react'
import PropTypes from 'prop-types'
import { connect } from 'react-redux'
import { FormSpy } from 'react-final-form'
-import { Table, Header, Popup, Loader } from 'semantic-ui-react'
+import { Table, Header, Popup, Loader, Divider } from 'semantic-ui-react'
import { loadFamilyDetails } from 'redux/rootReducer'
-import { getFamiliesByGuid, getIndividualsByGuid, getFamilyDetailsLoading } from 'redux/selectors'
+import { getFamiliesByGuid, getIndividualsByGuid, getFamilyDetailsLoading, getElasticsearchEnabled } from 'redux/selectors'
import DataLoader from 'shared/components/DataLoader'
-import { Select } from 'shared/components/form/Inputs'
+import { helpLabel } from 'shared/components/form/FormHelpers'
+import { Select, InlineToggle } from 'shared/components/form/Inputs'
import PedigreeIcon from 'shared/components/icons/PedigreeIcon'
import PedigreeImagePanel from 'shared/components/panel/view-pedigree-image/PedigreeImagePanel'
import { AFFECTED, UNAFFECTED, AFFECTED_OPTIONS } from 'shared/utils/constants'
@@ -103,14 +104,45 @@ CustomInheritanceFilterContent.propTypes = {
individualsByGuid: PropTypes.object,
}
-const CustomInheritanceFilter = React.memo(({ load, loading, family, ...props }) => {
- if (!family) {
- return
- }
- return (
+const NO_CALL_FIELD = 'allowNoCall'
+
+const handleChange = (onChange, value) => update => onChange({ ...value, [NO_CALL_FIELD]: update })
+
+const NoCallToggle = ({ name, value, onChange, ...props }) => (
+
+)
+
+NoCallToggle.propTypes = {
+ name: PropTypes.string.isRequired,
+ value: PropTypes.object,
+ onChange: PropTypes.func.isRequired,
+}
+
+const CustomInheritanceFilter = React.memo(({ load, loading, family, esEnabled, individualsByGuid, ...props }) => {
+ const content = family ? (
-
+
+ ) :
+ if (esEnabled) {
+ return content
+ }
+ return (
+
)
})
@@ -118,6 +150,7 @@ const mapStateToProps = (state, ownProps) => ({
family: getFamiliesByGuid(state)[ownProps.familyGuid],
individualsByGuid: getIndividualsByGuid(state),
loading: !!getFamilyDetailsLoading(state)[ownProps.familyGuid],
+ esEnabled: getElasticsearchEnabled(state),
})
const mapDispatchToProps = {
@@ -127,7 +160,9 @@ const mapDispatchToProps = {
CustomInheritanceFilter.propTypes = {
load: PropTypes.func,
family: PropTypes.object,
+ individualsByGuid: PropTypes.object,
loading: PropTypes.bool,
+ esEnabled: PropTypes.bool,
}
const ConnectedCustomInheritanceFilter = connect(mapStateToProps, mapDispatchToProps)(CustomInheritanceFilter)
diff --git a/ui/pages/Search/constants.js b/ui/pages/Search/constants.js
index de484732e3..511f52ce36 100644
--- a/ui/pages/Search/constants.js
+++ b/ui/pages/Search/constants.js
@@ -135,6 +135,7 @@ export const INHERITANCE_FILTER_JSON_OPTIONS = INHERITANCE_FILTER_OPTIONS.map(
const CLINVAR_NAME = 'clinvar'
const CLIVAR_PATH = 'pathogenic'
const CLINVAR_LIKELY_PATH = 'likely_pathogenic'
+const CLINVAR_CONFLICTING_P_LP = 'conflicting_p_lp'
const CLINVAR_UNCERTAIN = 'vus_or_conflicting'
const CLINVAR_OPTIONS = [
{
@@ -145,6 +146,11 @@ const CLINVAR_OPTIONS = [
text: 'Likely Pathogenic (LP)',
value: CLINVAR_LIKELY_PATH,
},
+ {
+ description: 'Clinvar variant of with conflicting interpretations, at least one of which is Pathogenic (P) or Likely Pathogenic (LP)',
+ text: 'Conflicting with P/LP',
+ value: CLINVAR_CONFLICTING_P_LP,
+ },
{
description: 'Clinvar variant of uncertain significance or variant with conflicting interpretations',
text: 'VUS or Conflicting',
@@ -214,14 +220,14 @@ export const HGMD_PATHOGENICITY_FILTER_OPTIONS = [
{
text: 'Pathogenic/ Likely Path.',
value: {
- [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH],
+ [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_P_LP],
[HGMD_NAME]: [HGMD_DM],
},
},
{
text: 'Not Benign',
value: {
- [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_UNCERTAIN],
+ [CLINVAR_NAME]: [CLIVAR_PATH, CLINVAR_LIKELY_PATH, CLINVAR_CONFLICTING_P_LP, CLINVAR_UNCERTAIN],
[HGMD_NAME]: HGMD_OPTIONS.map(({ value }) => value),
},
},
diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js
index 52c59c9183..14278ecabb 100644
--- a/ui/pages/Search/selectors.js
+++ b/ui/pages/Search/selectors.js
@@ -130,14 +130,15 @@ const getIntitialProjectFamilies = createProjectFamiliesSelector(
)
export const getIntitialSearch = createSelector(
+ getCurrentSearchHash,
getCurrentSearchParams,
getIntitialProjectFamilies,
- (searchParams, projectFamilies) => {
+ (previousSearchHash, searchParams, projectFamilies) => {
if (searchParams) {
- return searchParams
+ return { ...searchParams, previousSearchHash }
}
- return projectFamilies ? { projectFamilies: [projectFamilies] } : null
+ return projectFamilies ? { projectFamilies: [projectFamilies], previousSearchHash } : null
},
)
diff --git a/ui/pages/Search/selectors.test.js b/ui/pages/Search/selectors.test.js
index e261f012d2..78b77f1bc7 100644
--- a/ui/pages/Search/selectors.test.js
+++ b/ui/pages/Search/selectors.test.js
@@ -20,7 +20,7 @@ test('getIntitialSearch', () => {
expect(getIntitialSearch(NO_SEARCH_STATE, { match: { params: {} } })).toEqual(null)
- expect(getIntitialSearch(STATE, { match: { params: { searchHash: SEARCH_HASH } } })).toEqual(SEARCH)
+ expect(getIntitialSearch(STATE, { match: { params: { searchHash: SEARCH_HASH } } })).toEqual({ ...SEARCH, previousSearchHash: SEARCH_HASH })
expect(getIntitialSearch(
NO_SEARCH_STATE, { match: { params: { projectGuid: PROJECT_GUID } } })
diff --git a/ui/shared/components/panel/family/Family.jsx b/ui/shared/components/panel/family/Family.jsx
index 659a8f5739..eba08fa0b8 100644
--- a/ui/shared/components/panel/family/Family.jsx
+++ b/ui/shared/components/panel/family/Family.jsx
@@ -129,7 +129,7 @@ const FAMILY_FIELD_RENDER_LOOKUP = {
[FAMILY_FIELD_DISCOVERY_MONDO_ID]: {
...MONDO_FIELD,
internal: true,
- canEditFamily: ({ discoveryTags }) => discoveryTags?.length > 0,
+ canEditFamily: ({ discoveryGeneIds }) => discoveryGeneIds?.length > 0,
},
[FAMILY_FIELD_OMIM_NUMBERS]: {
canEditFamily: ({ postDiscoveryOmimOptions }) => Object.keys(postDiscoveryOmimOptions || {}).length > 0,
diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js
index 74991660a0..4195f8e4b7 100644
--- a/ui/shared/utils/constants.js
+++ b/ui/shared/utils/constants.js
@@ -1505,7 +1505,7 @@ const REVERSE_PRED_COLOR_MAP = [...PRED_COLOR_MAP].reverse()
export const ORDERED_PREDICTOR_FIELDS = [
{ field: 'cadd', group: CODING_IN_SILICO_GROUP, thresholds: [0.151, 22.8, 25.3, 28.1, undefined], min: 1, max: 99, fieldTitle: 'CADD', requiresCitation: true },
{ field: 'revel', group: MISSENSE_IN_SILICO_GROUP, thresholds: [0.0161, 0.291, 0.644, 0.773, 0.932], fieldTitle: 'REVEL', requiresCitation: true },
- { field: 'alphamissense', fieldTitle: 'AlphaMissense', displayOnly: true },
+ { field: 'alphamissense', fieldTitle: 'AlphaMissense', group: MISSENSE_IN_SILICO_GROUP, thresholds: [0.34, 0.34, 0.564, 0.564] },
{ field: 'vest', thresholds: [undefined, 0.45, 0.764, 0.861, 0.965], fieldTitle: 'VEST', requiresCitation: true },
{ field: 'mpc', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, undefined, 1.36, 1.828, undefined], max: 5, fieldTitle: 'MPC' },
{