Skip to content

Dev #4982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hanars merged 13 commits into from
Sep 3, 2025
Merged

Dev #4982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d1dfe49
squash clickhouse migrations
hanars Aug 28, 2025
e4bdf89
empty old migrations
hanars Aug 28, 2025
4fd3c4c
remove old migrations
hanars Aug 28, 2025
4294c00
move splice ai up in predicotr order
hanars Aug 28, 2025
f838ce6
Merge pull request #4970 from broadinstitute/in-silico-order
hanars Aug 28, 2025
c919076
af types (#4973)
bpblanken Aug 29, 2025
794d05f
Merge pull request #4969 from broadinstitute/squash-clickhouse-migrat…
hanars Aug 29, 2025
0da720b
change genotype map to only list samples once
hanars Aug 29, 2025
c8a1f0a
Merge pull request #4976 from broadinstitute/all-project-memory-reduc…
hanars Sep 2, 2025
781e230
handle null sv "genes"
hanars Sep 2, 2025
2847887
Merge pull request #4978 from broadinstitute/sv-gene-search-null-gene
hanars Sep 2, 2025
7e9ba3b
add efault values for required metadata fields
hanars Sep 2, 2025
1df74c1
Merge pull request #4979 from broadinstitute/gregor-defaults
hanars Sep 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions clickhouse_search/fixtures/clickhouse_search.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"populations": [
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[927, 0.03445, 26912, 0.04028, 0, 48],
[927, 0.03444932, 26912, 0.04027665, 0, 48],
[20757, 0.07842, 264690, 20757, 0]
],
"sorted_transcript_consequences": [],
Expand All @@ -46,10 +46,10 @@
"screen_region_type": null,
"predictions": [20.9, 2.001, 0, 5.86851, 0.28205, null, 3, 0.1, 0.46558, 0.197, 0.05, 0, 4, 0.211],
"populations": [
[35805, 0.295, 121372, 0.4153, 0, 24061, 5872],
[72672, 0.289, 251462, 0.41165, 0, 11567],
[35805, 0.29499999, 121372, 0.41530353, 0, 24061, 5872],
[72672, 0.28899795, 251462, 0.4116475, 0, 11567],
[0, 0, 0, 0, 0, 0],
[65154, 0.24615, 264690, 47604, 8775]
[65154, 0.246152, 264690, 47604, 8775]
],
"sorted_transcript_consequences": [
[0.99779, 1, ["missense_variant"], false, null, "ENSG00000177000"],
Expand Down Expand Up @@ -84,7 +84,7 @@
"populations": [
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[57757, 0.38041, 151828, 0.47978, 0, 12204],
[57757, 0.38041073, 151828, 0.47977865, 0, 12204],
[95998, 0.362681, 264690, 57260, 19369]
],
"sorted_transcript_consequences": [
Expand Down Expand Up @@ -119,7 +119,7 @@
"populations": [
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[39, 0.00027, 147062, 0.0015, 0, 0],
[39, 0.00026519, 147062, 0.00150301, 0, 0],
[0, 0, 0, 0, 0]
],
"sorted_transcript_consequences": [
Expand Down Expand Up @@ -156,7 +156,7 @@
"populations": [
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[2, 0.00012, 16090, 0.00233, 0, 0],
[2, 0.0001243, 16090, 0.00233, 0, 0],
[0, 0, 0, 0, 0]
],
"sorted_transcript_consequences": [],
Expand Down Expand Up @@ -654,10 +654,10 @@
"hgmd": [null, null],
"predictions": [13.02, 3.951, null, null, null, null, null, 0.49064, null, null, 0, 4,null],
"populations": [
[66593, 0.63, 104352, 0.81988, 0, 22269, 22162],
[137532, 0.63542, 216442, 0.82261, 0, 45869],
[14649, 0.61365, 23872, 0.82844, 0, 4584],
[65461, 0.52132, 125568, 33149, 16156]
[66593, 0.63, 104352, 0.81987739, 0, 22269, 22162],
[137532, 0.63542193, 216442, 0.82261163, 0, 45869],
[14649, 0.61364776, 23872, 0.82843894, 0, 4584],
[65461, 0.52131897, 125568, 33149, 16156]
],
"sorted_transcript_consequences": [
[1, ["missense_variant"], "ENSG00000271079"],
Expand Down Expand Up @@ -729,7 +729,7 @@
[0, 0, 56419],
[0, 0, 56419, 0],
[0, 0, 195983],
[3, 0.00002, 195983, 0.20635]
[3, 0.00001531, 195983, 0.20635]
],
"sorted_transcript_consequences": [
[
Expand Down Expand Up @@ -782,10 +782,10 @@
"mitomap_pathogenic": true,
"predictions": [null, true, null, null, null, null, 0.7514],
"populations": [
[3118, 0.05535, 56336],
[3, 0.00005, 56336, 1],
[9573, 0.04885, 195983],
[18, 0.00009, 195983, 0.96269]
[3118, 0.05534649, 56336],
[3, 0.00005325, 56336, 1],
[9573, 0.04884607, 195983],
[18, 0.00009184, 195983, 0.96269]
],
"sorted_transcript_consequences": [
[
Expand Down Expand Up @@ -1069,7 +1069,7 @@
"num_exon": 4,
"predictions": [0.181],
"populations": [
[1763, 0.07649, 23048, 0, 0]
[1763, 0.07649254, 23048, 0, 0]
],
"sorted_gene_consequences": [
["ENSG00000129562", "COPY_GAIN"]
Expand All @@ -1092,7 +1092,7 @@
"num_exon": 26,
"predictions": [0.548],
"populations": [
[284, 0.01232, 23047, 0, 0]
[284, 0.01232211, 23047, 0, 0]
],
"sorted_gene_consequences": [
["ENSG00000013364", "LOF"],
Expand Down Expand Up @@ -1121,7 +1121,7 @@
"num_exon": 3,
"predictions": [0.786],
"populations": [
[35, 0.00152, 23048, 0, 0]
[35, 0.00151857, 23048, 0, 0]
],
"sorted_gene_consequences": [
["ENSG00000275023", "LOF"]
Expand All @@ -1144,7 +1144,7 @@
"num_exon": 8,
"predictions": [0.71],
"populations": [
[115, 0.00499, 23048, 0, 0]
[115, 0.00498959, 23048, 0, 0]
],
"sorted_gene_consequences": [
["ENSG00000275023", "LOF"],
Expand Down
22 changes: 16 additions & 6 deletions clickhouse_search/managers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,7 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit
return self._annotate_calls(entries, sample_data, annotate_hom_alts, skip_individual_guid, multi_sample_type_families)

def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, inheritance_mode, individual_genotype_filter, quality_filter, clinvar_override_q, annotate_carriers, custom_affected):
samples_by_gt = defaultdict(list)
samples_by_genotype = defaultdict(list)
affected_samples = []
unaffected_samples = []
family_missing_type_samples = defaultdict(lambda: defaultdict(list))
Expand All @@ -815,8 +815,7 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i
if affected == UNAFFECTED and annotate_carriers:
unaffected_samples.append(sample['sample_id'])
if (inheritance_mode and inheritance_mode != ANY_AFFECTED) or individual_genotype_filter:
for gt in self.genotype_lookup[genotype]:
samples_by_gt[gt].append(sample['sample_id'])
samples_by_genotype[genotype].append(sample['sample_id'])
if sample['family_guid'] in multi_sample_type_families:
sample_type = sample['sample_type']
missing_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS
Expand All @@ -830,9 +829,20 @@ def _get_inheritance_quality_qs(self, sample_data, multi_sample_type_families, i
'gt': (self.genotype_lookup[HAS_ALT], 'has({value}, {field})'),
'sampleId': (affected_samples, 'has({value}, {field})'),
})
elif samples_by_gt:
gt_filter_map = ', '.join([f"{gt}, {samples_by_gt[gt]}" for gt in [-1, 0, 1, 2]])
gt_filter = (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')
elif samples_by_genotype:
if all(len(self.genotype_lookup[genotype]) == 1 for genotype in samples_by_genotype.keys()):
samples_by_gt = {self.genotype_lookup[genotype][0]: samples for genotype, samples in samples_by_genotype.items()}
gt_filter_map = ', '.join([f"{gt}, {samples_by_gt.get(gt, [])}" for gt in [-1, 0, 1, 2]])
gt_filter = (gt_filter_map, 'has(map({value})[ifNull({field}, -1)], x.sampleId)')
else:
genotype_sample_map = ', '.join([f"'{genotype or 'any'}', {samples}" for genotype, samples in samples_by_genotype.items()])
gt_genotypes = defaultdict(list)
for genotype in samples_by_genotype.keys():
for gt in self.genotype_lookup[genotype]:
gt_genotypes[gt].append(genotype or 'any')
gt_genotype_map = ', '.join([f"{gt}, {gt_genotypes[gt]}" for gt in [-1, 0, 1, 2]])
genotype_maps = f'genotype -> map({genotype_sample_map})[genotype], map({gt_genotype_map})'
gt_filter = (genotype_maps, 'has(arrayFlatten(arrayMap({value}[ifNull({field}, -1)])), x.sampleId)')
inheritance_q = Q(calls__array_all={'gt': gt_filter})

quality_q = self._quality_q(quality_filter, affected_samples, clinvar_override_q)
Expand Down
Loading