From 18137048253f01020bb1adaafec0dcdfe13cf12e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 10:32:42 -0400 Subject: [PATCH 01/13] run unit tests on updated python version --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index cc339c0049..59043e6cf8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9] + python-version: [3.9, 3.11] services: # Label used to access the service container From 351f0788593a3265ad57648dc51e08342678167c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 13:59:52 -0400 Subject: [PATCH 02/13] reimplement basic urlib activate --- seqr/utils/search/elasticsearch/es_utils_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 79015b80b9..a86f4f3d4a 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -39,6 +39,11 @@ def replace_json(self, url, *args, **kwargs): def call_request_json(self, index=-1): return json.loads(self.calls[index].request.body) + def activate(self, func): + def wrapper(*args, **kwargs): + with self: + return func(*args, **kwargs) + return wrapper urllib3_responses = Urllib3Responses() From 14e0b90bc03424a4ea9e00c56981601982f31e5d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 14:35:37 -0400 Subject: [PATCH 03/13] inline constants --- reference_data/models.py | 73 ++++++++++++++++++++-------- reference_data/utils/dbnsfp_utils.py | 24 --------- 2 files changed, 52 insertions(+), 45 deletions(-) delete mode 100644 reference_data/utils/dbnsfp_utils.py diff --git a/reference_data/models.py b/reference_data/models.py index 5c5397c55e..e7206f416d 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -9,7 +9,6 @@ import requests from tqdm import tqdm -from reference_data.utils.dbnsfp_utils import DBNSFP_FIELD_MAP, DBNSFP_EXCLUDE_FIELDS from reference_data.utils.download_utils import download_file from reference_data.utils.gencode_utils import parse_gencode_record, GENCODE_URL_TEMPLATE, GENCODE_FILE_HEADER from seqr.views.utils.export_utils import write_multiple_files @@ -370,8 +369,8 @@ def parse_record(cls, record, skipped_genes=None, **kwargs): record = None yield record - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return record @classmethod @@ -459,8 +458,8 @@ class GeneConstraint(GeneMetadataModel): class Meta: json_fields = ['mis_z', 'mis_z_rank', 'pLI', 'pLI_rank', 'louef', 'louef_rank'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_id': record['gene_id'].split(".")[0], 'gene_symbol': record['gene'], @@ -489,8 +488,8 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): class Meta: json_fields = ['pHI', 'pTS'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['#gene'], 'pHI': float(record['pHaplo']), @@ -508,8 +507,8 @@ class GeneShet(GeneMetadataModel): class Meta: json_fields = ['post_mean'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_id': record['ensg'], 'post_mean': float(record['post_mean']), @@ -656,6 +655,38 @@ class dbNSFPGene(GeneMetadataModel): CURRENT_VERSION = 'dbNSFP4.0_gene' URL = f'http://storage.googleapis.com/seqr-reference-data/dbnsfp/{CURRENT_VERSION}' + # based on dbNSFP_gene schema README: https://drive.google.com/file/d/0B60wROKy6OqcNGJ2STJlMTJONk0/view + FIELD_MAP = { + 'Ensembl_gene': "gene_id", + 'Pathway(Uniprot)': "pathway_uniprot", + 'Pathway(BioCarta)_short': "pathway_biocarta_short", + # Short name of the Pathway(s) the gene belongs to (from BioCarta) + 'Pathway(BioCarta)_full': "pathway_biocarta_full", + # Full name(s) of the Pathway(s) the gene belongs to (from BioCarta) + 'Pathway(ConsensusPathDB)': "pathway_consensus_path_db", + # Pathway(s) the gene belongs to (from ConsensusPathDB) + 'Pathway(KEGG)_id': "pathway_kegg_id", # ID(s) of the Pathway(s) the gene belongs to (from KEGG) + 'Pathway(KEGG)_full': "pathway_kegg_full", # Full name(s) of the Pathway(s) the gene belongs to (from KEGG) + 'Function_description': "function_desc", # Function description of the gene (from Uniprot) + 'Disease_description': "disease_desc", # Disease(s) the gene caused or associated with (from Uniprot) + 'Trait_association(GWAS)': "trait_association_gwas", # Trait(s) the gene associated with (from GWAS catalog) + 'Expression(egenetics)': "expression_egenetics", + # Tissues/organs the gene expressed in (egenetics data from BioMart) + 'Expression(GNF/Atlas)': "expression_gnf_atlas", + # Tissues/organs the gene expressed in (GNF/Atlas data from BioMart) + 'ZFIN_zebrafish_gene': "zebrafish_gene", # Homolog zebrafish gene name from ZFIN + 'ZFIN_zebrafish_structure': "zebrafish_structure", # Affected structure of the homolog zebrafish gene from ZFIN + 'ZFIN_zebrafish_phenotype_quality': "zebrafish_phenotype_quality", + # Phenotype description for the homolog zebrafish gene from ZFIN + 'ZFIN_zebrafish_phenotype_tag': "zebrafish_phenotype_tag", + # Phenotype tag for the homolog zebrafish gene from ZFIN + } + + EXCLUDE_FIELDS = ( + 'Gene', 'P(', 'RVIS_percentile_ExAC', 'Known_rec_info', 'GDI', 'LoF', 'ExAC', 'Interactions', 'Orphanet', + 'gnomAD','SORVA_LOF', 'Essential_gene', 'chr', 'MIM', 'OMIM', 'RVIS_percentile_EVS', 'RVIS_EVS', 'HIPred', + ) + gene_names = models.TextField(blank=True) function_desc = models.TextField(null=True, blank=True) @@ -694,10 +725,10 @@ class dbNSFPGene(GeneMetadataModel): class Meta: json_fields = ['function_desc', 'disease_desc', 'gene_names'] - @staticmethod - def parse_gene_record(record): - parsed_record = {DBNSFP_FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') - for k, v in record.items() if not k.startswith(DBNSFP_EXCLUDE_FIELDS)} + @classmethod + def parse_gene_record(cls, record): + parsed_record = {cls.FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') + for k, v in record.items() if not k.startswith(cls.EXCLUDE_FIELDS)} parsed_record["function_desc"] = parsed_record["function_desc"].replace("FUNCTION: ", "") parsed_record['gene_id'] = parsed_record['gene_id'].split(';')[0] if not parsed_record['gene_id']: @@ -722,8 +753,8 @@ class PrimateAI(GeneMetadataModel): class Meta: json_fields = ['percentile_25', 'percentile_75'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['genesymbol'], 'percentile_25': float(record['pcnt25']), @@ -746,8 +777,8 @@ class Meta: def get_file_header(f): return ['gene_symbol', 'entrez_gene_id', 'mouse_gene_symbol', 'marker_id', 'phenotype_ids'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return {k: v.strip() for k, v in record.items() if k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} @classmethod @@ -791,8 +822,8 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['gene_symbol'], 'hgnc_id': record['gene_curie'], @@ -844,8 +875,8 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['gene_symbol'], 'haploinsufficiency': record['haploinsufficiency'].replace(' for Haploinsufficiency', ''), diff --git a/reference_data/utils/dbnsfp_utils.py b/reference_data/utils/dbnsfp_utils.py deleted file mode 100644 index dd32bb4d00..0000000000 --- a/reference_data/utils/dbnsfp_utils.py +++ /dev/null @@ -1,24 +0,0 @@ -# based on dbNSFP_gene schema README: https://drive.google.com/file/d/0B60wROKy6OqcNGJ2STJlMTJONk0/view -DBNSFP_FIELD_MAP = { - 'Ensembl_gene': "gene_id", - 'Pathway(Uniprot)': "pathway_uniprot", - 'Pathway(BioCarta)_short': "pathway_biocarta_short", # Short name of the Pathway(s) the gene belongs to (from BioCarta) - 'Pathway(BioCarta)_full': "pathway_biocarta_full", # Full name(s) of the Pathway(s) the gene belongs to (from BioCarta) - 'Pathway(ConsensusPathDB)': "pathway_consensus_path_db", # Pathway(s) the gene belongs to (from ConsensusPathDB) - 'Pathway(KEGG)_id': "pathway_kegg_id", # ID(s) of the Pathway(s) the gene belongs to (from KEGG) - 'Pathway(KEGG)_full': "pathway_kegg_full", # Full name(s) of the Pathway(s) the gene belongs to (from KEGG) - 'Function_description': "function_desc", # Function description of the gene (from Uniprot) - 'Disease_description': "disease_desc", # Disease(s) the gene caused or associated with (from Uniprot) - 'Trait_association(GWAS)': "trait_association_gwas", # Trait(s) the gene associated with (from GWAS catalog) - 'Expression(egenetics)': "expression_egenetics", # Tissues/organs the gene expressed in (egenetics data from BioMart) - 'Expression(GNF/Atlas)': "expression_gnf_atlas", # Tissues/organs the gene expressed in (GNF/Atlas data from BioMart) - 'ZFIN_zebrafish_gene': "zebrafish_gene", # Homolog zebrafish gene name from ZFIN - 'ZFIN_zebrafish_structure': "zebrafish_structure", # Affected structure of the homolog zebrafish gene from ZFIN - 'ZFIN_zebrafish_phenotype_quality': "zebrafish_phenotype_quality", # Phenotype description for the homolog zebrafish gene from ZFIN - 'ZFIN_zebrafish_phenotype_tag': "zebrafish_phenotype_tag", # Phenotype tag for the homolog zebrafish gene from ZFIN -} - -DBNSFP_EXCLUDE_FIELDS = ( - 'Gene', 'P(', 'RVIS_percentile_ExAC', 'Known_rec_info', 'GDI', 'LoF', 'ExAC', 'Interactions', 'Orphanet', 'gnomAD', - 'SORVA_LOF', 'Essential_gene', 'chr', 'MIM', 'OMIM', 'RVIS_percentile_EVS', 'RVIS_EVS', 'HIPred', -) From cab2eee22b552507a760f173869668974d1e98c5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 15:13:33 -0400 Subject: [PATCH 04/13] Revert "inline constants" This reverts commit 14e0b90bc03424a4ea9e00c56981601982f31e5d. --- reference_data/models.py | 73 ++++++++-------------------- reference_data/utils/dbnsfp_utils.py | 24 +++++++++ 2 files changed, 45 insertions(+), 52 deletions(-) create mode 100644 reference_data/utils/dbnsfp_utils.py diff --git a/reference_data/models.py b/reference_data/models.py index e7206f416d..5c5397c55e 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -9,6 +9,7 @@ import requests from tqdm import tqdm +from reference_data.utils.dbnsfp_utils import DBNSFP_FIELD_MAP, DBNSFP_EXCLUDE_FIELDS from reference_data.utils.download_utils import download_file from reference_data.utils.gencode_utils import parse_gencode_record, GENCODE_URL_TEMPLATE, GENCODE_FILE_HEADER from seqr.views.utils.export_utils import write_multiple_files @@ -369,8 +370,8 @@ def parse_record(cls, record, skipped_genes=None, **kwargs): record = None yield record - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return record @classmethod @@ -458,8 +459,8 @@ class GeneConstraint(GeneMetadataModel): class Meta: json_fields = ['mis_z', 'mis_z_rank', 'pLI', 'pLI_rank', 'louef', 'louef_rank'] - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_id': record['gene_id'].split(".")[0], 'gene_symbol': record['gene'], @@ -488,8 +489,8 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): class Meta: json_fields = ['pHI', 'pTS'] - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_symbol': record['#gene'], 'pHI': float(record['pHaplo']), @@ -507,8 +508,8 @@ class GeneShet(GeneMetadataModel): class Meta: json_fields = ['post_mean'] - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_id': record['ensg'], 'post_mean': float(record['post_mean']), @@ -655,38 +656,6 @@ class dbNSFPGene(GeneMetadataModel): CURRENT_VERSION = 'dbNSFP4.0_gene' URL = f'http://storage.googleapis.com/seqr-reference-data/dbnsfp/{CURRENT_VERSION}' - # based on dbNSFP_gene schema README: https://drive.google.com/file/d/0B60wROKy6OqcNGJ2STJlMTJONk0/view - FIELD_MAP = { - 'Ensembl_gene': "gene_id", - 'Pathway(Uniprot)': "pathway_uniprot", - 'Pathway(BioCarta)_short': "pathway_biocarta_short", - # Short name of the Pathway(s) the gene belongs to (from BioCarta) - 'Pathway(BioCarta)_full': "pathway_biocarta_full", - # Full name(s) of the Pathway(s) the gene belongs to (from BioCarta) - 'Pathway(ConsensusPathDB)': "pathway_consensus_path_db", - # Pathway(s) the gene belongs to (from ConsensusPathDB) - 'Pathway(KEGG)_id': "pathway_kegg_id", # ID(s) of the Pathway(s) the gene belongs to (from KEGG) - 'Pathway(KEGG)_full': "pathway_kegg_full", # Full name(s) of the Pathway(s) the gene belongs to (from KEGG) - 'Function_description': "function_desc", # Function description of the gene (from Uniprot) - 'Disease_description': "disease_desc", # Disease(s) the gene caused or associated with (from Uniprot) - 'Trait_association(GWAS)': "trait_association_gwas", # Trait(s) the gene associated with (from GWAS catalog) - 'Expression(egenetics)': "expression_egenetics", - # Tissues/organs the gene expressed in (egenetics data from BioMart) - 'Expression(GNF/Atlas)': "expression_gnf_atlas", - # Tissues/organs the gene expressed in (GNF/Atlas data from BioMart) - 'ZFIN_zebrafish_gene': "zebrafish_gene", # Homolog zebrafish gene name from ZFIN - 'ZFIN_zebrafish_structure': "zebrafish_structure", # Affected structure of the homolog zebrafish gene from ZFIN - 'ZFIN_zebrafish_phenotype_quality': "zebrafish_phenotype_quality", - # Phenotype description for the homolog zebrafish gene from ZFIN - 'ZFIN_zebrafish_phenotype_tag': "zebrafish_phenotype_tag", - # Phenotype tag for the homolog zebrafish gene from ZFIN - } - - EXCLUDE_FIELDS = ( - 'Gene', 'P(', 'RVIS_percentile_ExAC', 'Known_rec_info', 'GDI', 'LoF', 'ExAC', 'Interactions', 'Orphanet', - 'gnomAD','SORVA_LOF', 'Essential_gene', 'chr', 'MIM', 'OMIM', 'RVIS_percentile_EVS', 'RVIS_EVS', 'HIPred', - ) - gene_names = models.TextField(blank=True) function_desc = models.TextField(null=True, blank=True) @@ -725,10 +694,10 @@ class dbNSFPGene(GeneMetadataModel): class Meta: json_fields = ['function_desc', 'disease_desc', 'gene_names'] - @classmethod - def parse_gene_record(cls, record): - parsed_record = {cls.FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') - for k, v in record.items() if not k.startswith(cls.EXCLUDE_FIELDS)} + @staticmethod + def parse_gene_record(record): + parsed_record = {DBNSFP_FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') + for k, v in record.items() if not k.startswith(DBNSFP_EXCLUDE_FIELDS)} parsed_record["function_desc"] = parsed_record["function_desc"].replace("FUNCTION: ", "") parsed_record['gene_id'] = parsed_record['gene_id'].split(';')[0] if not parsed_record['gene_id']: @@ -753,8 +722,8 @@ class PrimateAI(GeneMetadataModel): class Meta: json_fields = ['percentile_25', 'percentile_75'] - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_symbol': record['genesymbol'], 'percentile_25': float(record['pcnt25']), @@ -777,8 +746,8 @@ class Meta: def get_file_header(f): return ['gene_symbol', 'entrez_gene_id', 'mouse_gene_symbol', 'marker_id', 'phenotype_ids'] - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return {k: v.strip() for k, v in record.items() if k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} @classmethod @@ -822,8 +791,8 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_symbol': record['gene_symbol'], 'hgnc_id': record['gene_curie'], @@ -875,8 +844,8 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @classmethod - def parse_gene_record(cls, record): + @staticmethod + def parse_gene_record(record): return { 'gene_symbol': record['gene_symbol'], 'haploinsufficiency': record['haploinsufficiency'].replace(' for Haploinsufficiency', ''), diff --git a/reference_data/utils/dbnsfp_utils.py b/reference_data/utils/dbnsfp_utils.py new file mode 100644 index 0000000000..dd32bb4d00 --- /dev/null +++ b/reference_data/utils/dbnsfp_utils.py @@ -0,0 +1,24 @@ +# based on dbNSFP_gene schema README: https://drive.google.com/file/d/0B60wROKy6OqcNGJ2STJlMTJONk0/view +DBNSFP_FIELD_MAP = { + 'Ensembl_gene': "gene_id", + 'Pathway(Uniprot)': "pathway_uniprot", + 'Pathway(BioCarta)_short': "pathway_biocarta_short", # Short name of the Pathway(s) the gene belongs to (from BioCarta) + 'Pathway(BioCarta)_full': "pathway_biocarta_full", # Full name(s) of the Pathway(s) the gene belongs to (from BioCarta) + 'Pathway(ConsensusPathDB)': "pathway_consensus_path_db", # Pathway(s) the gene belongs to (from ConsensusPathDB) + 'Pathway(KEGG)_id': "pathway_kegg_id", # ID(s) of the Pathway(s) the gene belongs to (from KEGG) + 'Pathway(KEGG)_full': "pathway_kegg_full", # Full name(s) of the Pathway(s) the gene belongs to (from KEGG) + 'Function_description': "function_desc", # Function description of the gene (from Uniprot) + 'Disease_description': "disease_desc", # Disease(s) the gene caused or associated with (from Uniprot) + 'Trait_association(GWAS)': "trait_association_gwas", # Trait(s) the gene associated with (from GWAS catalog) + 'Expression(egenetics)': "expression_egenetics", # Tissues/organs the gene expressed in (egenetics data from BioMart) + 'Expression(GNF/Atlas)': "expression_gnf_atlas", # Tissues/organs the gene expressed in (GNF/Atlas data from BioMart) + 'ZFIN_zebrafish_gene': "zebrafish_gene", # Homolog zebrafish gene name from ZFIN + 'ZFIN_zebrafish_structure': "zebrafish_structure", # Affected structure of the homolog zebrafish gene from ZFIN + 'ZFIN_zebrafish_phenotype_quality': "zebrafish_phenotype_quality", # Phenotype description for the homolog zebrafish gene from ZFIN + 'ZFIN_zebrafish_phenotype_tag': "zebrafish_phenotype_tag", # Phenotype tag for the homolog zebrafish gene from ZFIN +} + +DBNSFP_EXCLUDE_FIELDS = ( + 'Gene', 'P(', 'RVIS_percentile_ExAC', 'Known_rec_info', 'GDI', 'LoF', 'ExAC', 'Interactions', 'Orphanet', 'gnomAD', + 'SORVA_LOF', 'Essential_gene', 'chr', 'MIM', 'OMIM', 'RVIS_percentile_EVS', 'RVIS_EVS', 'HIPred', +) From 055b8cb0a687bf0c4755a486e38346af75d6f1e6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 15:24:49 -0400 Subject: [PATCH 05/13] update coverage version --- requirements-dev.in | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index f85a139ed5..b44a43aea8 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,5 @@ -c requirements.txt # use the generated reqs as a constraint -coverage<5.2 +coverage<6.6 django-compressor django-debug-toolbar # https://github.com/jazzband/django-debug-toolbar mock # mock objects for unit tests diff --git a/requirements-dev.txt b/requirements-dev.txt index 912bef8526..63a9e63bbe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ charset-normalizer==3.0.1 # requests click==8.1.3 # via pip-tools -coverage==5.1 +coverage==6.5 # via -r requirements-dev.in django==4.2.24 # via From bb5e4d2607753e8bbf16d41fd2511fcc9f93e4d9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 17:19:59 -0400 Subject: [PATCH 06/13] test clickhouse interval with offset --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8aa7186072..5869c0924d 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -599,7 +599,7 @@ def test_location_search(self): [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, ) self._assert_expected_search( - [SV_VARIANT1], locus={'rawItems': 'chr1:9292894-9369532'} + [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'} ) self._reset_search_families() From a8d1407abda74da9cb239e40396a46886a06a823 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 17:25:28 -0400 Subject: [PATCH 07/13] remove unreachable code --- clickhouse_search/managers.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7351b83809..8f78c87766 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1117,11 +1117,7 @@ def search_padded_interval(self, chrom, pos, padding): return self.filter(interval_q).result_values() @staticmethod - def _interval_query(chrom, start, end, offset=None, **kwargs): - if offset: - offset_pos = int((end - start) * offset) - start = max(start - offset_pos, MIN_POS) - end = min(end + offset_pos, MAX_POS) + def _interval_query(chrom, start, end, **kwargs): return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) def _filter_seqr_frequency(self, entries, ac=None, hh=None, **kwargs): From f10d03a80b6297dbdf0d011fef4254f1f4055ba7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 17:44:22 -0400 Subject: [PATCH 08/13] remove unused passthrough user email --- clickhouse_search/search.py | 18 ++++--------- .../management/commands/utils/__init__.py | 0 .../management/commands/utils/update_utils.py | 26 ------------------- .../check_for_new_samples_from_pipeline.py | 5 ++-- .../commands/reload_saved_variant_json.py | 2 +- seqr/utils/communication_utils.py | 3 +-- seqr/views/utils/variant_utils.py | 4 +-- 7 files changed, 11 insertions(+), 47 deletions(-) delete mode 100644 reference_data/management/commands/utils/__init__.py delete mode 100644 reference_data/management/commands/utils/update_utils.py diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fd8106fea8..061d2c63b0 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -189,15 +189,6 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, ]), } - elif results.has_annotation('secondary_genotypes'): - genotype_expressions = { - 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ - {1: ('secondary_familyGuids', 'has({value}, {field})')}, - ]), - 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ - {2: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, - ]), - } else: genotype_expressions = { 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ @@ -699,10 +690,11 @@ def delete_clickhouse_family(project, family_guid, dataset_type, sample_type=Non return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}' +SV_DATASET_TYPES = { + Sample.SAMPLE_TYPE_WGS: Sample.DATASET_TYPE_SV_CALLS, + Sample.SAMPLE_TYPE_WES: 'GCNV', +} def _clickhouse_dataset_type(dataset_type, sample_type): if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - if not sample_type: - raise ValueError('sample_type is required when dataset_type is SV') - if sample_type == Sample.SAMPLE_TYPE_WES: - return 'GCNV' + dataset_type = SV_DATASET_TYPES[sample_type] return dataset_type \ No newline at end of file diff --git a/reference_data/management/commands/utils/__init__.py b/reference_data/management/commands/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/reference_data/management/commands/utils/update_utils.py b/reference_data/management/commands/utils/update_utils.py deleted file mode 100644 index c94a654ca1..0000000000 --- a/reference_data/management/commands/utils/update_utils.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.core.management.base import BaseCommand - -from reference_data.utils.gene_utils import get_genes_by_id_and_symbol -from reference_data.models import GeneMetadataModel - - -class ReferenceDataHandler(object): - - model_cls = GeneMetadataModel - - def __init__(self, **kwargs): - gene_ids_to_gene, gene_symbols_to_gene = get_genes_by_id_and_symbol() - self.gene_reference = { - 'gene_symbols_to_gene': gene_symbols_to_gene, - 'gene_ids_to_gene': gene_ids_to_gene, - } - - def update_records(self, **kwargs): - self.model_cls.update_records(**self.gene_reference, **kwargs) - - -class GeneCommand(BaseCommand): - reference_data_handler = ReferenceDataHandler - - def handle(self, *args, **options): - self.reference_data_handler(**options).update_records() diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 27d38ed371..9111e6baf7 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -36,7 +36,6 @@ 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES', Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', } -USER_EMAIL = 'manage_command' RELATEDNESS_CHECK_NAME = 'relatedness_check' PDO_COPY_FIELDS = [ @@ -255,7 +254,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers # Reload saved variant JSON update_projects_saved_variant_json([ (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) + ], dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -390,7 +389,7 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) ) @classmethod - def _update_project_saved_variant_genotypes(cls, project_id, genome_version, user_email, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): + def _update_project_saved_variant_genotypes(cls, project_id, genome_version, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): updates = {} for family_guid in family_guids: variant_models_by_key = { diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 6d64f991e7..ce3e0edbab 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -30,5 +30,5 @@ def handle(self, *args, **options): family_ids = [family_guid] if family_guid else None project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, user_email='manage_command', update_function=update_project_saved_variant_json) + update_projects_saved_variant_json(project_list, update_function=update_project_saved_variant_json) logger.info("Done") diff --git a/seqr/utils/communication_utils.py b/seqr/utils/communication_utils.py index 7a57cfc173..128b73dd29 100644 --- a/seqr/utils/communication_utils.py +++ b/seqr/utils/communication_utils.py @@ -25,10 +25,9 @@ def _post_to_slack(channel, message): return None slack = Slacker(SLACK_TOKEN) - response = slack.chat.post_message( + slack.chat.post_message( channel, message, as_user=False, icon_emoji=":beaker:", username="Beaker (engineering-minion)", ) - return response.raw def send_welcome_email(user, referrer): diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a1663e3f65..96ca9f4a68 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -35,7 +35,7 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, user_email, update_function, **kwargs): +def update_projects_saved_variant_json(projects, update_function, **kwargs): success = {} skipped = {} error = {} @@ -43,7 +43,7 @@ def update_projects_saved_variant_json(projects, user_email, update_function, ** for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: updated_saved_variants = update_function( - project_id, genome_version, user_email=user_email, family_guids=family_guids, project_guid=project_guid, **kwargs) + project_id, genome_version, family_guids=family_guids, project_guid=project_guid, **kwargs) if updated_saved_variants is None: skipped[project_name] = True else: From b2159f4b8ae9ce411b3da72ceb547204e59a3dde Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 17:59:00 -0400 Subject: [PATCH 09/13] clean up unreachable tests --- seqr/utils/search/utils.py | 8 ----- seqr/views/apis/data_manager_api_tests.py | 3 -- seqr/views/apis/dataset_api_tests.py | 6 ---- seqr/views/apis/variant_search_api_tests.py | 35 +++++++-------------- 4 files changed, 12 insertions(+), 40 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2122c99e7c..02d0c5508f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -560,14 +560,6 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): - if offset: - offset_pos = int((end - start) * offset) - start = max(start - offset_pos, MIN_POS) - end = min(end + offset_pos, MAX_POS) - return [chrom, start, end] - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 035be59d6d..3bccba7935 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1274,9 +1274,6 @@ def _test_expected_vcf_responses(self, response, url): response = self.client.get(url, content_type='application/json') self.assertEqual(response.status_code, 200) - def _assert_expected_read_vcf_header_subprocess_calls(self, body): - return True - def test_validate_callset(self): url = reverse(validate_callset) self.check_pm_login(url) diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py index 091f3abc08..af8b8a2fe2 100644 --- a/seqr/views/apis/dataset_api_tests.py +++ b/seqr/views/apis/dataset_api_tests.py @@ -451,12 +451,6 @@ class LocalDatasetAPITest(AuthenticationTestCase, DatasetAPITest): fixtures = ['users', '1kg_project'] -def assert_no_anvil_calls(self): - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_not_called() - self.assert_no_extra_anvil_calls() - - # Test for permissions from AnVIL only class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): fixtures = ['users', 'social_auth', '1kg_project'] diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index e637e90bad..a434bddca0 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -327,20 +327,17 @@ def _assert_expected_rnaseq_response(self, response_json): 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] ) - def _assert_expected_results_family_context(self, response_json, locus_list_detail=False, skip_gene_context=False): - if not skip_gene_context: - self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) + def _assert_expected_results_family_context(self, response_json, locus_list_detail=False): + self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) family_fields = {'individualGuids'} family_fields.update(FAMILY_FIELDS) if len(response_json['familiesByGuid']) > 1: self.assertSetEqual(set(response_json['familiesByGuid']['F000002_2'].keys()), family_fields) - if not skip_gene_context: - family_fields.add('tpmGenes') + family_fields.add('tpmGenes') self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) - if not skip_gene_context: - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) + self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) self.assertEqual(len(response_json['individualsByGuid']), len(response_json['familiesByGuid'])*3) individual_fields = {'igvSampleGuids'} @@ -353,8 +350,7 @@ def _assert_expected_results_family_context(self, response_json, locus_list_deta self.assertEqual(len(response_json['familyNotesByGuid']), 3) self.assertSetEqual(set(response_json['familyNotesByGuid']['FAN000001_1'].keys()), FAMILY_NOTE_FIELDS) - if not skip_gene_context: - self._assert_expected_rnaseq_response(response_json) + self._assert_expected_rnaseq_response(response_json) def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True): gene_fields = {'locusListGuids'} @@ -869,7 +865,7 @@ def test_query_single_variant(self, mock_get_variant): self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['error'], 'Variant not found') - def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, no_metadata=False, **expected_response): + def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response): omit_fields = {'search', *(omit_fields or [])} expected_search_response = deepcopy({**self.EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) @@ -879,25 +875,18 @@ def _assert_expected_single_variant_results_context(self, response_json, omit_fi }) for k in omit_fields: expected_search_response.pop(k) - if no_metadata: - expected_search_response.update({k: {} for k in { - 'savedVariantsByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'genesById', - 'rnaSeqData', 'phenotypeGeneScores', 'mmeSubmissionsByGuid' - }}) - expected_search_response.pop('transcriptsById', None) - else: - expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') - expected_search_response['variantTagsByGuid'] = { - k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] - for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} - } + expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') + expected_search_response['variantTagsByGuid'] = { + k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] + for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} + } expected_search_response['variantNotesByGuid'] = {} expected_search_response['genesById'] = { k: v for k, v in expected_search_response['genesById'].items() if k in {'ENSG00000227232', 'ENSG00000268903'} } self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_family_context(response_json, locus_list_detail=True, skip_gene_context=no_metadata) + self._assert_expected_results_family_context(response_json, locus_list_detail=True) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1'}) From 77f61cec8fcb2ad47ba393a7261aae0a9027463c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 17 Sep 2025 18:03:51 -0400 Subject: [PATCH 10/13] clean up unused --- seqr/views/utils/variant_utils.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 96ca9f4a68..ebd3e0d9c8 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -68,7 +68,7 @@ def update_projects_saved_variant_json(projects, update_function, **kwargs): logger.info(f' {k}: {v}') -def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): +def get_saved_variants(genome_version, project_id=None, family_guids=None, clickhouse_dataset_type=None): saved_variants = SavedVariant.objects.filter( Q(saved_variant_json__genomeVersion__isnull=True) | Q(saved_variant_json__genomeVersion=genome_version.replace('GRCh', '')) @@ -77,24 +77,11 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, datas saved_variants = saved_variants.filter(family__project_id=project_id) if family_guids: saved_variants = saved_variants.filter(family__guid__in=family_guids) - if dataset_type: - saved_variants = saved_variants.filter(**saved_variants_dataset_type_filter(dataset_type)) elif clickhouse_dataset_type: saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) return saved_variants -def saved_variants_dataset_type_filter(dataset_type): - xpos_filter_key = 'xpos__gte' if dataset_type == Sample.DATASET_TYPE_MITO_CALLS else 'xpos__lt' - dataset_filter = {xpos_filter_key: get_xpos('M', 1)} - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - dataset_filter['alt__isnull'] = True - else: - # Filter out manual variants with invalid characters, such as those used for STRs - dataset_filter['alt__regex'] = '^[ACGT]+$' - return dataset_filter - - def parse_saved_variant_json(variant_json, family_id, variant_id=None,): if 'xpos' not in variant_json: variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos']) From f3ff552c5f51f0426b76975e6671140cbf585d64 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 10:10:13 -0400 Subject: [PATCH 11/13] fix conditional --- seqr/views/utils/variant_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index ebd3e0d9c8..1371b142de 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -77,7 +77,7 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, click saved_variants = saved_variants.filter(family__project_id=project_id) if family_guids: saved_variants = saved_variants.filter(family__guid__in=family_guids) - elif clickhouse_dataset_type: + if clickhouse_dataset_type: saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) return saved_variants From da126497d475ca267a27e584798c8caf8f11cb1a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 17:29:27 -0400 Subject: [PATCH 12/13] query discovery genes by correct dataset type --- seqr/views/utils/project_context_utils.py | 58 ++++++++++++++++------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 82121c696f..546aa2fb2c 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,9 +1,11 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models.functions import JSONObject -from clickhouse_search.search import get_transcripts_by_key +from clickhouse_search.search import get_transcripts_by_key, get_annotations_queryset from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ - VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote + VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample from seqr.utils.gene_utils import get_genes from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.utils import backend_specific_call @@ -119,22 +121,22 @@ def families_discovery_tags(families, genome_version, project=None): families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} - discovery_tags_by_key = {v.pop('key'): v for v in SavedVariant.objects.filter( + discovery_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ).values( - 'key', 'family__guid', selectedMainTranscriptId=F('selected_main_transcript_id'), - transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), - )} + ) try: - backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) + discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)( + discovery_variants, genome_version=genome_version, + ) except Exception as e: logger.error(f'Error loading discovery genes from clickhouse: {e}', None) + discovery_tags = [] gene_ids = set() - for tag in discovery_tags_by_key.values(): + for tag in discovery_tags: tag['transcripts'] = tag.get('transcripts') or {} gene_ids.update(list(tag['transcripts'].keys())) - families_by_guid[tag.pop('family__guid')]['discoveryTags'].append(tag) + families_by_guid[tag.pop('family_guid')]['discoveryTags'].append(tag) return { 'familiesByGuid': families_by_guid, @@ -142,15 +144,37 @@ def families_discovery_tags(families, genome_version, project=None): } -def _add_clickhouse_transcripts(discovery_tags_by_key, genome_version): - transcripts_by_key = get_transcripts_by_key(genome_version, discovery_tags_by_key.keys()) - for key, tag in discovery_tags_by_key.items(): - if key in transcripts_by_key: +def _get_no_key_tags(discovery_variants, **kwargs): + return discovery_variants.values( + family_guid=F('family__guid'), selectedMainTranscriptId=F('selected_main_transcript_id'), + transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), + ) + + +def _get_clickhouse_tags(discovery_variants, genome_version): + discovery_tags = list(_get_no_key_tags(discovery_variants.filter(key__isnull=True))) + + tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( + keys=ArrayAgg('key', distinct=True), + tags=ArrayAgg(JSONObject(key='key', family_guid='family__guid', selectedMainTranscriptId='selected_main_transcript_id')), + ) + + for dataset_type, keys, tags in tags_by_dataset_type.values_list('dataset_type', 'keys', 'tags'): + if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + transcripts_by_key = get_transcripts_by_key(genome_version, keys) + else: + qs = get_annotations_queryset(genome_version, dataset_type, keys) + transcripts_by_key = dict(qs.values_list('key', qs.transcript_field)) + for tag in tags: + key = tag.pop('key') tag['transcripts'] = transcripts_by_key[key] - tag['mainTranscriptId'] = next( + tag['mainTranscriptId'] = next(( t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts - if t['transcriptRank'] == 0 - ) + if t.get('transcriptRank') == 0 + ), None) + discovery_tags.append(tag) + + return discovery_tags MME_TAG_NAME = 'MME Submission' From 136be63f914fbc209fa04ab19c0d25851b095642 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 17:38:20 -0400 Subject: [PATCH 13/13] update tests --- seqr/views/apis/project_api_tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index b896e2368e..81d8c13f87 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -390,7 +390,7 @@ def test_project_families(self): gene_ids = self._assert_expected_project_families(url, response_keys) self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'}) - def _assert_expected_project_families(self, url, response_keys): + def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False): response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -436,12 +436,14 @@ def _assert_expected_project_families(self, url, response_keys): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) - self.assertListEqual(family_1['discoveryTags'], [{ + family_1_tags = [] if no_discovery_tags else [{ 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, 'mainTranscriptId': 'ENST00000258436', 'selectedMainTranscriptId': None, - }]) - self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG]) + }] + self.assertListEqual(family_1['discoveryTags'], family_1_tags) + family_2_tags = [] if no_discovery_tags else [self.DISCOVERY_TAG] + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], family_2_tags) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families @@ -802,8 +804,8 @@ def _assert_expected_project_families(self, *args, **kwargs): self.reset_logs() connections['clickhouse'].close() self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}} - no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) - self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) + no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True) + self.assertSetEqual(no_clickhouse_gene_ids, set()) self.assert_json_logs(None, [ ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { 'severity': 'ERROR',